diff --git a/CHANGES b/CHANGES index 3d7118c750..31fddce833 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,48 @@ +2.2-9 | 2013-11-18 14:03:21 -0800 + + * Update local.bro for Bro >= 2.2. The commented out Notice::policy + example didn't work anymore. (Daniel Thayer) + +2.2-6 | 2013-11-15 07:05:15 -0800 + + * Make "install-example-configs" target use DESTDIR. (Jon Siwek) + +2.2-5 | 2013-11-11 13:47:54 -0800 + + * Fix the irc_reply event for certain server message types. (Seth + Hall) + + * Fixed Segmentation fault in SQLite Writer. (Jon Crussell) + +2.2 | 2013-11-07 10:25:50 -0800 + + * Release 2.2. + + * Removing location information from ssh.log in external tests. + (Robin Sommer) + +2.2-beta-199 | 2013-11-07 00:36:46 -0800 + + * Fixing warnings during doc build. (Robin Sommer) + +2.2-beta-198 | 2013-11-06 22:54:30 -0800 + + * Update docs and tests for a recent change to detect-MHR.bro + (Daniel Thayer) + + * Update tests and baselines for sumstats docs. (Daniel Thayer) + +2.2-beta-194 | 2013-11-06 14:39:50 -0500 + + * Remove resp_size from the ssh log. Refactor when we write out to + the log a bit. Geodata now works reliably. (Vlad Grigorescu) + + * Update VirusTotal URL to work with changes to their website and + changed it to a redef. (Vlad Grigorescu) + + * Added a document for the SumStats framework. (Seth Hall) + 2.2-beta-184 | 2013-11-03 22:53:42 -0800 * Remove swig-ruby from required packages section of install doc. @@ -31,7 +75,7 @@ (Vlad Grigorescu) * New version of the threading queue deadlock fix. (Robin Sommer) - + * Updating README with download/git information. (Robin Sommer) 2.2-beta-161 | 2013-10-25 15:48:15 -0700 @@ -60,7 +104,7 @@ 2.2-beta-152 | 2013-10-24 18:16:49 -0700 * Fix for input readers occasionally dead-locking. (Robin Sommer) - + 2.2-beta-151 | 2013-10-24 16:52:26 -0700 * Updating submodule(s). @@ -80,7 +124,7 @@ (Daniel Thayer) * Intel framework notes added to NEWS. (Seth Hall) - + * Temporary OSX Mavericks libc++ issue workaround for getline() problem in ASCII reader. (Bernhard Amann) @@ -96,16 +140,16 @@ 2.2-beta-133 | 2013-10-23 09:50:16 -0700 * Fix record coercion tolerance of optional fields. (Jon Siwek) - + * Add NEWS about incompatible local.bro changes, addresses BIT-1047. (Jon Siwek) * Fix minor formatting problem in NEWS. (Jon Siwek) - + 2.2-beta-129 | 2013-10-23 09:47:29 -0700 * Another batch of documentation fixes and updates. (Daniel Thayer) - + 2.2-beta-114 | 2013-10-18 14:17:57 -0700 * Moving the SQLite examples into separate Bro files to turn them @@ -114,7 +158,7 @@ 2.2-beta-112 | 2013-10-18 13:47:13 -0700 * A larger chunk of documentation fixes and cleanup. (Daniel Thayer) - + Apart from many smaller improves this includes in particular: * Add README files for most Bro frameworks and base/protocols. @@ -148,7 +192,7 @@ 2.2-beta-68 | 2013-10-14 09:26:09 -0700 * Add check for curl command to active-http.test. (Daniel Thayer) - + 2.2-beta-64 | 2013-10-14 09:20:04 -0700 * Review usage of Reporter::InternalError, addresses BIT-1045. @@ -156,7 +200,7 @@ Replaced some with InternalWarning or AnalyzerError, the later being a new method which signals the analyzer to not process further input. (Jon Siwek) - + * Add new event for TCP content file write failures: "contents_file_write_failure". (Jon Siwek) @@ -167,9 +211,9 @@ 2.2-beta-55 | 2013-10-10 13:36:38 -0700 * A couple of new TLS extension numbers. (Bernhard Amann) - + * Suport for three more new TLS ciphers. (Bernhard Amann) - + * Removing ICSI notary from default site config. (Robin Sommer) 2.2-beta-51 | 2013-10-07 17:33:56 -0700 @@ -178,9 +222,9 @@ (Robin Sommer) * Fixing the historical CHANGES record. (Robin Sommer) - + * Updating copyright notice. (Robin Sommer) - + 2.2-beta-38 | 2013-10-02 11:03:29 -0700 * Fix uninitialized (or unused) fields. (Jon Siwek) @@ -190,31 +234,31 @@ * Remove dead/unfinished code in unary not expression. (Jon Siwek) * Fix logic for failed DNS TXT lookups. (Jon Siwek) - + * A couple null ptr checks. (Jon Siwek) - + * Improve return value checking and error handling. (Jon Siwek) - + * Remove unused variable assignments. (Jon Siwek) - + * Prevent division/modulo by zero in scripts. (Jon Siwek) - + * Fix unintentional always-false condition. (Jon Siwek) - + * Fix invalidated iterator usage. (Jon Siwek) * Fix DNS_Mgr iterator mismatch. (Jon Siwek) - + * Set safe umask when creating script profiler tmp files. (Jon Siwek) * Fix nesting/indent level whitespace mismatch. (Jon Siwek) - + * Add checks to avoid improper negative values use. (Jon Siwek) - + 2.2-beta-18 | 2013-10-02 10:28:17 -0700 * Add support for further TLS cipher suites. (Bernhard Amann) - + 2.2-beta-13 | 2013-10-01 11:31:55 -0700 * Updating bifcl usage message. (Robin Sommer) @@ -228,7 +272,7 @@ 2.2-beta-4 | 2013-09-24 13:23:30 -0700 * Fix for setting REPO in Makefile. (Robin Sommer) - + * Whitespace fix. (Robin Sommer) * Removing :doc: roles so that we can render this with docutils @@ -245,9 +289,9 @@ * Updating NEWS. (Robin Sommer) * Fixing an always false condition. (Robin Sommer) - + * Fix required for compiling with clang 3.3. (Robin Sommer) - + 2.1-1377 | 2013-09-20 14:38:15 -0700 * Updates to the scripting introduction. (Scott Runnels) @@ -262,7 +306,7 @@ 2.1-1364 | 2013-09-19 15:12:08 -0700 * Add links to Intelligence Framework documentation. (Daniel Thayer) - + * Update Mozilla root CA list. (Bernhard Amann, Jon Siwek) * Update documentation of required packages. (Daniel Thayer) @@ -282,27 +326,27 @@ 2.1-1352 | 2013-09-18 14:42:28 -0700 * Fix a number of compiler warnings. (Daniel Thayer) - + * Fix cmake warning about ENABLE_PERFTOOLS not being used. (Daniel Thayer) 2.1-1344 | 2013-09-16 16:20:55 -0500 * Refactor Analyzer::AddChildAnalyzer and usages. (Jon Siwek) - + * Minor refactor to SSL BinPAC grammer. (Jon Siwek) * Minor refactor to Broxygen enum comments. (Jon Siwek) - + * Fix possible (unlikely) use of uninitialized value. (Jon Siwek) - + * Fix/improve dereference-before-null-checks. (Jon Siwek) - + * Fix out-of-bounds memory accesses, and remove a variable-length-array usage. (Jon Siwek) * Fix potential mem leak. (Jon Siwek) - + * Fix double-free and deallocator mismatch. (Jon Siwek) * Fix another function val reference counting bug. (Jon Siwek) @@ -333,7 +377,7 @@ * Reorganized and signifcantly extended documentation. This includes two new chapters contributed by Scott Runnels. - + 2.1-1216 | 2013-08-31 10:39:40 -0700 @@ -351,25 +395,25 @@ 2.1-1154 | 2013-08-30 08:27:45 -0700 * Fix global opaque val segfault. Addresses BIT-1071. (Jon Siwek) - + * Fix malloc/delete mismatch. (Jon Siwek) - + * Fix invalid pointer dereference in AsciiFormatter. (Jon Siwek) 2.1-1150 | 2013-08-29 13:43:01 -0700 * Fix input framework memory leaks. (Jon Siwek) - + * Fix memory leak in SOCKS analyzer for bad addr types. (Jon Siwek) - + * Fix Bloom filter memory leaks. (Jon Siwek) 2.1-1144 | 2013-08-28 18:51:06 -0700 * Add bits_per_uid unit test. Addresses BIT-1016. (Jon Siwek) - + * UID optimizations. Addresses BIT-1016. (Jon Siwek) - + * Added a $unique_max field to Reducers for the SumStats::UNIQUE calculation, and using the new option in scan.bro and the FTP bruteforce detection. (Seth Hall) @@ -382,11 +426,11 @@ 2.1-1135 | 2013-08-27 12:16:26 -0700 * More SumStats fixes. (Seth Hall) - + * Increase UIDs to 96 bits. (Jon Siwek) - + - The bit-length is adjustable via redef'ing bits_per_uid. - + - Prefix 'C' is added to connection UIDS (including IP tunnels) and 'F' to files. @@ -395,9 +439,9 @@ 2.1-1128 | 2013-08-24 10:27:29 -0700 * Remove code relict in input framework. (Jon Siwek) - + * Fix documentation for mkdir BIF. (Jon Siwek) - + * File extraction tweaks. (Jon Siwek) - Default extraction limit of 100MB now provided via a tuning @@ -411,11 +455,11 @@ 2.1-1124 | 2013-08-23 16:33:52 -0700 * Fixed a number of object bugs DNP3 analyzer. (Hui Lin) - + 2.1-1122 | 2013-08-22 16:52:27 -0700 * Use macros to create file analyzer plugin classes. (Jon Siwek) - + * Add options to limit extracted file sizes w/ 100MB default. (Jon Siwek) @@ -425,13 +469,13 @@ improvements. (Jon Siwek) * Make memory leak tests able to time out. (Jon Siwek) - + * Fix a compiler warning regarding strncat misuse. (Jon Siwek) - + 2.1-1103 | 2013-08-21 19:11:34 -0400 * A number of sumstats fixes. (Seth Hall, Vlad Grigorescu) - + * Fix memory leak w/ when statements. Addresses BIT-1058. (Jon Siwek) @@ -462,12 +506,12 @@ turning them into events. (Seth Hall) * Fixing intel framework tests. (Seth Hall) - + 2.1-1059 | 2013-08-13 23:52:41 -0400 * Add file name support to intel framework. (Seth Hall) - * Add file support to intel framework and slightly restructure + * Add file support to intel framework and slightly restructure intel http handling. (Seth Hall) 2.1-1052 | 2013-08-12 14:38:14 -0700 @@ -489,9 +533,9 @@ 2.1-1039 | 2013-08-09 15:30:15 -0700 * Fix mem leak in DHCP analyzer. (Jon Siwek) - + * Fix a unit test outdated by recent sumstats changes. (Jon Siwek) - + 2.1-1036 | 2013-08-05 17:29:11 -0400 * Fix the SSL infinite loop I just created. (Seth Hall) @@ -546,7 +590,7 @@ 2.1-1009 | 2013-08-02 17:19:08 -0700 * A number of exec module and raw input reader fixes. (Jon Siwek) - + 2.1-1007 | 2013-08-01 15:41:54 -0700 * More function documentation. (Bernhard Amann) @@ -618,11 +662,11 @@ compressed log representation. (Seth Hall) * Added mime types to http.log (Seth Hall) - + * Add jar files to the default MHR lookups. (Seth Hall) - + * Adding CAB files for MHR checking. (Seth Hall) - + * Improve malware hash registry script. - Include a link to a virustotal search in the notice sub message field. @@ -655,15 +699,15 @@ * Updates for the Intel Framework. (Seth Hall) - - policy/frameworks/intel/seen is the new location for the + - policy/frameworks/intel/seen is the new location for the scripts that push data into the intel framework for checking. - - The new policy/frameworks/intel/do_notice script adds an + - The new policy/frameworks/intel/do_notice script adds an example mechanism for data driven notices. - - Remove the Intel insertion after heuristically detecting SSH + - Remove the Intel insertion after heuristically detecting SSH bruteforcing. - + - Intel importing format has changed (refer to docs). - All string matching is now case insensitive. @@ -724,7 +768,7 @@ make it deterministic. (Robin Sommer) * Small raw reader tweaks that got left our earlier. (Robin Sommer) - + 2.1-814 | 2013-07-15 18:18:20 -0700 * Fixing raw reader crash when accessing nonexistant file, and @@ -850,12 +894,12 @@ input data on to the file analysis framework. (Jon Siwek) * File analysis framework interface simplifications. (Jon Siwek) - + - Remove script-layer data input interface (will be managed directly by input framework later). - Only track files internally by file id hash. Chance of collision - too small to justify also tracking unique file string. + too small to justify also tracking unique file string. 2.1-741 | 2013-06-07 17:28:50 -0700 @@ -906,14 +950,14 @@ 2.1-659 | 2013-05-24 17:24:18 -0700 * Fix broken/missing documentation. (Jon Siwek) - + * Fixing test that would fail without ES/curl support. (Robin Sommer) 2.1-656 | 2013-05-17 15:58:07 -0700 * Fix mutex lock problem for writers. (Bernhard Amann) - + 2.1-654 | 2013-05-17 13:49:52 -0700 * Tweaks to sqlite3 configuration to address threading issues. @@ -931,9 +975,9 @@ 2.1-647 | 2013-05-17 07:47:14 -0700 * Fixing Broxygen generation to have BROMAGIC set. (Robin Sommer) - + * Fix for 'fchmod undeclared here' on FreeBSD. (Robin Sommer) - + * CMake policy fix to avoid errors with older versions. (Robin Sommer) @@ -1046,7 +1090,7 @@ 2.1-386 | 2013-03-22 12:41:50 -0700 * Added reverse() function to strings.bif. (Yun Zheng Hu) - + 2.1-384 | 2013-03-22 12:10:14 -0700 * Fix record constructors in table initializer indices. Addresses @@ -1055,16 +1099,16 @@ 2.1-382 | 2013-03-22 12:01:34 -0700 * Add support for 802.1ah (Q-in-Q). Addresses #641. (Seth Hall) - + 2.1-380 | 2013-03-18 12:18:10 -0700 * Fix gcc compile warnings in base64 encoder and benchmark reader. (Bernhard Amann) - + 2.1-377 | 2013-03-17 17:36:09 -0700 * Fixing potential leak in DNS error case. (Vlad Grigorescu) - + 2.1-375 | 2013-03-17 13:14:26 -0700 * Add base64 encoding functionality, including new BiFs @@ -1076,14 +1120,14 @@ * Adding a test for extract-certs-pem.pem. (Robin Sommer) * Renaming Base64Decoder to Base64Converter. (Robin Sommer) - + 2.1-366 | 2013-03-17 12:35:59 -0700 * Correctly handle DNS lookups for software version ranges. (Seth Hall) * Improvements to vulnerable software detection. (Seth Hall) - + - Add a DNS based updating method. This needs to be tested still. @@ -1117,9 +1161,9 @@ 2.1-351 | 2013-03-07 13:27:29 -0800 * Fix new/delete mismatch. Addresses #958. (Jacob Baines) - + * Fix compiler warnings. (Jon Siwek) - + 2.1-347 | 2013-03-06 16:48:44 -0800 * Remove unused parameter from vector assignment method. (Bernhard Amann) @@ -1168,9 +1212,9 @@ 2.1-328 | 2013-02-05 01:34:29 -0500 - * New script to query the ICSI Certificate Notary + * New script to query the ICSI Certificate Notary (http://notary.icsi.berkeley.edu/) over DNS and add information - to the SSL log at runtime. (Matthias Vallentin) + to the SSL log at runtime. (Matthias Vallentin) * Add delayed logging to SSL base scripts. (Matthias Vallentin) @@ -1221,7 +1265,7 @@ * Changing test=suite's btest call to use "-j" instead of "-j 5". (Robin Sommer) - + * Require "case" blocks to end with either "break", "return", or a new "fallthrough" statement that passes control on to the subsequent case. This gives us the best mix of safety, @@ -1237,7 +1281,7 @@ ElasticSearch writer. (Gilbert Clark) * Removing unused class member. (Robin Sommer) - + * Add opaque type-ignoring for the accept_unsupported_types input framework option. (Bernhard Amann) @@ -1282,7 +1326,7 @@ sha256_*, and entropy_*, respectively. Note that these functions have changed their signatures to work with opaques types rather than global state as it was before. - + 2.1-240 | 2012-12-20 15:21:07 -0800 * Improve error for invalid use of types as values. Addresses #923. @@ -1407,7 +1451,7 @@ 2.1-195 | 2012-12-03 14:50:33 -0800 * Catching out-of-memory in patricia tree code. (Bill Parker) - + 2.1-194 | 2012-12-03 14:36:26 -0800 * Renaming ASCII writer filter option 'only_single_header_row' to @@ -1468,7 +1512,7 @@ Hall) * Adding NEWS placeholder for hooks and CSV mode. (Robin Sommer) - + 2.1-178 | 2012-11-23 19:35:32 -0800 * The ASCII writer now supports a new filter config option @@ -1523,7 +1567,7 @@ 2.1-112 | 2012-11-05 13:58:20 -0800 - * New base script for detecting cases of checksum offloading. + * New base script for detecting cases of checksum offloading. Reporter messages will now tell if one has bad checksums. (Seth Hall) @@ -1533,9 +1577,9 @@ 2.1-109 | 2012-11-05 13:39:34 -0800 * Add detection rate threshold for MHR. (Vlad Grigorescu) - + * lookup_hostname_txt fixes. (Vlad Grigorescu) - + 2.1-104 | 2012-11-01 10:37:50 -0700 * A new built-in function lookup_hostname_txt() provides support for @@ -1660,7 +1704,7 @@ Addresses #877. (Jon Siwek) * Add --with-curl option to ./configure. Addresses #877. (Jon Siwek) - + 2.1-61 | 2012-10-12 09:32:48 -0700 * Fix bug in the input framework: the config table did not work. @@ -1703,7 +1747,7 @@ * Remove deprecated script functionality (see NEWS for details). (Daniel Thayer) - + 2.1-39 | 2012-09-29 14:09:16 -0700 * Reliability adjustments to istate tests with network @@ -1715,7 +1759,7 @@ an error. (Daniel Thayer) * Fix parsing of large integers on 32-bit systems. (Daniel Thayer) - + * Serialize language.when unit test with the "comm" group. (Jon Siwek) @@ -1726,7 +1770,7 @@ 2.1-26 | 2012-09-23 08:46:03 -0700 * Add an item to FAQ page about broctl options. (Daniel Thayer) - + * Add more language tests. We now have tests of all built-in Bro data types (including different representations of constant values, and max./min. values), keywords, and operators (including @@ -1749,7 +1793,7 @@ * Adjusting some unit tests that do cluster communication. (Jon Siwek) * Small change to non-blocking DNS initialization. (Jon Siwek) - + * Reorder a few statements in scan.l to make 1.5msecs etc work. Adresses #872. (Bernhard Amann) @@ -1781,9 +1825,9 @@ Siwek) * Parse 64-bit consts in Bro scripts correctly. (Bernhard Amann) - + * Output 64-bit counts correctly on 32-bit machines (Bernhard Amann) - + * Input framework fixes, including: (Bernhard Amann) - One of the change events got the wrong parameters. @@ -1797,7 +1841,7 @@ - Hashing of lines just containing zero-length-strings was broken. - Make set_separators different from , work for input framework. - + - Input framework was not handling counts and ints out of 32-bit-range correctly. @@ -1805,20 +1849,20 @@ the line, log it, and continue. * Update documentation for builtin types. (Daniel Thayer) - + - Add missing description of interval "msec" unit. - + - Improved description of pattern by clarifying the issue of operand order and difference between exact and embedded matching. * Documentation fixes for signature 'eval' conditions. (Jon Siwek) - + * Remove orphaned 1.5 unit tests. (Jon Siwek) * Add type checking for signature 'eval' condition functions. (Jon Siwek) - + * Adding an identifier to the SMTP blocklist notices for duplicate suppression. (Seth Hall) @@ -1849,7 +1893,7 @@ 2.1-beta-31 | 2012-08-21 15:46:05 -0700 * Tweak to rotate-custom.bro unit test. (Jon Siwek) - + * Ignore small mem leak every rotation interval for dataseries logs. (Jon Siwek) @@ -1904,13 +1948,13 @@ 2.1-beta-6 | 2012-08-10 12:22:52 -0700 * Fix bug in input framework with an edge case. (Bernhard Amann) - + * Fix small bug in input framework test script. (Bernhard Amann) - + 2.1-beta-3 | 2012-08-03 10:46:49 -0700 * Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer) - + * Fix configure script to exit with non-zero status on error (Jon Siwek) @@ -1961,7 +2005,7 @@ * Input framework: Make want_record=T the default for events (Bernhard Amann) - + * Changing the start/end markers in logs to open/close now reflecting wall clock. (Robin Sommer) @@ -1972,16 +2016,16 @@ * New test for input framework that fails to find a file. (Robin Sommer) - + * Improving error handling for threads. (Robin Sommer) - + * Tweaking the custom-rotate test to produce stable output. (Robin Sommer) 2.0-884 | 2012-07-26 14:33:21 -0700 * Add comprehensive error handling for close() calls. (Jon Siwek) - + * Add more test cases for input framework. (Bernhard Amann) * Input framework: make error output for non-matching event types @@ -1990,14 +2034,14 @@ 2.0-877 | 2012-07-25 17:20:34 -0700 * Fix double close() in FilerSerializer class. (Jon Siwek) - + * Fix build warnings. (Daniel Thayer) * Fixes to ElasticSearch plugin to make libcurl handle http responses correctly. (Seth Hall) * Fixing FreeBSD compiler error. (Robin Sommer) - + * Silencing compiler warnings. (Robin Sommer) 2.0-871 | 2012-07-25 13:08:00 -0700 @@ -2016,7 +2060,7 @@ 2.0-866 | 2012-07-24 16:02:07 -0700 * Correct a typo in usage message. (Daniel Thayer) - + * Fix file permissions of log files (which were created with execute permissions after a recent change). (Daniel Thayer) @@ -4688,7 +4732,7 @@ away. (Robin Sommer) - Smarter way to increase the communication module's pipe's socket - buffer size, resulting in a value closer to the allowed maximum. + buffer size, resulting in a value closer to the allowed maximum. (Craig Leres) - BroControl now also maintains links from the log archive to the @@ -4731,7 +4775,7 @@ - http-header.bro now includes a global "include_header: set[string]" If it contains any strings, then only those headers will be processed. If left empty, then you continue to get the current behavior of processing all - headers. (Robin Sommer). + headers. (Robin Sommer). - Several changes to drop.bro (Robin Sommer): diff --git a/NEWS b/NEWS index 1b6f3114d0..4ae5669466 100644 --- a/NEWS +++ b/NEWS @@ -4,8 +4,8 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as BroControl and Broccoli, come with their own ``CHANGES``.) -Bro 2.2 Beta -============ +Bro 2.2 +======= New Functionality ----------------- diff --git a/VERSION b/VERSION index 24723a69bb..ac3e73a062 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-beta-184 +2.2-9 diff --git a/aux/binpac b/aux/binpac index 0f20a50afa..54b321009b 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 0f20a50afacb68154b4035b6da63164d154093e4 +Subproject commit 54b321009b750268526419bdbd841f421c839313 diff --git a/aux/bro-aux b/aux/bro-aux index ce8a9733c4..ebf9c0d88a 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit ce8a9733c4f8c4c8e8b2e9e440acfcf985f39cd8 +Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8 diff --git a/aux/broccoli b/aux/broccoli index 02e5c1e1f9..17ec437752 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 02e5c1e1f993ef0fea3e2a59c34df9f40839e398 +Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d diff --git a/aux/broctl b/aux/broctl index aeb8501a50..6e01d6972f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit aeb8501a50dcf33c53e7fe776b6e333327c11861 +Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f diff --git a/aux/btest b/aux/btest index cfc8fe7ddf..26c3136d56 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit cfc8fe7ddf5ba3a9f957d1d5a98e9cfe1e9692ac +Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9 diff --git a/cmake b/cmake index d902e23fd1..e7a46cb82e 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit d902e23fd14624eb9caf0b4a0e693014bf5bd684 +Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20 diff --git a/doc/frameworks/index.rst b/doc/frameworks/index.rst index d5b771b15e..f8c681d795 100644 --- a/doc/frameworks/index.rst +++ b/doc/frameworks/index.rst @@ -13,4 +13,5 @@ Frameworks logging notice signatures + sumstats diff --git a/doc/frameworks/sumstats-countconns.bro b/doc/frameworks/sumstats-countconns.bro new file mode 100644 index 0000000000..a10be54376 --- /dev/null +++ b/doc/frameworks/sumstats-countconns.bro @@ -0,0 +1,36 @@ +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } \ No newline at end of file diff --git a/doc/frameworks/sumstats-toy-scan.bro b/doc/frameworks/sumstats-toy-scan.bro new file mode 100644 index 0000000000..8e035142d7 --- /dev/null +++ b/doc/frameworks/sumstats-toy-scan.bro @@ -0,0 +1,45 @@ +@load base/frameworks/sumstats + +# We use the connection_attempt event to limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is provided for when a key crosses the threshold. + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } diff --git a/doc/frameworks/sumstats.rst b/doc/frameworks/sumstats.rst new file mode 100644 index 0000000000..057cc87fc1 --- /dev/null +++ b/doc/frameworks/sumstats.rst @@ -0,0 +1,102 @@ +================== +Summary Statistics +================== + +.. rst-class:: opening + + Measuring aspects of network traffic is an extremely common task in Bro. + Bro provides data structures which make this very easy as well in + simplistic cases such as size limited trace file processing. In + real-world deployments though, there are difficulties that arise from + clusterization (many processes sniffing traffic) and unbounded data sets + (traffic never stops). The Summary Statistics (otherwise referred to as + SumStats) framework aims to define a mechanism for consuming unbounded + data sets and making them measurable in practice on large clustered and + non-clustered Bro deployments. + +.. contents:: + +Overview +======== + +The Sumstat processing flow is broken into three pieces. Observations, where +some aspect of an event is observed and fed into the Sumstats framework. +Reducers, where observations are collected and measured, typically by taking +some sort of summary statistic measurement like average or variance (among +others). Sumstats, where reducers have an epoch (time interval) that their +measurements are performed over along with callbacks for monitoring thresholds +or viewing the collected and measured data. + +Terminology +=========== + + Observation + + A single point of data. Observations have a few components of their + own. They are part of an arbitrarily named observation stream, they + have a key that is something the observation is about, and the actual + observation itself. + + Reducer + + Calculations are applied to an observation stream here to reduce the + full unbounded set of observations down to a smaller representation. + Results are collected within each reducer per-key so care must be + taken to keep the total number of keys tracked down to a reasonable + level. + + Sumstat + + The final definition of a Sumstat where one or more reducers is + collected over an interval, also known as an epoch. Thresholding can + be applied here along with a callback in the event that a threshold is + crossed. Additionally, a callback can be provided to access each + result (per-key) at the end of each epoch. + +Examples +======== + +These examples may seem very simple to an experienced Bro script developer and +they're intended to look that way. Keep in mind that these scripts will work +on small single process Bro instances as well as large many-worker clusters. +The complications from dealing with flow based load balancing can be ignored +by developers writing scripts that use Sumstats due to its built-in cluster +transparency. + +Printing the number of connections +---------------------------------- + +Sumstats provides a simple way of approaching the problem of trying to count +the number of connections over a given time interval. Here is a script with +inline documentation that does this with the Sumstats framework: + +.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-countconns.bro + +When run on a sample PCAP file from the Bro test suite, the following output +is created: + +.. btest:: sumstats-countconns + + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro + + +Toy scan detection +------------------ + +Taking the previous example even further, we can implement a simple detection +to demonstrate the thresholding functionality. This example is a toy to +demonstrate how thresholding works in Sumstats and is not meant to be a +real-world functional example, that is left to the +:doc:`/scripts/policy/misc/scan` script that is included with Bro. + +.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro + +Let's see if there are any hosts that crossed the threshold in a PCAP file +containing a host running nmap: + +.. btest:: sumstats-toy-scan + + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro + +It seems the host running nmap was detected! + diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index 2118e73715..e42aa55e2c 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -66,7 +66,7 @@ are ensuring the Files framework, the Notice framework and the script to hash al been loaded by Bro. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 10-31 + :lines: 10-36 The export section redefines an enumerable constant that describes the type of notice we will generate with the Notice framework. Bro @@ -87,7 +87,7 @@ Up until this point, the script has merely done some basic setup. With the next the script starts to define instructions to take in a given event. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 33-57 + :lines: 38-62 The workhorse of the script is contained in the event handler for ``file_hash``. The :bro:see:`file_hash` event allows scripts to access diff --git a/doc/scripts/notices.rst b/doc/scripts/notices.rst new file mode 100644 index 0000000000..412cd58509 --- /dev/null +++ b/doc/scripts/notices.rst @@ -0,0 +1,8 @@ + +.. Not nice but I don't find a way to link to the notice index +.. directly from the upper level TOC tree. + +Notices +======= + +See the `Bro Notice Index <../bro-noticeindex.html>`_. diff --git a/scripts/base/frameworks/packet-filter/cluster.bro b/scripts/base/frameworks/packet-filter/cluster.bro index 34f0600d18..6e41a6045f 100644 --- a/scripts/base/frameworks/packet-filter/cluster.bro +++ b/scripts/base/frameworks/packet-filter/cluster.bro @@ -1,4 +1,7 @@ +@load base/frameworks/cluster +@load ./main + module PacketFilter; event remote_connection_handshake_done(p: event_peer) &priority=3 diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 5b452d432c..33b0c84147 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -37,12 +37,6 @@ export { client: string &log &optional; ## Software string from the server. server: string &log &optional; - ## Amount of data returned from the server. This is currently - ## the only measure of the success heuristic and it is logged to - ## assist analysts looking at the logs to make their own - ## determination about the success on a case-by-case basis. - resp_size: count &log &default=0; - ## Indicate if the SSH session is done being watched. done: bool &default=F; }; @@ -107,10 +101,10 @@ function check_ssh_connection(c: connection, done: bool) # this matches the conditions for a failed login. Failed # logins are only detected at connection state removal. - if ( # Require originators to have sent at least 50 bytes. - c$orig$size > 50 && + if ( # Require originators and responders to have sent at least 50 bytes. + c$orig$size > 50 && c$resp$size > 50 && # Responders must be below 4000 bytes. - c$resp$size < 4000 && + c$resp$size < authentication_data_size && # Responder must have sent fewer than 40 packets. c$resp$num_pkts < 40 && # If there was a content gap we can't reliably do this heuristic. @@ -122,7 +116,7 @@ function check_ssh_connection(c: connection, done: bool) event SSH::heuristic_failed_login(c); } - if ( c$resp$size > authentication_data_size ) + if ( c$resp$size >= authentication_data_size ) { c$ssh$status = "success"; event SSH::heuristic_successful_login(c); @@ -132,7 +126,7 @@ function check_ssh_connection(c: connection, done: bool) { # If this connection is still being tracked, then it's possible # to watch for it to be a successful connection. - if ( c$resp$size > authentication_data_size ) + if ( c$resp$size >= authentication_data_size ) { c$ssh$status = "success"; event SSH::heuristic_successful_login(c); @@ -150,8 +144,6 @@ function check_ssh_connection(c: connection, done: bool) # after detection is done. c$ssh$done=T; - Log::write(SSH::LOG, c$ssh); - if ( skip_processing_after_detection ) { # Stop watching this connection, we don't care about it anymore. @@ -161,10 +153,24 @@ function check_ssh_connection(c: connection, done: bool) } +event heuristic_successful_login(c: connection) &priority=-5 + { + Log::write(SSH::LOG, c$ssh); + } + +event heuristic_failed_login(c: connection) &priority=-5 + { + Log::write(SSH::LOG, c$ssh); + } + event connection_state_remove(c: connection) &priority=-5 { if ( c?$ssh ) + { check_ssh_connection(c, T); + if ( c$ssh$status == "undetermined" ) + Log::write(SSH::LOG, c$ssh); + } } event ssh_watcher(c: connection) diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro index 5ed8715c9b..1a26b9be32 100644 --- a/scripts/policy/frameworks/files/detect-MHR.bro +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -23,6 +23,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold @@ -48,7 +53,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro index 3abc19d337..a5fed986ef 100644 --- a/scripts/policy/protocols/ssh/geo-data.bro +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -24,21 +24,29 @@ export { const watched_countries: set[string] = {"RO"} &redef; } +function get_location(c: connection): geo_location + { + local lookup_ip = (c$ssh$direction == OUTBOUND) ? c$id$resp_h : c$id$orig_h; + return lookup_location(lookup_ip); + } + event SSH::heuristic_successful_login(c: connection) &priority=5 { - local location: geo_location; - location = (c$ssh$direction == OUTBOUND) ? - lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h); - # Add the location data to the SSH record. - c$ssh$remote_location = location; + c$ssh$remote_location = get_location(c); - if ( location?$country_code && location$country_code in watched_countries ) + if ( c$ssh$remote_location?$country_code && c$ssh$remote_location$country_code in watched_countries ) { NOTICE([$note=Watched_Country_Login, $conn=c, $msg=fmt("SSH login %s watched country: %s", (c$ssh$direction == OUTBOUND) ? "to" : "from", - location$country_code)]); + c$ssh$remote_location$country_code)]); } } + +event SSH::heuristic_failed_login(c: connection) &priority=5 + { + # Add the location data to the SSH record. + c$ssh$remote_location = get_location(c); + } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 2f4eed9865..ddaee42a93 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -30,10 +30,6 @@ # This adds signatures to detect cleartext forward and reverse windows shells. @load-sigs frameworks/signatures/detect-windows-shells -# Uncomment the following line to begin receiving (by default hourly) emails -# containing all of your notices. -# redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 0] }; - # Load all of the scripts that detect software in various protocols. @load protocols/ftp/software @load protocols/smtp/software diff --git a/src/analyzer/protocol/irc/IRC.cc b/src/analyzer/protocol/irc/IRC.cc index 71f591635f..d621ce2cce 100644 --- a/src/analyzer/protocol/irc/IRC.cc +++ b/src/analyzer/protocol/irc/IRC.cc @@ -164,7 +164,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) if ( code > 0 ) { switch ( code ) { - // Ignore unimportant messages. + /* case 1: // RPL_WELCOME case 2: // RPL_YOURHOST case 3: // RPL_CREATED @@ -183,6 +183,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) case 376: // RPL_ENDOFMOTD case 331: // RPL_NOTOPIC break; + */ // Count of users, services and servers in whole network. case 251: diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 37e3134659..46d1f17130 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -308,7 +308,7 @@ int SQLite::AddParams(Value* val, int pos) if ( j > 0 ) desc.AddRaw(set_separator); - io->Describe(&desc, val->val.set_val.vals[j], fields[pos]->name); + io->Describe(&desc, val->val.set_val.vals[j], fields[pos-1]->name); } desc.RemoveEscapeSequence(set_separator); @@ -330,7 +330,7 @@ int SQLite::AddParams(Value* val, int pos) if ( j > 0 ) desc.AddRaw(set_separator); - io->Describe(&desc, val->val.vector_val.vals[j], fields[pos]->name); + io->Describe(&desc, val->val.vector_val.vals[j], fields[pos-1]->name); } desc.RemoveEscapeSequence(set_separator); diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output new file mode 100644 index 0000000000..0ec0c9ce70 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output @@ -0,0 +1,40 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-countconns.bro + +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } \ No newline at end of file diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output new file mode 100644 index 0000000000..b1b46b3b39 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output @@ -0,0 +1,49 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-toy-scan.bro + +@load base/frameworks/sumstats + +# We use the connection_attempt event to limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is provided for when a key crosses the threshold. + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output index 9f05c43669..aa4509513f 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output @@ -27,6 +27,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold @@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output index 92f077de7d..30c6b1040e 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output @@ -18,6 +18,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output index 57d2d7ee50..64ef286c39 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output @@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 b/testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 new file mode 100644 index 0000000000..b7126957ce --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r workshop_2011_browse.trace sumstats-countconns.bro + Number of connections established: 6 + diff --git a/testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 b/testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 new file mode 100644 index 0000000000..f5db95c5f3 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r nmap-vsn.trace sumstats-toy-scan.bro + 192.168.1.71 attempted 5 or more connections + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select new file mode 100644 index 0000000000..71bf36b865 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select @@ -0,0 +1 @@ +CC,AA,BB diff --git a/testing/btest/Traces/nmap-vsn.trace b/testing/btest/Traces/nmap-vsn.trace new file mode 100644 index 0000000000..b276ed3d2f Binary files /dev/null and b/testing/btest/Traces/nmap-vsn.trace differ diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest new file mode 100644 index 0000000000..0ec0c9ce70 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest @@ -0,0 +1,40 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-countconns.bro + +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } \ No newline at end of file diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest new file mode 100644 index 0000000000..b1b46b3b39 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest @@ -0,0 +1,49 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-toy-scan.bro + +@load base/frameworks/sumstats + +# We use the connection_attempt event to limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is provided for when a key crosses the threshold. + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest index 9f05c43669..aa4509513f 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest @@ -27,6 +27,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold @@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest index 92f077de7d..30c6b1040e 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest @@ -18,6 +18,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest index 57d2d7ee50..64ef286c39 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest @@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/doc/sphinx/sumstats-countconns.btest b/testing/btest/doc/sphinx/sumstats-countconns.btest new file mode 100644 index 0000000000..fd375af5fb --- /dev/null +++ b/testing/btest/doc/sphinx/sumstats-countconns.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro diff --git a/testing/btest/doc/sphinx/sumstats-toy-scan.btest b/testing/btest/doc/sphinx/sumstats-toy-scan.btest new file mode 100644 index 0000000000..8756f1cfc0 --- /dev/null +++ b/testing/btest/doc/sphinx/sumstats-toy-scan.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro diff --git a/testing/btest/scripts/base/frameworks/logging/sqlite/set.bro b/testing/btest/scripts/base/frameworks/logging/sqlite/set.bro new file mode 100644 index 0000000000..7507316996 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/sqlite/set.bro @@ -0,0 +1,50 @@ +# +# Check if set works in last position (the describe call in sqlite.cc has a good +# chance of being off by one if someone changes it). +# +# @TEST-REQUIRES: which sqlite3 +# @TEST-REQUIRES: has-writer SQLite +# @TEST-GROUP: sqlite +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from ssh' > ssh.select +# @TEST-EXEC: btest-diff ssh.select +# +# Testing all possible types. + +redef LogSQLite::unset_field = "(unset)"; + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + ss: set[string]; + } &log; +} + +function foo(i : count) : string + { + if ( i > 0 ) + return "Foo"; + else + return "Bar"; + } + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + Log::remove_filter(SSH::LOG, "default"); + + local filter: Log::Filter = [$name="sqlite", $path="ssh", $writer=Log::WRITER_SQLITE]; + Log::add_filter(SSH::LOG, filter); + + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(SSH::LOG, [ + $ss=set("AA", "BB", "CC") + ]); +} + diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index 37a51fa72f..aabe9218e6 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -8,6 +8,10 @@ if [ "$1" == "capture_loss.log" ]; then addl="`dirname $0`/diff-remove-fractions" fi +if [ "$1" == "ssh.log" ]; then + addl="`dirname $0`/diff-remove-fields remote_location" +fi + `dirname $0`/diff-remove-timestamps \ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-file-ids \ diff --git a/testing/scripts/diff-remove-fields b/testing/scripts/diff-remove-fields new file mode 100755 index 0000000000..7f18748a5f --- /dev/null +++ b/testing/scripts/diff-remove-fields @@ -0,0 +1,34 @@ +#! /usr/bin/env bash +# +# A diff canonifier that removes all fields starting with a given +# prefix. + +if [ $# != 1 ]; then + echo "usage: `basename $0` " + exit 1 +fi + +awk -v "PREFIX=$1" ' + +BEGIN { FS="\t"; OFS="\t"; } + +/^#fields/ { + for ( i = 2; i <= NF; ++i ) + { + if ( index($i, PREFIX) == 1 ) + rem[i-1] = 1; + } + print; + next; +} + +{ + for ( i in rem ) + # Mark that it iss set, but ignore content. + $i = "+"; + + print; +} + +' +