diff --git a/.update-changes.cfg b/.update-changes.cfg new file mode 100644 index 0000000000..25e4765f27 --- /dev/null +++ b/.update-changes.cfg @@ -0,0 +1,19 @@ + +function new_version_hook + { + # This can be used to automatically adapt version numbers in files, + # except it's not used for that currently. Instead it updates the files + # which store the commit hashes associated with the version of external + # test suite repos to check out on a CI system. + version=$1 + + if [ -d testing/external/bro-testing ]; then + ( cd testing/external/bro-testing && git fetch origin && git rev-parse origin/master ) > testing/external/commit-hash.bro-testing + git add testing/external/commit-hash.bro-testing + fi + + if [ -d testing/external/bro-testing-private ]; then + ( cd testing/external/bro-testing-private && git fetch origin && git rev-parse origin/master ) > testing/external/commit-hash.bro-testing-private + git add testing/external/commit-hash.bro-testing-private + fi + } diff --git a/CHANGES b/CHANGES index 0eb7336ac2..c720e4950e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,107 @@ +2.5-906 | 2018-08-24 14:57:55 -0500 + + * Stabilize a cluster logging unit test (Jon Siwek, Corelight) + +2.5-905 | 2018-08-24 10:21:35 -0500 + + * Detect MaxMind DB changes and auto-reload (Jonathan Perkins, Corelight) + +2.5-903 | 2018-08-23 16:54:24 -0500 + + * Fix finding of kerberos and libmaxminddb CMake < 3.3 (Daniel Thayer) + + * BIT-1885: fix "kill" threading message (Jon Siwek, Corelight) + +2.5-900 | 2018-08-23 15:18:48 -0500 + + * Improve readability of the Travis job log (Daniel Thayer) + + * Fix tracking of DCE-RPC context identifier mappings + + This adds previously-missing support for "Alter Context" + request/response PDUs (initial patch contributed by Mark Fernandez). + + Also, context ID arguments were added to dce_rpc_bind, dce_rpc_request, + and dce_rpc_response in order to properly track what endpoint/operation + a given opnum maps to. (Jon Siwek, Corelight) + +2.5-897 | 2018-08-23 15:53:16 +0000 + + * BIT-1885: Fix input framework memory leak. For input threads that + get joined during run-time, messages could remain in the thread's + queue and leak. (Jon Siwek, Corelight) + + * Increase timeout for a memleak test. (Jon Siwek, Corelight) + +2.5-894 | 2018-08-22 12:05:19 -0500 + + * Ensure external test repo hashes track origin/master (Jon Siwek, Corelight) + +2.5-892 | 2018-08-22 11:49:12 -0500 + + * Fix "unused CMake variable" configuration warnings (Jon Siwek, Corelight) + +2.5-890 | 2018-08-21 16:47:52 -0500 + + * Fix Travis CI script to checkout particular commits of external tests + (Jon Siwek, Corelight) + + * Fix signed/unsigned comparison warning (Jon Siwek, Corelight) + +2.5-888 | 2018-08-21 15:54:56 -0500 + + * Add --with-broker configure option (Jon Siwek, Corelight) + +2.5-887 | 2018-08-21 14:54:12 -0500 + + * Change default snaplen to 9216 bytes to better accommodate jumbo frames + (Justin Azoff) + +2.5-884 | 2018-08-20 15:39:21 -0500 + + * Fix outdated documentation test baselines (Jon Siwek, Corelight) + + * Add 'smtp_excessive_pending_cmds' weird (Jon Siwek, Corelight) + + * Fix SMTP command string comparisons (Jon Siwek, Corelight) + + * Improve handling of empty lines in several text protocol analyzers + (Jon Siwek, Corelight) + + * Add rate-limiting sampling mechanism for weird events + + The generation of weird events, by default, are now rate-limited + according to these tunable options: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + The new get_reporter_stats() BIF also allows one to query the + total number of weirds generated (pre-sampling) which the new + policy/misc/weird-stats.bro script uses periodically to populate + a weird_stats.log. + + There's also new reporter BIFs to allow generating weirds from the + script-layer such that they go through the same, internal + rate-limiting/sampling mechanisms: + + - Reporter::conn_weird + - Reporter::flow_weird + - Reporter::net_weird + + Some of the code was adapted from previous work by Johanna Amann. + (Jon Siwek, Corelight) + + * Teach timestamp canonifier about timestamps before ~2001 + (Jon Siwek, Corelight) + +2.5-877 | 2018-08-20 14:58:58 -0500 + + * Remove the node-specific local-*.bro scripts (Daniel Thayer) + 2.5-875 | 2018-08-20 12:45:32 -0500 * Improve diff-remove-abspath canonifier: collapse '/' sequences diff --git a/CMakeLists.txt b/CMakeLists.txt index 452f2834cc..80e3d5049a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -95,14 +95,6 @@ FindRequiredPackage(OpenSSL) FindRequiredPackage(BIND) FindRequiredPackage(ZLIB) -find_package(CAF COMPONENTS core io openssl) -if (CAF_FOUND) - # e.g. if not using embedded CAF, then need to know where to look - # for CAF headers since that may differ from where Broker headers - # are found (and including a Broker header may pull in CAF headers). - include_directories(BEFORE ${CAF_INCLUDE_DIRS}) -endif () - if (NOT BINPAC_EXE_PATH AND EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/aux/binpac/CMakeLists.txt) add_subdirectory(aux/binpac) @@ -142,7 +134,7 @@ include_directories(BEFORE set(USE_GEOIP false) find_package(LibMMDB) -if (LibMMDB_FOUND) +if (LIBMMDB_FOUND) set(USE_GEOIP true) include_directories(BEFORE ${LibMMDB_INCLUDE_DIR}) list(APPEND OPTLIBS ${LibMMDB_LIBRARY}) @@ -151,7 +143,7 @@ endif () set(USE_KRB5 false) if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux ) find_package(LibKrb5) - if (LibKrb5_FOUND) + if (LIBKRB5_FOUND) set(USE_KRB5 true) list(APPEND OPTLIBS ${LibKrb5_LIBRARY}) endif () @@ -255,7 +247,11 @@ else () set(BRO_CONFIG_BINPAC_ROOT_DIR ${BRO_ROOT_DIR}) endif () -set(BRO_CONFIG_BROKER_ROOT_DIR ${BRO_ROOT_DIR}) +if ( BROKER_ROOT_DIR ) + set(BRO_CONFIG_BROKER_ROOT_DIR ${BROKER_ROOT_DIR}) +else () + set(BRO_CONFIG_BROKER_ROOT_DIR ${BRO_ROOT_DIR}) +endif () configure_file(${CMAKE_CURRENT_SOURCE_DIR}/bro-config.in ${CMAKE_CURRENT_BINARY_DIR}/bro-config @ONLY) @@ -267,10 +263,22 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/bro ######################################################################## ## Recurse on sub-directories -add_subdirectory(aux/broker) -set(brodeps ${brodeps} broker) -include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker - ${CMAKE_CURRENT_BINARY_DIR}/aux/broker) +if ( BROKER_ROOT_DIR ) + find_package(Broker REQUIRED) + find_package(CAF COMPONENTS core io openssl REQUIRED) + + set(brodeps ${brodeps} ${BROKER_LIBRARY} ${CAF_LIBRARIES}) + include_directories(BEFORE ${BROKER_INCLUDE_DIR}) +else () + add_subdirectory(aux/broker) + + set(brodeps ${brodeps} broker) + include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker + ${CMAKE_CURRENT_BINARY_DIR}/aux/broker) +endif () + +# CAF headers aren't necessarily in same location as Broker headers and +# inclusion of a Broker header may pull in CAF headers. include_directories(BEFORE ${CAF_INCLUDE_DIR_CORE}) include_directories(BEFORE ${CAF_INCLUDE_DIR_IO}) include_directories(BEFORE ${CAF_INCLUDE_DIR_OPENSSL}) diff --git a/NEWS b/NEWS index afe19d91b9..7c6312aedb 100644 --- a/NEWS +++ b/NEWS @@ -409,6 +409,7 @@ Changed Functionality - The string_to_pattern() built-in (and the now-deprecated merge_pattern() built-in) is no longer restricted to only be called at initialization time. + - GeoIP Legacy Database support has been replaced with GeoIP2 MaxMind DB format support. @@ -418,6 +419,34 @@ Changed Functionality after January 2, 2019. It's also noted that all GeoIP Legacy databases may be discontinued as they are superseded by GeoIP2. +- "Weird" events are now generally suppressed/sampled by default according to + some tunable parameters: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + Those options can be changed if one needs the previous behavior of + a "net_weird", "flow_weird", or "conn_weird" event being raised for + every single event. Otherwise, there is a new weird_stats.log which + contains concise summaries of weird counts per type per time period + and the original weird.log may not differ much either, except in + the cases where a particular weird type exceeds the sampling threshold. + +- Improved DCE-RPC analysis via tracking of context identifier mappings + + - These DCE-RPC events now contain an additional context-id argument: + + - dce_rpc_bind + - dce_rpc_request + - dce_rpc_response + + - Added new events: + + - dce_rpc_alter_context + - dce_rpc_alter_context_resp + Removed Functionality --------------------- @@ -432,6 +461,11 @@ Removed Functionality - BroControl: The "IPv6Comm" and "ZoneID" options are no longer available (though Broker should be able to handle IPv6 automatically). +- The node-specific ``site/local-*.bro`` scripts have been removed. + +- The default value of ``Pcap::snaplen`` changed from 8192 to 9216 bytes + to better accommodate jumbo frames. + Deprecated Functionality ------------------------ diff --git a/VERSION b/VERSION index f77e8fcc00..14c6d8d750 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-875 +2.5-906 diff --git a/aux/bifcl b/aux/bifcl index e99152c00a..b0ba021208 160000 --- a/aux/bifcl +++ b/aux/bifcl @@ -1 +1 @@ -Subproject commit e99152c00aad8f81c684a01bc4d40790a295f85c +Subproject commit b0ba021208c0c5fe8d7c92412b919d6667f63961 diff --git a/aux/binpac b/aux/binpac index 74cf55ace0..ff8c94964f 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 74cf55ace0de2bf061bbbf285ccf47cba122955f +Subproject commit ff8c94964fccbf60abae401d03c9fb35a8894c16 diff --git a/aux/bro-aux b/aux/bro-aux index 53aae82024..9866d31e45 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 53aae820242c02790089e384a9fe2d3174799ab1 +Subproject commit 9866d31e45f5e1467c0f1fd4a1cd9948b067ce43 diff --git a/aux/broccoli b/aux/broccoli index edf754ea6e..1898e81bba 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit edf754ea6e89a84ad74eff69a454c5e285c4b81b +Subproject commit 1898e81bba7c595108bc20c5148df7f0f6a2258f diff --git a/aux/broctl b/aux/broctl index 70a8b2e151..486bbb9d9e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 70a8b2e15105f4c238765a882151718162e46208 +Subproject commit 486bbb9d9ee7c66b55003e58f986d18e951902ec diff --git a/aux/broker b/aux/broker index 2727afff78..18d56b7055 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 2727afff785201168b33b0c7448c886643f10e08 +Subproject commit 18d56b70558db61f424dc90e5d010f57b75de459 diff --git a/cmake b/cmake index 4cc3e344cf..433676df0a 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 4cc3e344cf2698010a46684d32a2907a943430e3 +Subproject commit 433676df0af266540b8a3d9216cfd9dbc1361b09 diff --git a/configure b/configure index 90dda2fdd7..a26a4bc5a2 100755 --- a/configure +++ b/configure @@ -69,8 +69,10 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-flex=PATH path to flex executable --with-bison=PATH path to bison executable --with-python=PATH path to Python executable - --with-caf=PATH path to C++ Actor Framework installation for using external version - (a required Broker dependency) + --with-broker=PATH path to Broker install root + (Bro uses an embedded version by default) + --with-caf=PATH path to C++ Actor Framework install root + (a Broker dependency that is embedded by default) Optional Packages in Non-Standard Locations: --with-geoip=PATH path to the libmaxminddb install root @@ -142,8 +144,6 @@ append_cache_entry INSTALL_BROCTL BOOL true append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING append_cache_entry ENABLE_MOBILE_IPV6 BOOL false append_cache_entry DISABLE_PERFTOOLS BOOL false -append_cache_entry DISABLE_RUBY_BINDINGS BOOL true -append_cache_entry ENABLE_COVERAGE BOOL false # parse arguments while [ $# -ne 0 ]; do @@ -220,6 +220,7 @@ while [ $# -ne 0 ]; do append_cache_entry ENABLE_JEMALLOC BOOL true ;; --enable-broccoli) + append_cache_entry DISABLE_RUBY_BINDINGS BOOL true append_cache_entry INSTALL_BROCCOLI BOOL yes ;; --disable-broctl) @@ -285,6 +286,9 @@ while [ $# -ne 0 ]; do --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; + --with-broker=*) + append_cache_entry BROKER_ROOT_DIR PATH $optarg + ;; --with-caf=*) append_cache_entry CAF_ROOT_DIR PATH $optarg ;; diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 712d86c26c..992845cbba 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -152,6 +152,8 @@ Miscellaneous +----------------------------+---------------------------------------+---------------------------------+ | weird.log | Unexpected network-level activity | :bro:type:`Weird::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| weird-stats.log | Statistics about unexpected activity | :bro:type:`WeirdStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ Bro Diagnostics --------------- diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index fb20ba6fdb..96c682871a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -14,19 +14,3 @@ InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro ${BRO_SCRIPT_INSTALL_PATH}/site local.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-manager.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-logger.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-logger.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-proxy.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-worker.bro) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 8febc9dae3..70d59b30cf 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -82,6 +82,13 @@ type addr_vec: vector of addr; ## directly and then remove this alias. type table_string_of_string: table[string] of string; +## A table of counts indexed by strings. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type table_string_of_count: table[string] of count; + ## A set of file analyzer tags. ## ## .. todo:: We need this type definition only for declaring builtin functions @@ -626,6 +633,17 @@ type BrokerStats: record { num_ids_outgoing: count; }; +## Statistics about reporter messages and weirds. +## +## .. bro:see:: get_reporter_stats +type ReporterStats: record { + ## Number of total weirds encountered, before any rate-limiting. + weirds: count; + ## Number of times each individual weird is encountered, before any + ## rate-limiting is applied. + weirds_by_type: table[string] of count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere @@ -4795,7 +4813,7 @@ export { module Pcap; export { ## Number of bytes per packet to capture from live interfaces. - const snaplen = 8192 &redef; + const snaplen = 9216 &redef; ## Number of Mbytes to provide as buffer space when capturing from live ## interfaces. @@ -4826,6 +4844,35 @@ export { type Cluster::Pool: record {}; } +module Weird; +export { + ## Prevents rate-limiting sampling of any weirds named in the table. + const sampling_whitelist: set[string] &redef; + + ## How many weirds of a given type to tolerate before sampling begins. + ## i.e. this many consecutive weirds of a given type will be allowed to + ## raise events for script-layer handling before being rate-limited. + const sampling_threshold = 25 &redef; + + ## The rate-limiting sampling rate. One out of every of this number of + ## rate-limited weirds of a given type will be allowed to raise events + ## for further script-layer handling. + const sampling_rate = 1000 &redef; + + ## How long a weird of a given type is allowed to keep state/counters in + ## memory. For "net" weirds an expiration timer starts per weird name when + ## first initializing its counter. For "flow" weirds an expiration timer + ## starts once per src/dst IP pair for the first weird of any name. For + ## "conn" weirds, counters and expiration timers are kept for the duration + ## of the connection for each named weird and reset when necessary. e.g. + ## if a "conn" weird by the name of "foo" is seen more than + ## :bro:see:`Weird::sampling_threshold` times, then an expiration timer + ## begins for "foo" and upon triggering will reset the counter for "foo" + ## and unthrottle its rate-limiting until it once again exceeds the + ## threshold. + const sampling_duration = 10min &redef; +} + module GLOBAL; ## Seed for hashes computed internally for probabilistic data structures. Using diff --git a/scripts/base/protocols/dce-rpc/main.bro b/scripts/base/protocols/dce-rpc/main.bro index 0d40b11878..e73bfa4b2b 100644 --- a/scripts/base/protocols/dce-rpc/main.bro +++ b/scripts/base/protocols/dce-rpc/main.bro @@ -37,6 +37,7 @@ export { type State: record { uuid : string &optional; named_pipe : string &optional; + ctx_to_uuid: table[count] of string &optional; }; # This is to store the log and state information @@ -100,11 +101,30 @@ function set_session(c: connection, fid: count) set_state(c, state_x); } -event dce_rpc_bind(c: connection, fid: count, uuid: string, ver_major: count, ver_minor: count) &priority=5 +event dce_rpc_bind(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5 { set_session(c, fid); local uuid_str = uuid_to_string(uuid); + + if ( ! c$dce_rpc_state?$ctx_to_uuid ) + c$dce_rpc_state$ctx_to_uuid = table(); + + c$dce_rpc_state$ctx_to_uuid[ctx_id] = uuid_str; + c$dce_rpc_state$uuid = uuid_str; + c$dce_rpc$endpoint = uuid_endpoint_map[uuid_str]; + } + +event dce_rpc_alter_context(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5 + { + set_session(c, fid); + + local uuid_str = uuid_to_string(uuid); + + if ( ! c$dce_rpc_state?$ctx_to_uuid ) + c$dce_rpc_state$ctx_to_uuid = table(); + + c$dce_rpc_state$ctx_to_uuid[ctx_id] = uuid_str; c$dce_rpc_state$uuid = uuid_str; c$dce_rpc$endpoint = uuid_endpoint_map[uuid_str]; } @@ -120,7 +140,12 @@ event dce_rpc_bind_ack(c: connection, fid: count, sec_addr: string) &priority=5 } } -event dce_rpc_request(c: connection, fid: count, opnum: count, stub_len: count) &priority=5 +event dce_rpc_alter_context_resp(c: connection, fid: count) &priority=5 + { + set_session(c, fid); + } + +event dce_rpc_request(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count) &priority=5 { set_session(c, fid); @@ -130,7 +155,7 @@ event dce_rpc_request(c: connection, fid: count, opnum: count, stub_len: count) } } -event dce_rpc_response(c: connection, fid: count, opnum: count, stub_len: count) &priority=5 +event dce_rpc_response(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count) &priority=5 { set_session(c, fid); @@ -146,15 +171,26 @@ event dce_rpc_response(c: connection, fid: count, opnum: count, stub_len: count) } } - if ( c?$dce_rpc && c$dce_rpc?$endpoint ) + if ( c?$dce_rpc ) { - c$dce_rpc$operation = operations[c$dce_rpc_state$uuid, opnum]; - if ( c$dce_rpc$ts != network_time() ) - c$dce_rpc$rtt = network_time() - c$dce_rpc$ts; + if ( c$dce_rpc?$endpoint ) + { + c$dce_rpc$operation = operations[c$dce_rpc_state$uuid, opnum]; + if ( c$dce_rpc$ts != network_time() ) + c$dce_rpc$rtt = network_time() - c$dce_rpc$ts; + } + + if ( c$dce_rpc_state?$ctx_to_uuid && + ctx_id in c$dce_rpc_state$ctx_to_uuid ) + { + local u = c$dce_rpc_state$ctx_to_uuid[ctx_id]; + c$dce_rpc$endpoint = uuid_endpoint_map[u]; + c$dce_rpc$operation = operations[u, opnum]; + } } } -event dce_rpc_response(c: connection, fid: count, opnum: count, stub_len: count) &priority=-5 +event dce_rpc_response(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count) &priority=-5 { if ( c?$dce_rpc ) { diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index c80d99c70b..78e3a68d0f 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string, c$http$uri = unescaped_URI; if ( method !in http_methods ) - event conn_weird("unknown_HTTP_method", c, method); + Reporter::conn_weird("unknown_HTTP_method", c, method); } event http_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index c2a78e32af..1089f0c1a4 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version: c$sip$uri = original_URI; if ( method !in sip_methods ) - event conn_weird("unknown_SIP_method", c, method); + Reporter::conn_weird("unknown_SIP_method", c, method); } event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/smb/smb1-main.bro b/scripts/base/protocols/smb/smb1-main.bro index 44210e88f0..54add50b1f 100644 --- a/scripts/base/protocols/smb/smb1-main.bro +++ b/scripts/base/protocols/smb/smb1-main.bro @@ -309,7 +309,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count) if ( ! f?$uuid ) { # TODO: figure out why this is happening. - event conn_weird("smb_pipe_request_missing_uuid", c, ""); + Reporter::conn_weird("smb_pipe_request_missing_uuid", c, ""); return; } local arg = fmt("%s: %s", diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 11c64b9b40..6339d66557 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) { c$ssl$server_name = names[0]; if ( |names| > 1 ) - event conn_weird("SSL_many_server_names", c, cat(names)); + Reporter::conn_weird("SSL_many_server_names", c, cat(names)); } } diff --git a/scripts/policy/misc/weird-stats.bro b/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..d08ce9381d --- /dev/null +++ b/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,100 @@ +##! Log weird statistics. + +@load base/frameworks/sumstats +@load base/frameworks/cluster + +module WeirdStats; + +export { + redef enum Log::ID += { LOG }; + + ## How often stats are reported. + const weird_stat_interval = 15min &redef; + + type Info: record { + ## Timestamp for the measurement. + ts: time &log; + ## Name of the weird. + name: string &log; + ## Number of times weird was seen since the last stats interval. + num_seen: count &log; + }; + + global log_weird_stats: event(rec: Info); +} + +global this_epoch_weirds: table[string] of double; +global last_epoch_weirds: table[string] of double; + +function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result) + { + this_epoch_weirds[key$str]=result["weirds.encountered"]$sum; + } + +function weird_epoch_finished(ts: time) + { + for ( n in this_epoch_weirds ) + { + local last_count: double = 0.0; + + if ( n in last_epoch_weirds ) + last_count = last_epoch_weirds[n]; + + local num_seen: double = this_epoch_weirds[n] - last_count; + + if ( num_seen > 0.0 ) + Log::write(LOG, Info($ts = ts, $name = n, + $num_seen = double_to_count(num_seen))); + } + + last_epoch_weirds = this_epoch_weirds; + this_epoch_weirds = table(); + } + +event bro_init() &priority=5 + { + Log::create_stream(WeirdStats::LOG, + [$columns = Info, $ev = log_weird_stats, + $path="weird_stats"]); + local r1 = SumStats::Reducer($stream = "weirds.encountered", + $apply = set(SumStats::SUM)); + SumStats::create([$name = "weirds.statistics", + $epoch = weird_stat_interval, $reducers = set(r1), + $epoch_result = weird_epoch_results, + $epoch_finished = weird_epoch_finished]); + } + +module SumStats; + +function observe_weird_stats() + { + local rs = get_reporter_stats(); + + for ( n in rs$weirds_by_type ) + SumStats::observe("weirds.encountered", SumStats::Key($str = n), + SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0)); + } + +@if ( Cluster::is_enabled() ) + +# I'm not sure if this is a hack or not: the manager will generate this +# event at the end of its epoch so workers can handle it just in time to +# generate the necessary stats. Alternative may be workers generating the +# stats individually/proactively in their own finish_epoch, but that may be +# less synchronized? +event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10 + { + observe_weird_stats(); + } + +@else + +event SumStats::finish_epoch(ss: SumStat) &priority=10 + { + if ( ss$name != "weirds.statistics" ) + return; + + observe_weird_stats(); + } + +@endif diff --git a/scripts/site/local-logger.bro b/scripts/site/local-logger.bro deleted file mode 100644 index 0642e86ce3..0000000000 --- a/scripts/site/local-logger.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the logger if Bro is running as a cluster. diff --git a/scripts/site/local-manager.bro b/scripts/site/local-manager.bro deleted file mode 100644 index 5e6005f21e..0000000000 --- a/scripts/site/local-manager.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the manager if Bro is running as a cluster. diff --git a/scripts/site/local-proxy.bro b/scripts/site/local-proxy.bro deleted file mode 100644 index 478ba6d048..0000000000 --- a/scripts/site/local-proxy.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the proxies if Bro is running as a cluster. diff --git a/scripts/site/local-worker.bro b/scripts/site/local-worker.bro deleted file mode 100644 index b2a100e135..0000000000 --- a/scripts/site/local-worker.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the workers if Bro is running as a cluster. \ No newline at end of file diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 9ccfdaa670..11824c2c6a 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -54,6 +54,7 @@ @load misc/profiling.bro @load misc/scan.bro @load misc/stats.bro +@load misc/weird-stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @load protocols/conn/known-services.bro diff --git a/src/3rdparty b/src/3rdparty index 6cdefdd1d4..7c95b51de2 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit 6cdefdd1d45465ca09aba9e05c7ca12e1484ccc0 +Subproject commit 7c95b51de202ac534b27dd721da5778b773dd614 diff --git a/src/Conn.cc b/src/Conn.cc index 447f730418..c69f155731 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -1071,3 +1071,28 @@ void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label) else saw_first_resp_packet = 1; } + +bool Connection::PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration) + { + auto& state = weird_state[name]; + ++state.count; + + if ( state.count < threshold ) + return true; + + if ( state.count == threshold ) + state.sampling_start_time = network_time; + else + { + if ( network_time > state.sampling_start_time + duration ) + { + state.sampling_start_time = 0; + state.count = 1; + return true; + } + } + + auto num_above_threshold = state.count - threshold; + return num_above_threshold % rate == 0; + } diff --git a/src/Conn.h b/src/Conn.h index 07765ee474..ae639d6341 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -5,6 +5,9 @@ #include +#include +#include + #include "Dict.h" #include "Val.h" #include "Timer.h" @@ -275,6 +278,9 @@ public: uint32 GetOrigFlowLabel() { return orig_flow_label; } uint32 GetRespFlowLabel() { return resp_flow_label; } + bool PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration); + protected: Connection() { persistent = 0; } @@ -339,6 +345,14 @@ protected: analyzer::pia::PIA* primary_PIA; Bro::UID uid; // Globally unique connection ID. + + struct WeirdState { + WeirdState() { count = 0; sampling_start_time = 0; } + uint64 count = 0; + double sampling_start_time = 0; + }; + + std::unordered_map weird_state; }; class ConnectionTimer : public Timer { diff --git a/src/Func.cc b/src/Func.cc index 703cc04d29..015829e2a1 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -722,6 +722,7 @@ void init_builtin_funcs() FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); BrokerStats = internal_type("BrokerStats")->AsRecordType(); + ReporterStats = internal_type("ReporterStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/Reporter.cc b/src/Reporter.cc index eb89a29d30..59e877bb7c 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -10,6 +10,7 @@ #include "NetVar.h" #include "Net.h" #include "Conn.h" +#include "Timer.h" #include "plugin/Plugin.h" #include "plugin/Manager.h" @@ -36,6 +37,11 @@ Reporter::Reporter() warnings_to_stderr = true; errors_to_stderr = true; + weird_count = 0; + weird_sampling_rate = 0; + weird_sampling_duration = 0; + weird_sampling_threshold = 0; + openlog("bro", 0, LOG_LOCAL5); } @@ -49,6 +55,24 @@ void Reporter::InitOptions() info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool(); warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool(); errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool(); + weird_sampling_rate = internal_const_val("Weird::sampling_rate")->AsCount(); + weird_sampling_threshold = internal_const_val("Weird::sampling_threshold")->AsCount(); + weird_sampling_duration = internal_const_val("Weird::sampling_duration")->AsInterval(); + auto wl_val = internal_const_val("Weird::sampling_whitelist")->AsTableVal(); + auto wl_table = wl_val->AsTable(); + + HashKey* k; + IterCookie* c = wl_table->InitForIteration(); + TableEntryVal* v; + + while ( (v = wl_table->NextEntry(k, c)) ) + { + auto index = wl_val->RecoverIndex(k); + string key = index->Index(0)->AsString()->CheckString(); + weird_sampling_whitelist.emplace(move(key)); + Unref(index); + delete k; + } } void Reporter::Info(const char* fmt, ...) @@ -221,23 +245,121 @@ void Reporter::WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const cha delete vl; } +void Reporter::UpdateWeirdStats(const char* name) + { + ++weird_count; + ++weird_count_by_type[name]; + } + +class NetWeirdTimer : public Timer { +public: + NetWeirdTimer(double t, const char* name, double timeout) + : Timer(t + timeout, TIMER_NET_WEIRD_EXPIRE), weird_name(name) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetNetWeird(weird_name); } + + std::string weird_name; +}; + +class FlowWeirdTimer : public Timer { +public: + using IPPair = std::pair; + + FlowWeirdTimer(double t, IPPair p, double timeout) + : Timer(t + timeout, TIMER_FLOW_WEIRD_EXPIRE), endpoints(p) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetFlowWeird(endpoints.first, endpoints.second); } + + IPPair endpoints; +}; + +void Reporter::ResetNetWeird(const std::string& name) + { + net_weird_state.erase(name); + } + +void Reporter::ResetFlowWeird(const IPAddr& orig, const IPAddr& resp) + { + flow_weird_state.erase(std::make_pair(orig, resp)); + } + +bool Reporter::PermitNetWeird(const char* name) + { + auto& count = net_weird_state[name]; + ++count; + + if ( count == 1 ) + timer_mgr->Add(new NetWeirdTimer(network_time, name, + weird_sampling_duration)); + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + +bool Reporter::PermitFlowWeird(const char* name, + const IPAddr& orig, const IPAddr& resp) + { + auto endpoints = std::make_pair(orig, resp); + auto& map = flow_weird_state[endpoints]; + + if ( map.empty() ) + timer_mgr->Add(new FlowWeirdTimer(network_time, endpoints, + weird_sampling_duration)); + + auto& count = map[name]; + ++count; + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + void Reporter::Weird(const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitNetWeird(name) ) + return; + } + WeirdHelper(net_weird, 0, 0, "%s", name); } void Reporter::Weird(Connection* conn, const char* name, const char* addl) { - WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); - } + UpdateWeirdStats(name); -void Reporter::Weird(Val* conn_val, const char* name, const char* addl) - { - WeirdHelper(conn_weird, conn_val, addl, "%s", name); + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! conn->PermitWeird(name, weird_sampling_threshold, + weird_sampling_rate, weird_sampling_duration) ) + return; + } + + WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); } void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitFlowWeird(name, orig, resp) ) + return; + } + WeirdFlowHelper(orig, resp, "%s", name); } diff --git a/src/Reporter.h b/src/Reporter.h index 85b9a483a2..f5090ee91e 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -7,6 +7,10 @@ #include #include +#include +#include +#include +#include #include "util.h" #include "EventHandler.h" @@ -36,6 +40,11 @@ protected: class Reporter { public: + using IPPair = std::pair; + using WeirdCountMap = std::unordered_map; + using WeirdFlowMap = std::map; + using WeirdSet = std::unordered_set; + Reporter(); ~Reporter(); @@ -76,7 +85,6 @@ public: // that may lead to incorrectly processing a connnection. void Weird(const char* name); // Raises net_weird(). void Weird(Connection* conn, const char* name, const char* addl = ""); // Raises conn_weird(). - void Weird(Val* conn_val, const char* name, const char* addl = ""); // Raises conn_weird(). void Weird(const IPAddr& orig, const IPAddr& resp, const char* name); // Raises flow_weird(). // Syslog a message. This methods does nothing if we're running @@ -120,6 +128,30 @@ public: // Signals that we're done processing an error handler event. void EndErrorHandler() { --in_error_handler; } + /** + * Reset/cleanup state tracking for a "net" weird. + */ + void ResetNetWeird(const std::string& name); + + /** + * Reset/cleanup state tracking for a "flow" weird. + */ + void ResetFlowWeird(const IPAddr& orig, const IPAddr& resp); + + /** + * Return the total number of weirds generated (counts weirds before + * any rate-limiting occurs). + */ + uint64 GetWeirdCount() const + { return weird_count; } + + /** + * Return number of weirds generated per weird type/name (counts weirds + * before any rate-limiting occurs). + */ + const WeirdCountMap& GetWeirdsByType() const + { return weird_count_by_type; } + private: void DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, @@ -129,6 +161,11 @@ private: // contain format specifiers void WeirdHelper(EventHandlerPtr event, Val* conn_val, const char* addl, const char* fmt_name, ...) __attribute__((format(printf, 5, 6)));; void WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const char* fmt_name, ...) __attribute__((format(printf, 4, 5)));; + void UpdateWeirdStats(const char* name); + inline bool WeirdOnSamplingWhiteList(const char* name) + { return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); } + bool PermitNetWeird(const char* name); + bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r); int errors; bool via_events; @@ -138,6 +175,18 @@ private: bool errors_to_stderr; std::list > locations; + + uint64 weird_count; + WeirdCountMap weird_count_by_type; + + WeirdCountMap net_weird_state; + WeirdFlowMap flow_weird_state; + + WeirdSet weird_sampling_whitelist; + uint64 weird_sampling_threshold; + uint64 weird_sampling_rate; + double weird_sampling_duration; + }; extern Reporter* reporter; diff --git a/src/Timer.cc b/src/Timer.cc index 4fe998606b..101733028c 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -18,12 +18,14 @@ const char* TimerNames[] = { "ConnectionStatusUpdateTimer", "DNSExpireTimer", "FileAnalysisInactivityTimer", + "FlowWeirdTimer", "FragTimer", "IncrementalSendTimer", "IncrementalWriteTimer", "InterconnTimer", "IPTunnelInactivityTimer", "NetbiosExpireTimer", + "NetWeirdTimer", "NetworkTimer", "NTPExpireTimer", "ProfileTimer", diff --git a/src/Timer.h b/src/Timer.h index ea410e5c7b..8d6de857a0 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -23,12 +23,14 @@ enum TimerType { TIMER_CONN_STATUS_UPDATE, TIMER_DNS_EXPIRE, TIMER_FILE_ANALYSIS_INACTIVITY, + TIMER_FLOW_WEIRD_EXPIRE, TIMER_FRAG, TIMER_INCREMENTAL_SEND, TIMER_INCREMENTAL_WRITE, TIMER_INTERCONN, TIMER_IP_TUNNEL_INACTIVITY, TIMER_NB_EXPIRE, + TIMER_NET_WEIRD_EXPIRE, TIMER_NETWORK, TIMER_NTP_EXPIRE, TIMER_PROFILE, diff --git a/src/analyzer/protocol/dce-rpc/dce_rpc-analyzer.pac b/src/analyzer/protocol/dce-rpc/dce_rpc-analyzer.pac index 53402fd9a0..2bf8550ce5 100644 --- a/src/analyzer/protocol/dce-rpc/dce_rpc-analyzer.pac +++ b/src/analyzer/protocol/dce-rpc/dce_rpc-analyzer.pac @@ -54,6 +54,7 @@ refine connection DCE_RPC_Conn += { BifEvent::generate_dce_rpc_bind(bro_analyzer(), bro_analyzer()->Conn(), fid, + ${req.id}, bytestring_to_val(${req.abstract_syntax.uuid}), ${req.abstract_syntax.ver_major}, ${req.abstract_syntax.ver_minor}); @@ -62,6 +63,22 @@ refine connection DCE_RPC_Conn += { return true; %} + function process_dce_rpc_alter_context(req: ContextRequest): bool + %{ + if ( dce_rpc_alter_context ) + { + BifEvent::generate_dce_rpc_alter_context(bro_analyzer(), + bro_analyzer()->Conn(), + fid, + ${req.id}, + bytestring_to_val(${req.abstract_syntax.uuid}), + ${req.abstract_syntax.ver_major}, + ${req.abstract_syntax.ver_minor}); + } + + return true; + %} + function process_dce_rpc_bind_ack(bind: DCE_RPC_Bind_Ack): bool %{ if ( dce_rpc_bind_ack ) @@ -86,6 +103,17 @@ refine connection DCE_RPC_Conn += { return true; %} + function process_dce_rpc_alter_context_resp(bind: DCE_RPC_AlterContext_Resp): bool + %{ + if ( dce_rpc_alter_context_resp ) + { + BifEvent::generate_dce_rpc_alter_context_resp(bro_analyzer(), + bro_analyzer()->Conn(), + fid); + } + return true; + %} + function process_dce_rpc_request(req: DCE_RPC_Request): bool %{ if ( dce_rpc_request ) @@ -93,6 +121,7 @@ refine connection DCE_RPC_Conn += { BifEvent::generate_dce_rpc_request(bro_analyzer(), bro_analyzer()->Conn(), fid, + ${req.context_id}, ${req.opnum}, ${req.stub}.length()); } @@ -109,6 +138,7 @@ refine connection DCE_RPC_Conn += { BifEvent::generate_dce_rpc_response(bro_analyzer(), bro_analyzer()->Conn(), fid, + ${resp.context_id}, get_cont_id_opnum_map(${resp.context_id}), ${resp.stub}.length()); } @@ -127,13 +157,20 @@ refine typeattr DCE_RPC_Header += &let { }; refine typeattr ContextRequest += &let { - proc = $context.connection.process_dce_rpc_bind(this); + proc = case ptype of { + DCE_RPC_BIND -> $context.connection.process_dce_rpc_bind(this); + DCE_RPC_ALTER_CONTEXT -> $context.connection.process_dce_rpc_alter_context(this); + }; }; refine typeattr DCE_RPC_Bind_Ack += &let { proc = $context.connection.process_dce_rpc_bind_ack(this); }; +refine typeattr DCE_RPC_AlterContext_Resp += &let { + proc = $context.connection.process_dce_rpc_alter_context_resp(this); +}; + refine typeattr DCE_RPC_Request += &let { proc = $context.connection.process_dce_rpc_request(this); }; diff --git a/src/analyzer/protocol/dce-rpc/dce_rpc-protocol.pac b/src/analyzer/protocol/dce-rpc/dce_rpc-protocol.pac index e610efcadc..ad71373423 100644 --- a/src/analyzer/protocol/dce-rpc/dce_rpc-protocol.pac +++ b/src/analyzer/protocol/dce-rpc/dce_rpc-protocol.pac @@ -71,7 +71,7 @@ type Syntax = record { ver_minor : uint16; }; -type ContextRequest = record { +type ContextRequest(ptype: uint8) = record { id : uint16; num_syntaxes : uint8; reserved : padding[1]; @@ -85,11 +85,11 @@ type ContextReply = record { syntax : Syntax; }; -type ContextList(is_request: bool) = record { +type ContextList(is_request: bool, ptype: uint8) = record { num_contexts : uint8; reserved : padding[3]; req_reply : case is_request of { - true -> request_contexts : ContextRequest[num_contexts]; + true -> request_contexts : ContextRequest(ptype)[num_contexts]; false -> reply_contexts : ContextReply[num_contexts]; }; }; @@ -98,7 +98,7 @@ type DCE_RPC_Bind = record { max_xmit_frag : uint16; max_recv_frag : uint16; assoc_group_id : uint32; - context_list : ContextList(1); + context_list : ContextList(1, DCE_RPC_BIND); }; type DCE_RPC_Bind_Ack = record { @@ -108,7 +108,7 @@ type DCE_RPC_Bind_Ack = record { sec_addr_length : uint16; sec_addr : bytestring &length=sec_addr_length; pad : padding align 4; - contexts : ContextList(0); + contexts : ContextList(0, DCE_RPC_BIND_ACK); }; type DCE_RPC_Request(h: DCE_RPC_Header) = record { @@ -136,15 +136,17 @@ type DCE_RPC_AlterContext = record { max_xmit_frag : uint16; max_recv_frag : uint16; assoc_group_id : uint32; - contexts : ContextList(0); + context_list : ContextList(1, DCE_RPC_ALTER_CONTEXT); }; type DCE_RPC_AlterContext_Resp = record { - max_xmit_frag : uint16; - max_recv_frag : uint16; - assoc_group_id : uint32; - sec_addr_len : uint16; - contexts : ContextList(0); + max_xmit_frag : uint16; + max_recv_frag : uint16; + assoc_group_id : uint32; + sec_addr_length : uint16; + sec_addr : bytestring &length=sec_addr_length; + pad : padding align 4; + contexts : ContextList(0, DCE_RPC_ALTER_CONTEXT_RESP); }; type DCE_RPC_Body(header: DCE_RPC_Header) = case header.PTYPE of { @@ -152,9 +154,8 @@ type DCE_RPC_Body(header: DCE_RPC_Header) = case header.PTYPE of { DCE_RPC_BIND_ACK -> bind_ack : DCE_RPC_Bind_Ack; DCE_RPC_REQUEST -> request : DCE_RPC_Request(header); DCE_RPC_RESPONSE -> response : DCE_RPC_Response; - # TODO: Something about the two following structures isn't being handled correctly. - #DCE_RPC_ALTER_CONTEXT -> alter_context : DCE_RPC_AlterContext; - #DCE_RPC_ALTER_CONTEXT_RESP -> alter_resp : DCE_RPC_AlterContext_Resp; + DCE_RPC_ALTER_CONTEXT -> alter_context : DCE_RPC_AlterContext; + DCE_RPC_ALTER_CONTEXT_RESP -> alter_resp : DCE_RPC_AlterContext_Resp; default -> other : bytestring &restofdata; }; diff --git a/src/analyzer/protocol/dce-rpc/events.bif b/src/analyzer/protocol/dce-rpc/events.bif index b2a5ef1c4d..1e4a4e0d51 100644 --- a/src/analyzer/protocol/dce-rpc/events.bif +++ b/src/analyzer/protocol/dce-rpc/events.bif @@ -16,7 +16,7 @@ event dce_rpc_message%(c: connection, is_orig: bool, fid: count, ptype_id: count, ptype: DCE_RPC::PType%); ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` bind request message. -## Since RPC offers the ability for a client to request connections to multiple endpoints, this event can occur +## Since RPC offers the ability for a client to request connections to multiple endpoints, this event can occur ## multiple times for a single RPC message. ## ## c: The connection. @@ -25,6 +25,8 @@ event dce_rpc_message%(c: connection, is_orig: bool, fid: count, ptype_id: count ## message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was ## not transported over a pipe. ## +## ctx_id: The context identifier of the data representation. +## ## uuid: The string interpretted uuid of the endpoint being requested. ## ## ver_major: The major version of the endpoint being requested. @@ -32,7 +34,28 @@ event dce_rpc_message%(c: connection, is_orig: bool, fid: count, ptype_id: count ## ver_minor: The minor version of the endpoint being requested. ## ## .. bro:see:: dce_rpc_message dce_rpc_bind_ack dce_rpc_request dce_rpc_response -event dce_rpc_bind%(c: connection, fid: count, uuid: string, ver_major: count, ver_minor: count%); +event dce_rpc_bind%(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count%); + +## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` alter context request message. +## Since RPC offers the ability for a client to request connections to multiple endpoints, this event can occur +## multiple times for a single RPC message. +## +## c: The connection. +## +## fid: File ID of the PIPE that carried the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` +## message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was +## not transported over a pipe. +## +## ctx_id: The context identifier of the data representation. +## +## uuid: The string interpretted uuid of the endpoint being requested. +## +## ver_major: The major version of the endpoint being requested. +## +## ver_minor: The minor version of the endpoint being requested. +## +## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_request dce_rpc_response dce_rpc_alter_context_resp +event dce_rpc_alter_context%(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count%); ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` bind request ack message. ## @@ -47,6 +70,17 @@ event dce_rpc_bind%(c: connection, fid: count, uuid: string, ver_major: count, v ## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_request dce_rpc_response event dce_rpc_bind_ack%(c: connection, fid: count, sec_addr: string%); +## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` alter context response message. +## +## c: The connection. +## +## fid: File ID of the PIPE that carried the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` +## message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was +## not transported over a pipe. +## +## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_request dce_rpc_response dce_rpc_alter_context +event dce_rpc_alter_context_resp%(c: connection, fid: count%); + ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` request message. ## ## c: The connection. @@ -55,12 +89,14 @@ event dce_rpc_bind_ack%(c: connection, fid: count, sec_addr: string%); ## message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was ## not transported over a pipe. ## +## ctx_id: The context identifier of the data representation. +## ## opnum: Number of the RPC operation. ## ## stub_len: Length of the data for the request. ## ## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_response -event dce_rpc_request%(c: connection, fid: count, opnum: count, stub_len: count%); +event dce_rpc_request%(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count%); ## Generated for every :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` response message. ## @@ -70,9 +106,11 @@ event dce_rpc_request%(c: connection, fid: count, opnum: count, stub_len: count% ## message. Zero will be used if the :abbr:`DCE-RPC (Distributed Computing Environment/Remote Procedure Calls)` was ## not transported over a pipe. ## +## ctx_id: The context identifier of the data representation. +### ## opnum: Number of the RPC operation. ## ## stub_len: Length of the data for the response. ## ## .. bro:see:: dce_rpc_message dce_rpc_bind dce_rpc_bind_ack dce_rpc_request -event dce_rpc_response%(c: connection, fid: count, opnum: count, stub_len: count%); +event dce_rpc_response%(c: connection, fid: count, ctx_id: count, opnum: count, stub_len: count%); diff --git a/src/analyzer/protocol/finger/Finger.cc b/src/analyzer/protocol/finger/Finger.cc index e1be27e795..0aeb544a8a 100644 --- a/src/analyzer/protocol/finger/Finger.cc +++ b/src/analyzer/protocol/finger/Finger.cc @@ -41,6 +41,9 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig const char* line = (const char*) data; const char* end_of_line = line + length; + if ( length == 0 ) + return; + if ( is_orig ) { diff --git a/src/analyzer/protocol/ftp/FTP.cc b/src/analyzer/protocol/ftp/FTP.cc index 70d1be5777..b062f5aa7e 100644 --- a/src/analyzer/protocol/ftp/FTP.cc +++ b/src/analyzer/protocol/ftp/FTP.cc @@ -69,6 +69,10 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig) const char* line = (const char*) data; const char* end_of_line = line + length; + if ( length == 0 ) + // Could emit "ftp empty request/reply" weird, but maybe not worth it. + return; + val_list* vl = new val_list; vl->append(BuildConnVal()); diff --git a/src/analyzer/protocol/ident/Ident.cc b/src/analyzer/protocol/ident/Ident.cc index 27eafb5426..423028968c 100644 --- a/src/analyzer/protocol/ident/Ident.cc +++ b/src/analyzer/protocol/ident/Ident.cc @@ -56,6 +56,9 @@ void Ident_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig) if ( TCP() ) s = is_orig ? TCP()->Orig() : TCP()->Resp(); + if ( length == 0 ) + return; + if ( is_orig ) { if ( ! ident_request ) diff --git a/src/analyzer/protocol/pop3/POP3.cc b/src/analyzer/protocol/pop3/POP3.cc index b7d6aa0dcb..4f5e5819a6 100644 --- a/src/analyzer/protocol/pop3/POP3.cc +++ b/src/analyzer/protocol/pop3/POP3.cc @@ -132,6 +132,9 @@ static string trim_whitespace(const char* in) void POP3_Analyzer::ProcessRequest(int length, const char* line) { + if ( length == 0 ) + return; + if ( waitingForAuthentication ) { ++authLines; diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 8296f83cb3..b514b1f481 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -269,7 +269,9 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig) if ( smtp_request ) { int data_len = end_of_line - line; - RequestEvent(cmd_len, cmd, data_len, line); + + if ( cmd_len > 0 || data_len > 0 ) + RequestEvent(cmd_len, cmd, data_len, line); } if ( cmd_code != SMTP_CMD_END_OF_DATA ) @@ -379,7 +381,17 @@ void SMTP_Analyzer::NewCmd(const int cmd_code) if ( first_cmd < 0 ) first_cmd = cmd_code; else + { + auto constexpr max_pending_cmd_q_size = 1000; + + if ( pending_cmd_q.size() == max_pending_cmd_q_size ) + { + Weird("smtp_excessive_pending_cmds"); + pending_cmd_q.clear(); + } + pending_cmd_q.push_back(cmd_code); + } } else first_cmd = cmd_code; @@ -805,12 +817,22 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code, bool o #endif } +static bool istrequal(const char* s, const char* cmd, int s_len) + { + int cmd_len = strlen(cmd); + + if ( cmd_len != s_len ) + return false; + + return strncasecmp(s, cmd, s_len) == 0; + } + void SMTP_Analyzer::ProcessExtension(int ext_len, const char* ext) { if ( ! ext ) return; - if ( ! strncasecmp(ext, "PIPELINING", ext_len) ) + if ( istrequal(ext, "PIPELINING", ext_len) ) pipelining = 1; } @@ -820,11 +842,11 @@ int SMTP_Analyzer::ParseCmd(int cmd_len, const char* cmd) return -1; // special case because we cannot define our usual macros with "-" - if ( strncmp(cmd, "X-ANONYMOUSTLS", cmd_len) == 0 ) + if ( istrequal(cmd, "X-ANONYMOUSTLS", cmd_len) ) return SMTP_CMD_X_ANONYMOUSTLS; for ( int code = SMTP_CMD_EHLO; code < SMTP_CMD_LAST; ++code ) - if ( ! strncasecmp(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) + if ( istrequal(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) return code; return -1; diff --git a/src/bro.bif b/src/bro.bif index d6cacec96d..b2dd814317 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3598,6 +3598,8 @@ function lookup_hostname%(host: string%) : addr_set %%{ #ifdef USE_GEOIP +#include + extern "C" { #include #include @@ -3608,17 +3610,24 @@ extern "C" { class MMDB { public: - MMDB(const char* filename); + MMDB(const char* filename, struct stat info); ~MMDB(); MMDB_lookup_result_s Lookup(const struct sockaddr* const sa); + bool StaleDB(); + const char* Filename(); private: MMDB_s mmdb; + struct stat file_info; + bool lookup_error; + std::chrono::time_point last_check; }; -MMDB::MMDB(const char *filename) +MMDB::MMDB(const char* filename, struct stat info) + : file_info{info}, lookup_error{false}, + last_check{std::chrono::steady_clock::now()} { int status = MMDB_open(filename, MMDB_MODE_MMAP, &mmdb); @@ -3640,12 +3649,50 @@ MMDB_lookup_result_s MMDB::Lookup(const struct sockaddr* const sa) if ( MMDB_SUCCESS != mmdb_error ) { + lookup_error = true; throw std::runtime_error(MMDB_strerror(mmdb_error)); } return result; } +// Check to see if the Maxmind DB should be closed and reopened. This will +// happen if there was a lookup error or if the mmap'd file has been replaced +// by an external process. +bool MMDB::StaleDB() + { + struct stat buf; + using Clock = std::chrono::steady_clock; + std::chrono::time_point now = Clock::now(); + + if ( lookup_error ) + return true; + + // Only perform stat once per 5 minutes. + using Min = std::chrono::minutes; + if ( std::chrono::duration_cast(now - last_check).count() < 5 ) + return false; + + last_check = now; + + if ( 0 != stat(mmdb.filename, &buf) ) + return true; + + if ( buf.st_ino != file_info.st_ino || buf.st_mtime != file_info.st_mtime ) + { + reporter->Info("Inode change detected for MaxMind DB [%s]", + mmdb.filename); + return true; + } + + return false; + } + +const char* MMDB::Filename() + { + return mmdb.filename; + } + std::unique_ptr mmdb_loc; std::unique_ptr mmdb_asn; @@ -3662,11 +3709,11 @@ static bool mmdb_open(const char* filename, bool asn) { if ( asn ) { - mmdb_asn.reset(new MMDB(filename)); + mmdb_asn.reset(new MMDB(filename, buf)); } else { - mmdb_loc.reset(new MMDB(filename)); + mmdb_loc.reset(new MMDB(filename, buf)); } } @@ -3690,6 +3737,24 @@ static bool mmdb_open_asn(const char* filename) return mmdb_open(filename, true); } +static void mmdb_check_loc() + { + if ( mmdb_loc && mmdb_loc->StaleDB() ) + { + reporter->Info("Closing stale MaxMind DB [%s]", mmdb_loc->Filename()); + mmdb_loc.release(); + } + } + +static void mmdb_check_asn() + { + if ( mmdb_asn && mmdb_asn->StaleDB() ) + { + reporter->Info("Closing stale MaxMind DB [%s]", mmdb_asn->Filename()); + mmdb_asn.release(); + } + } + static bool mmdb_lookup(const IPAddr& addr, MMDB_lookup_result_s& result, bool asn) { @@ -3873,6 +3938,7 @@ function lookup_location%(a: addr%) : geo_location RecordVal* location = new RecordVal(geo_location); #ifdef USE_GEOIP + mmdb_check_loc(); if ( ! mmdb_loc ) { if ( ! mmdb_try_open_loc() ) @@ -3950,6 +4016,7 @@ function lookup_location%(a: addr%) : geo_location function lookup_asn%(a: addr%) : count %{ #ifdef USE_GEOIP + mmdb_check_asn(); if ( ! mmdb_asn ) { if ( ! mmdb_try_open_asn() ) diff --git a/src/reporter.bif b/src/reporter.bif index 3cdefc4da4..a6463f6226 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -71,3 +71,44 @@ function Reporter::fatal%(msg: string%): bool reporter->PopLocation(); return new Val(1, TYPE_BOOL); %} + +## Generates a "net" weird. +## +## name: the name of the weird. +## +## Returns: Always true. +function Reporter::net_weird%(name: string%): bool + %{ + reporter->Weird(name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "flow" weird. +## +## name: the name of the weird. +## +## orig: the originator host associated with the weird. +## +## resp: the responder host associated with the weird. +## +## Returns: Always true. +function Reporter::flow_weird%(name: string, orig: addr, resp: addr%): bool + %{ + reporter->Weird(orig->AsAddr(), resp->AsAddr(), name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "conn" weird. +## +## name: the name of the weird. +## +## c: the connection associated with the weird. +## +## addl: additional information to accompany the weird. +## +## Returns: Always true. +function Reporter::conn_weird%(name: string, c: connection, addl: string &default=""%): bool + %{ + reporter->Weird(c, name->CheckString(), addl->CheckString()); + return new Val(1, TYPE_BOOL); + %} diff --git a/src/stats.bif b/src/stats.bif index 2a525a1790..e3ce6f0245 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -16,6 +16,7 @@ RecordType* ThreadStats; RecordType* TimerStats; RecordType* FileAnalysisStats; RecordType* BrokerStats; +RecordType* ReporterStats; %%} ## Returns packet capture statistics. Statistics include the number of @@ -35,6 +36,7 @@ RecordType* BrokerStats; ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_net_stats%(%): NetStats %{ uint64 recv = 0; @@ -83,6 +85,7 @@ function get_net_stats%(%): NetStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_conn_stats%(%): ConnStats %{ RecordVal* r = new RecordVal(ConnStats); @@ -133,6 +136,7 @@ function get_conn_stats%(%): ConnStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_proc_stats%(%): ProcStats %{ struct rusage ru; @@ -189,6 +193,7 @@ function get_proc_stats%(%): ProcStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -215,6 +220,7 @@ function get_event_stats%(%): EventStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -243,6 +249,7 @@ function get_reassembler_stats%(%): ReassemblerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -276,6 +283,7 @@ function get_dns_stats%(%): DNSStats ## get_reassembler_stats ## get_thread_stats ## get_broker_stats +## get_reporter_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -303,6 +311,7 @@ function get_timer_stats%(%): TimerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -330,6 +339,7 @@ function get_file_analysis_stats%(%): FileAnalysisStats ## get_reassembler_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -355,6 +365,7 @@ function get_thread_stats%(%): ThreadStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_gap_stats%(%): GapStats %{ RecordVal* r = new RecordVal(GapStats); @@ -386,6 +397,7 @@ function get_gap_stats%(%): GapStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_matcher_stats%(%): MatcherStats %{ RecordVal* r = new RecordVal(MatcherStats); @@ -423,6 +435,7 @@ function get_matcher_stats%(%): MatcherStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_broker_stats%(%): BrokerStats %{ RecordVal* r = new RecordVal(BrokerStats); @@ -441,3 +454,39 @@ function get_broker_stats%(%): BrokerStats return r; %} + +## Returns statistics about reporter messages and weirds. +## +## Returns: A record with reporter statistics. +## +## .. bro:see:: get_conn_stats +## get_dns_stats +## get_event_stats +## get_file_analysis_stats +## get_gap_stats +## get_matcher_stats +## get_net_stats +## get_proc_stats +## get_reassembler_stats +## get_thread_stats +## get_timer_stats +## get_broker_stats +function get_reporter_stats%(%): ReporterStats + %{ + RecordVal* r = new RecordVal(ReporterStats); + int n = 0; + + TableVal* weirds_by_type = new TableVal(internal_type("table_string_of_count")->AsTableType()); + + for ( auto& kv : reporter->GetWeirdsByType() ) + { + Val* weird = new StringVal(kv.first); + weirds_by_type->Assign(weird, new Val(kv.second, TYPE_COUNT)); + Unref(weird); + } + + r->Assign(n++, new Val(reporter->GetWeirdCount(), TYPE_COUNT)); + r->Assign(n++, weirds_by_type); + + return r; + %} diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 449f2a8ad1..b07ab0d3d5 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -160,6 +160,7 @@ void Manager::Process() for ( all_thread_list::iterator i = to_delete.begin(); i != to_delete.end(); i++ ) { BasicThread* t = *i; + t->WaitForStop(); all_threads.remove(t); diff --git a/src/threading/MsgThread.cc b/src/threading/MsgThread.cc index c603f20625..dc399020da 100644 --- a/src/threading/MsgThread.cc +++ b/src/threading/MsgThread.cc @@ -21,6 +21,8 @@ public: network_time(network_time) { } virtual bool Process() { + if ( Object()->child_finished ) + return true; bool result = Object()->OnFinish(network_time); Object()->Finished(); return result; @@ -90,7 +92,13 @@ public: KillMeMessage(MsgThread* thread) : OutputMessage("ReporterMessage", thread) {} - virtual bool Process() { thread_mgr->KillThread(Object()); return true; } + virtual bool Process() + { + Object()->SignalStop(); + Object()->WaitForStop(); + thread_mgr->KillThread(Object()); + return true; + } }; #ifdef DEBUG diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 60da2d9668..1a5834c021 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -181,6 +181,7 @@ protected: friend class HeartbeatMessage; friend class FinishMessage; friend class FinishedMessage; + friend class KillMeMessage; /** * Pops a message sent by the child from the child-to-main queue. diff --git a/testing/btest/Baseline/bifs.dump_current_packet/1.hex b/testing/btest/Baseline/bifs.dump_current_packet/1.hex new file mode 100644 index 0000000000..69fc169df6 --- /dev/null +++ b/testing/btest/Baseline/bifs.dump_current_packet/1.hex @@ -0,0 +1,9 @@ +00000000 d4 c3 b2 a1 02 00 04 00 00 00 00 00 00 00 00 00 |................| +00000010 00 24 00 00 01 00 00 00 1f ad 83 4d 17 79 01 00 |.$.........M.y..| +00000020 57 00 00 00 57 00 00 00 01 00 5e 00 00 fb 00 30 |W...W.....^....0| +00000030 48 bd 3e c4 08 00 45 00 00 49 00 00 40 00 ff 11 |H.>...E..I..@...| +00000040 30 4f 8d 8e dc ca e0 00 00 fb 14 e9 14 e9 00 35 |0O.............5| +00000050 7f 62 00 00 00 00 00 01 00 00 00 00 00 00 06 67 |.b.............g| +00000060 65 6d 69 6e 69 09 5f 73 66 74 70 2d 73 73 68 04 |emini._sftp-ssh.| +00000070 5f 74 63 70 05 6c 6f 63 61 6c 00 00 21 00 01 |_tcp.local..!..| +0000007f diff --git a/testing/btest/Baseline/bifs.dump_current_packet/1.pcap b/testing/btest/Baseline/bifs.dump_current_packet/1.pcap deleted file mode 100644 index 786971a6cf..0000000000 Binary files a/testing/btest/Baseline/bifs.dump_current_packet/1.pcap and /dev/null differ diff --git a/testing/btest/Baseline/bifs.dump_current_packet/2.hex b/testing/btest/Baseline/bifs.dump_current_packet/2.hex new file mode 100644 index 0000000000..0753a3427f --- /dev/null +++ b/testing/btest/Baseline/bifs.dump_current_packet/2.hex @@ -0,0 +1,17 @@ +00000000 d4 c3 b2 a1 02 00 04 00 00 00 00 00 00 00 00 00 |................| +00000010 00 24 00 00 01 00 00 00 1f ad 83 4d f4 7a 01 00 |.$.........M.z..| +00000020 d5 00 00 00 d5 00 00 00 33 33 00 00 00 fb 00 17 |........33......| +00000030 f2 d7 cf 65 86 dd 60 00 00 00 00 9f 11 ff fe 80 |...e..`.........| +00000040 00 00 00 00 00 00 02 17 f2 ff fe d7 cf 65 ff 02 |.............e..| +00000050 00 00 00 00 00 00 00 00 00 00 00 00 00 fb 14 e9 |................| +00000060 14 e9 00 9f 44 e9 00 00 84 00 00 00 00 01 00 00 |....D...........| +00000070 00 04 06 67 65 6d 69 6e 69 09 5f 73 66 74 70 2d |...gemini._sftp-| +00000080 73 73 68 04 5f 74 63 70 05 6c 6f 63 61 6c 00 00 |ssh._tcp.local..| +00000090 21 80 01 00 00 00 78 00 0f 00 00 00 00 00 16 06 |!.....x.........| +000000a0 67 65 6d 69 6e 69 c0 22 c0 39 00 1c 80 01 00 00 |gemini.".9......| +000000b0 00 78 00 10 fe 80 00 00 00 00 00 00 02 17 f2 ff |.x..............| +000000c0 fe d7 cf 65 c0 39 00 01 80 01 00 00 00 78 00 04 |...e.9.......x..| +000000d0 8d 8e dc 32 c0 0c 00 2f 80 01 00 00 00 78 00 09 |...2.../.....x..| +000000e0 c0 0c 00 05 00 00 80 00 40 c0 39 00 2f 80 01 00 |........@.9./...| +000000f0 00 00 78 00 08 c0 39 00 04 40 00 00 08 |..x...9..@...| +000000fd diff --git a/testing/btest/Baseline/bifs.dump_current_packet/2.pcap b/testing/btest/Baseline/bifs.dump_current_packet/2.pcap deleted file mode 100644 index 6045568b4b..0000000000 Binary files a/testing/btest/Baseline/bifs.dump_current_packet/2.pcap and /dev/null differ diff --git a/testing/btest/Baseline/core.pcap.dumper/output b/testing/btest/Baseline/core.pcap.dumper/output index 1fef85cb6f..e547fcaf4f 100644 --- a/testing/btest/Baseline/core.pcap.dumper/output +++ b/testing/btest/Baseline/core.pcap.dumper/output @@ -1,4 +1,4 @@ 2c2 < 00000010 ff ff 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |...........Ns...| --- -> 00000010 00 20 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |. .........Ns...| +> 00000010 00 24 00 00 01 00 00 00 1d a2 b2 4e 73 00 07 00 |.$.........Ns...| diff --git a/testing/btest/Baseline/core.reporter-weird-sampling/output b/testing/btest/Baseline/core.reporter-weird-sampling/output new file mode 100644 index 0000000000..b9be6227dd --- /dev/null +++ b/testing/btest/Baseline/core.reporter-weird-sampling/output @@ -0,0 +1,351 @@ +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out index 70a635b24f..cac0156707 100644 --- a/testing/btest/Baseline/coverage.find-bro-logs/out +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -55,4 +55,5 @@ traceroute tunnel unified2 weird +weird_stats x509 diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output index 03ba9cb3cd..8efd4099ad 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output @@ -36,7 +36,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output index 30c6b1040e..f1d91b8392 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output @@ -27,5 +27,5 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-reporter.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-reporter.log index 7b8743734c..f4b240d619 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-reporter.log +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-reporter.log @@ -1,12 +1,2 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path reporter -#open 2018-04-27-22-48-04 -#fields _write_ts _stream _system_name ts level message location -#types time string string time enum string string -1524869284.624934 reporter manager-1 0.000000 Reporter::WARNING WriterFrontend broker/Log::WRITER_ASCII expected 9 fields in write, got 6. Skipping line. (empty) -1524869284.679015 reporter manager-1 0.000000 Reporter::WARNING WriterFrontend cluster/Log::WRITER_ASCII expected 6 fields in write, got 3. Skipping line. (empty) -1524869299.534389 reporter manager-1 0.000000 Reporter::INFO received termination signal (empty) -#close 2018-04-27-22-48-19 +1535139819.649067 Reporter::INFO qux /home/jon/projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.logging.field-extension-cluster-error/field-extension-cluster-error.bro, line XX +1535139821.906059 bah manager-1 0.000000 Reporter::INFO qux /home/jon/projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.logging.field-extension-cluster-error/field-extension-cluster-error.bro, line XX diff --git a/testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/dce_rpc.log b/testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/dce_rpc.log new file mode 100644 index 0000000000..383f8be565 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/dce_rpc.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path dce_rpc +#open 2018-08-23-19-54-19 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p rtt named_pipe endpoint operation +#types time string addr port addr port interval string string string +1347446180.330312 CHhAvVGS1DHFjwGM9 192.168.122.145 55614 192.168.122.3 1024 0.005544 \\PIPE\\drsuapi drsuapi DRSBind +1347446180.336310 CHhAvVGS1DHFjwGM9 192.168.122.145 55614 192.168.122.3 1024 0.000788 \\PIPE\\drsuapi drsuapi DRSCrackNames +1347446180.369165 CHhAvVGS1DHFjwGM9 192.168.122.145 55614 192.168.122.3 1024 0.000580 \\PIPE\\drsuapi drsuapi DRSUnbind +#close 2018-08-23-19-54-19 diff --git a/testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/out b/testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/out new file mode 100644 index 0000000000..b54910b5e6 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dce-rpc.context/out @@ -0,0 +1,15 @@ +dce_rpc_bind :: fid == 0 +dce_rpc_bind :: ctx_id == 0 +dce_rpc_bind :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2 +dce_rpc_bind :: fid == 0 +dce_rpc_bind :: ctx_id == 1 +dce_rpc_bind :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2 +dce_rpc_bind :: fid == 0 +dce_rpc_bind :: ctx_id == 2 +dce_rpc_bind :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2 +dce_rpc_bind_ack :: fid == 0 +dce_rpc_bind_ack :: sec_addr == \PIPE\drsuapi +dce_rpc_alter_context :: fid == 0 +dce_rpc_alter_context :: ctx_id == 0 +dce_rpc_alter_context :: uuid == e3514235-4b06-11d1-ab04-00c04fc2dcd2 +dce_rpc_alter_context_resp :: fid == 0 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log new file mode 100644 index 0000000000..001da38e49 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log @@ -0,0 +1,13 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-26-23-11-27 +#fields ts name num_seen +#types time string count +1532646687.827249 weird3 1 +1532646687.827249 weird2 1000 +1532646687.827249 weird1 2000 +1532646692.877464 weird1 2 +#close 2018-07-26-23-11-34 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log new file mode 100644 index 0000000000..e44145ed86 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-27-00-20-35 +#fields ts name num_seen +#types time string count +1532650834.978616 my_weird 1000 +1532650840.011592 my_weird 2000 +1532650845.043367 my_weird 10 +#close 2018-07-27-00-20-47 diff --git a/testing/btest/Traces/dce-rpc/cs_window7-join_stream092.pcap b/testing/btest/Traces/dce-rpc/cs_window7-join_stream092.pcap new file mode 100644 index 0000000000..c867e07e74 Binary files /dev/null and b/testing/btest/Traces/dce-rpc/cs_window7-join_stream092.pcap differ diff --git a/testing/btest/bifs/dump_current_packet.bro b/testing/btest/bifs/dump_current_packet.bro index 61c96384e4..0a852037a4 100644 --- a/testing/btest/bifs/dump_current_packet.bro +++ b/testing/btest/bifs/dump_current_packet.bro @@ -1,6 +1,12 @@ # @TEST-EXEC: bro -b -r $TRACES/wikipedia.trace %INPUT -# @TEST-EXEC: btest-diff 1.pcap -# @TEST-EXEC: btest-diff 2.pcap +# @TEST-EXEC: hexdump -C 1.pcap >1.hex +# @TEST-EXEC: hexdump -C 2.pcap >2.hex +# @TEST-EXEC: btest-diff 1.hex +# @TEST-EXEC: btest-diff 2.hex + +# Note that the hex output will contain global pcap header information, +# including Bro's snaplen setting (so maybe check that out in the case +# you are reading this message due to this test failing in the future). global i: count = 0; diff --git a/testing/btest/core/leaks/input-raw.bro b/testing/btest/core/leaks/input-raw.bro index 1a7315bc2a..608ea25030 100644 --- a/testing/btest/core/leaks/input-raw.bro +++ b/testing/btest/core/leaks/input-raw.bro @@ -6,9 +6,9 @@ # # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b %INPUT -# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 8 || (btest-bg-wait -k 1 && false) +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got2 60 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input2.log >> input.log -# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 8 || (btest-bg-wait -k 1 && false) +# @TEST-EXEC: $SCRIPTS/wait-for-file bro/got6 15 || (btest-bg-wait -k 1 && false) # @TEST-EXEC: cat input3.log >> input.log # @TEST-EXEC: btest-bg-wait 60 diff --git a/testing/btest/core/pcap/dumper.bro b/testing/btest/core/pcap/dumper.bro index facb86a830..8f89987b37 100644 --- a/testing/btest/core/pcap/dumper.bro +++ b/testing/btest/core/pcap/dumper.bro @@ -2,4 +2,8 @@ # @TEST-EXEC: hexdump -C $TRACES/workshop_2011_browse.trace >1 # @TEST-EXEC: hexdump -C dump >2 # @TEST-EXEC: diff 1 2 >output || true + +# Note that we're diff'ing the diff because there is an expected +# difference in the pcaps: namely, the snaplen setting stored in the +# global pcap header. # @TEST-EXEC: btest-diff output diff --git a/testing/btest/core/reporter-weird-sampling.bro b/testing/btest/core/reporter-weird-sampling.bro new file mode 100644 index 0000000000..d9d99681c4 --- /dev/null +++ b/testing/btest/core/reporter-weird-sampling.bro @@ -0,0 +1,55 @@ +# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output +# @TEST-EXEC: btest-diff output + +redef Weird::sampling_duration = 5sec; +redef Weird::sampling_threshold = 10; +redef Weird::sampling_rate = 10; +redef Weird::sampling_whitelist = set("whitelisted_net_weird", + "whitelisted_flow_weird", + "whitelisted_conn_weird"); + +event conn_weird(name: string, c: connection, addl: string) + { + print "conn_weird", name; + } + +event flow_weird(name: string, src: addr, dst: addr) + { + print "flow_weird", name; + } + +event net_weird(name: string) + { + print "net_weird", name; + } + +event gen_weirds(c: connection) + { + local num = 30; + + while ( num != 0 ) + { + Reporter::net_weird("my_net_weird"); + Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("my_conn_weird", c); + + Reporter::net_weird("whitelisted_net_weird"); + Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("whitelisted_conn_weird", c); + --num; + } + } + +global did_one_connection = F; + +event new_connection(c: connection) + { + if ( did_one_connection ) + return; + + did_one_connection = T; + event gen_weirds(c); # should permit 10 + 2 of each "my" weird + schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird + schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird + # Total of 27 "my" weirds of each type and 90 of each "whitelisted" type + } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest index 03ba9cb3cd..8efd4099ad 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest @@ -36,7 +36,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest index 30c6b1040e..f1d91b8392 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest @@ -27,5 +27,5 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro index 9c776fd974..57dca60afc 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro @@ -3,8 +3,9 @@ # @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT" # @TEST-EXEC: btest-bg-wait 20 -# @TEST-EXEC: cat manager-1/reporter.log | grep -v "reporter/" > manager-reporter.log -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | grep -v ^# | $SCRIPTS/diff-sort" btest-diff manager-reporter.log +# @TEST-EXEC: grep qux manager-1/reporter.log | sed 's#line ..#line XX#g' > manager-reporter.log +# @TEST-EXEC: grep qux manager-1/reporter-2.log | sed 's#line ..*#line XX#g' >> manager-reporter.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-canonifier | $SCRIPTS/diff-remove-abspath | grep -v ^# | $SCRIPTS/diff-sort" btest-diff manager-reporter.log @TEST-START-FILE cluster-layout.bro @@ -35,7 +36,7 @@ type Extension: record { function add_extension(path: string): Extension { return Extension($write_ts = network_time(), - $stream = path, + $stream = "bah", $system_name = peer_description); } @@ -56,16 +57,31 @@ event slow_death() event kill_worker() { + Reporter::info("qux"); Broker::publish("death", slow_death); } event bro_init() { if ( Cluster::node == "worker-1" ) + { Broker::subscribe("death"); + suspend_processing(); + } + } +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { if ( Cluster::node == "manager-1" ) - schedule 13sec { kill_worker() }; + { + schedule 2sec { kill_worker() }; + } + + if ( Cluster::node == "worker-1" ) + { + continue_processing(); + Reporter::info("qux"); + } } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro index 9ad0f526be..066c1983e7 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro @@ -57,13 +57,19 @@ event kill_worker() event bro_init() { if ( Cluster::node == "worker-1" ) + { + suspend_processing(); Broker::subscribe("death"); + } } event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) { if ( Cluster::node == "manager-1" ) schedule 2sec { kill_worker() }; + + if ( Cluster::node == "worker-1" ) + continue_processing(); } event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) diff --git a/testing/btest/scripts/base/protocols/dce-rpc/context.bro b/testing/btest/scripts/base/protocols/dce-rpc/context.bro new file mode 100644 index 0000000000..cb0d93383b --- /dev/null +++ b/testing/btest/scripts/base/protocols/dce-rpc/context.bro @@ -0,0 +1,31 @@ +# @TEST-EXEC: bro -b -C -r $TRACES/dce-rpc/cs_window7-join_stream092.pcap %INPUT >out +# @TEST-EXEC: btest-diff out +# @TEST-EXEC: btest-diff dce_rpc.log + +@load base/protocols/dce-rpc + +event dce_rpc_bind(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5 + { + print fmt("dce_rpc_bind :: fid == %s", fid); + print fmt("dce_rpc_bind :: ctx_id == %s", ctx_id); + print fmt("dce_rpc_bind :: uuid == %s", uuid_to_string(uuid)); + } + +event dce_rpc_alter_context(c: connection, fid: count, ctx_id: count, uuid: string, ver_major: count, ver_minor: count) &priority=5 + { + print fmt("dce_rpc_alter_context :: fid == %s", fid); + print fmt("dce_rpc_alter_context :: ctx_id == %s", ctx_id); + print fmt("dce_rpc_alter_context :: uuid == %s", uuid_to_string(uuid)); + } + + +event dce_rpc_bind_ack(c: connection, fid: count, sec_addr: string) &priority=5 + { + print fmt("dce_rpc_bind_ack :: fid == %s", fid); + print fmt("dce_rpc_bind_ack :: sec_addr == %s", sec_addr); + } + +event dce_rpc_alter_context_resp(c: connection, fid: count) &priority=5 + { + print fmt("dce_rpc_alter_context_resp :: fid == %s", fid); + } diff --git a/testing/btest/scripts/policy/misc/weird-stats-cluster.bro b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro new file mode 100644 index 0000000000..492b682c4d --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro @@ -0,0 +1,93 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 + +# @TEST-EXEC: btest-diff manager-1/weird_stats.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"], +}; +@TEST-END-FILE + +@load misc/weird-stats + +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + +redef Log::enable_local_logging = T; +redef Log::default_rotation_interval = 0secs; +redef WeirdStats::weird_stat_interval = 5secs; + +event terminate_me() + { + terminate(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event ready_again() + { + Reporter::net_weird("weird1"); + + if ( Cluster::node == "worker-2" ) + { + schedule 5secs { terminate_me() }; + } + } + +event ready_for_data() + { + local n = 0; + + if ( Cluster::node == "worker-1" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + ++n; + } + + Reporter::net_weird("weird3"); + } + else if ( Cluster::node == "worker-2" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + Reporter::net_weird("weird2"); + ++n; + } + } + + schedule 5secs { ready_again() }; + } + + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +event bro_init() + { + Broker::auto_publish(Cluster::worker_topic, ready_for_data); + } + +global peer_count = 0; + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peer_count; + + if ( peer_count == 2 ) + event ready_for_data(); + } + +@endif diff --git a/testing/btest/scripts/policy/misc/weird-stats.bro b/testing/btest/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..b26fce8e47 --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,32 @@ +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff bro/weird_stats.log + +@load misc/weird-stats.bro + +redef exit_only_after_terminate = T; +redef WeirdStats::weird_stat_interval = 5sec; + +event die() + { + terminate(); + } + +event gen_weirds(n: count, done: bool &default = F) + { + while ( n != 0 ) + { + Reporter::net_weird("my_weird"); + --n; + } + + if ( done ) + schedule 5sec { die() }; + } + +event bro_init() + { + event gen_weirds(1000); + schedule 7.5sec { gen_weirds(2000) } ; + schedule 12.5sec { gen_weirds(10, T) } ; + } diff --git a/testing/external/commit-hash.bro-testing b/testing/external/commit-hash.bro-testing new file mode 100644 index 0000000000..9b0c9547a2 --- /dev/null +++ b/testing/external/commit-hash.bro-testing @@ -0,0 +1 @@ +489007e9bd035777cd1d33a9e1db9857449f805b diff --git a/testing/external/commit-hash.bro-testing-private b/testing/external/commit-hash.bro-testing-private new file mode 100644 index 0000000000..3375ace0c2 --- /dev/null +++ b/testing/external/commit-hash.bro-testing-private @@ -0,0 +1 @@ +1917c0c602b18aa52150c7a940e9038a4382aa44 diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps index 770a181c59..325268d832 100755 --- a/testing/scripts/diff-remove-timestamps +++ b/testing/scripts/diff-remove-timestamps @@ -9,5 +9,5 @@ else sed="sed -E" fi -$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ +$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' diff --git a/testing/scripts/travis-job b/testing/scripts/travis-job index 01065900dd..6edf13922a 100644 --- a/testing/scripts/travis-job +++ b/testing/scripts/travis-job @@ -198,10 +198,24 @@ run() { make init fi + if [ -d bro-testing ]; then + commit=`cat commit-hash.bro-testing` + echo "Checking out $commit" + ( cd bro-testing && git checkout -q $commit ) + fi + + echo + if [ ! -d bro-testing-private ]; then get_private_tests fi + if [ -d bro-testing-private ]; then + commit=`cat commit-hash.bro-testing-private` + echo "Checking out $commit" + ( cd bro-testing-private && git checkout -q $commit ) + fi + echo echo "Running external tests ##############################################" echo