From fcaed26796416c24146066637229f02a2ea3a1c0 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 24 Jul 2018 12:49:10 -0500 Subject: [PATCH 01/24] Add script to support the old DHCP events --- .../protocols/dhcp/deprecated_events.bro | 278 ++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 scripts/policy/protocols/dhcp/deprecated_events.bro diff --git a/scripts/policy/protocols/dhcp/deprecated_events.bro b/scripts/policy/protocols/dhcp/deprecated_events.bro new file mode 100644 index 0000000000..ed84dfa78e --- /dev/null +++ b/scripts/policy/protocols/dhcp/deprecated_events.bro @@ -0,0 +1,278 @@ +##! Bro 2.6 removed certain DHCP events, but scripts in the Bro +##! ecosystem are still relying on those events. As a transition, this +##! script will handle the new event, and generate the old events, +##! which are marked as deprecated. Note: This script should be +##! removed in the next Bro version after 2.6 + +@load dhcp + +module DHCP; + +export { + ## A DHCP message. + ## + ## .. note:: This type is included to support the deprecated events dhcp_ack, + ## dhcp_decline, dhcp_discover, dhcp_inform, dhcp_nak, dhcp_offer, + ## dhcp_release and dhcp_request and is thus similarly deprecated + ## itself. Use dhcp_message instead. + ## + ## .. bro:see:: dhcp_message dhcp_ack dhcp_decline dhcp_discover + ## dhcp_inform dhcp_nak dhcp_offer dhcp_release dhcp_request + type dhcp_msg: record { + op: count; ##< Message OP code. 1 = BOOTREQUEST, 2 = BOOTREPLY + m_type: count; ##< The type of DHCP message. + xid: count; ##< Transaction ID of a DHCP session. + h_addr: string; ##< Hardware address of the client. + ciaddr: addr; ##< Original IP address of the client. + yiaddr: addr; ##< IP address assigned to the client. + }; + + ## A list of router addresses offered by a DHCP server. + ## + ## .. note:: This type is included to support the deprecated events dhcp_ack + ## and dhcp_offer and is thus similarly deprecated + ## itself. Use dhcp_message instead. + ## + ## .. bro:see:: dhcp_message dhcp_ack dhcp_offer + type dhcp_router_list: table[count] of addr; + + ## Generated for DHCP messages of type *DHCPDISCOVER* (client broadcast to locate + ## available servers). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## req_addr: The specific address requested by the client. + ## + ## host_name: The value of the host name option, if specified by the client. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request + ## dhcp_decline dhcp_ack dhcp_nak dhcp_release dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + ## .. note:: Bro does not support broadcast packets (as used by the DHCP + ## protocol). It treats broadcast addresses just like any other and + ## associates packets into transport-level flows in the same way as usual. + ## + global dhcp_discover: event(c: connection, msg: dhcp_msg, req_addr: addr, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPOFFER* (server to client in response + ## to DHCPDISCOVER with offer of configuration parameters). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## mask: The subnet mask specified by the message. + ## + ## router: The list of routers specified by the message. + ## + ## lease: The least interval specified by the message. + ## + ## serv_addr: The server address specified by the message. + ## + ## host_name: Optional host name value. May differ from the host name requested + ## from the client. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_request dhcp_decline + ## dhcp_ack dhcp_nak dhcp_release dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + ## .. note:: Bro does not support broadcast packets (as used by the DHCP + ## protocol). It treats broadcast addresses just like any other and + ## associates packets into transport-level flows in the same way as usual. + ## + global dhcp_offer: event(c: connection, msg: DHCP::dhcp_msg, mask: addr, router: DHCP::dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPREQUEST* (Client message to servers either + ## (a) requesting offered parameters from one server and implicitly declining offers + ## from all others, (b) confirming correctness of previously allocated address after, + ## e.g., system reboot, or (c) extending the lease on a particular network address.) + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## req_addr: The client address specified by the message. + ## + ## serv_addr: The server address specified by the message. + ## + ## host_name: The value of the host name option, if specified by the client. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_decline + ## dhcp_ack dhcp_nak dhcp_release dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + ## .. note:: Bro does not support broadcast packets (as used by the DHCP + ## protocol). It treats broadcast addresses just like any other and + ## associates packets into transport-level flows in the same way as usual. + ## + global dhcp_request: event(c: connection, msg: DHCP::dhcp_msg, req_addr: addr, serv_addr: addr, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPDECLINE* (Client to server indicating + ## network address is already in use). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## host_name: Optional host name value. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request + ## dhcp_ack dhcp_nak dhcp_release dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + ## .. note:: Bro does not support broadcast packets (as used by the DHCP + ## protocol). It treats broadcast addresses just like any other and + ## associates packets into transport-level flows in the same way as usual. + ## + global dhcp_decline: event(c: connection, msg: DHCP::dhcp_msg, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPACK* (Server to client with configuration + ## parameters, including committed network address). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## mask: The subnet mask specified by the message. + ## + ## router: The list of routers specified by the message. + ## + ## lease: The least interval specified by the message. + ## + ## serv_addr: The server address specified by the message. + ## + ## host_name: Optional host name value. May differ from the host name requested + ## from the client. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request + ## dhcp_decline dhcp_nak dhcp_release dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + global dhcp_ack: event(c: connection, msg: DHCP::dhcp_msg, mask: addr, router: DHCP::dhcp_router_list, lease: interval, serv_addr: addr, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPNAK* (Server to client indicating client's + ## notion of network address is incorrect (e.g., client has moved to new subnet) or + ## client's lease has expired). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## host_name: Optional host name value. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request + ## dhcp_decline dhcp_ack dhcp_release dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + ## .. note:: Bro does not support broadcast packets (as used by the DHCP + ## protocol). It treats broadcast addresses just like any other and + ## associates packets into transport-level flows in the same way as usual. + ## + global dhcp_nak: event(c: connection, msg: DHCP::dhcp_msg, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPRELEASE* (Client to server relinquishing + ## network address and cancelling remaining lease). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## host_name: The value of the host name option, if specified by the client. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request + ## dhcp_decline dhcp_ack dhcp_nak dhcp_inform + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + global dhcp_release: event(c: connection, msg: DHCP::dhcp_msg, host_name: string) &deprecated; + + ## Generated for DHCP messages of type *DHCPINFORM* (Client to server, asking only for + ## local configuration parameters; client already has externally configured network + ## address). + ## + ## c: The connection record describing the underlying UDP flow. + ## + ## msg: The parsed type-independent part of the DHCP message. + ## + ## host_name: The value of the host name option, if specified by the client. + ## + ## .. bro:see:: dhcp_message dhcp_discover dhcp_offer dhcp_request + ## dhcp_decline dhcp_ack dhcp_nak dhcp_release + ## + ## .. note:: This event has been deprecated, and will be removed in the next version. + ## Use dhcp_message instead. + ## + ## .. note:: Bro does not support broadcast packets (as used by the DHCP + ## protocol). It treats broadcast addresses just like any other and + ## associates packets into transport-level flows in the same way as usual. + ## + global dhcp_inform: event(c: connection, msg: DHCP::dhcp_msg, host_name: string) &deprecated; + +} + +event dhcp_message(c: connection, is_orig: bool, msg: DHCP::Msg, options: DHCP::Options) + { + local old_msg: dhcp_msg = [$op=msg$op, $m_type=msg$m_type, $xid=msg$xid, + $h_addr=msg$chaddr, $ciaddr=msg$ciaddr, $yiaddr=msg$yiaddr]; + + local routers: dhcp_router_list; + for ( i in options$routers ) { + routers[|routers|] = options$routers[i]; + } + + # These fields are technically optional, but aren't listed as such in the event. + # We give it some defaults in order to suppress errors. + local ar = ( options?$addr_request ) ? options$addr_request : 0.0.0.0; + local hn = ( options?$host_name ) ? options$host_name : ""; + local le = ( options?$lease ) ? options$lease : 0 secs; + local sm = ( options?$subnet_mask ) ? options$subnet_mask : 255.255.255.255; + local sa = ( options?$serv_addr ) ? options$serv_addr : 0.0.0.0; + + + switch ( DHCP::message_types[msg$m_type] ) + { + case "DISCOVER": + event dhcp_discover(c, old_msg, ar, hn); + break; + case "OFFER": + event dhcp_offer(c, old_msg, sm, routers, le, sa, hn); + break; + case "REQUEST": + event dhcp_request(c, old_msg, ar, sa, hn); + break; + case "DECLINE": + event dhcp_decline(c, old_msg, hn); + break; + case "ACK": + event dhcp_ack(c, old_msg, sm, routers, le, sa, hn); + break; + case "NAK": + event dhcp_nak(c, old_msg, hn); + break; + case "RELEASE": + event dhcp_release(c, old_msg, hn); + break; + case "INFORM": + event dhcp_inform(c, old_msg, hn); + break; + default: + # This isn't a weird, it's just a DHCP message type the old scripts don't handle + break; + } + } From e60b0bfb25f28a4ad374b736529d1e39ff058d89 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 26 Jul 2018 15:10:45 -0500 Subject: [PATCH 02/24] Teach timestamp canonifier about timestamps before ~2001 Still wouldn't work with stamps before ~1973, but that's likely ok. --- testing/scripts/diff-remove-timestamps | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/scripts/diff-remove-timestamps b/testing/scripts/diff-remove-timestamps index 770a181c59..325268d832 100755 --- a/testing/scripts/diff-remove-timestamps +++ b/testing/scripts/diff-remove-timestamps @@ -9,5 +9,5 @@ else sed="sed -E" fi -$sed 's/(0\.000000)|([0-9]{10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ +$sed 's/(0\.000000)|([0-9]{9,10}\.[0-9]{2,8})/XXXXXXXXXX.XXXXXX/g' | \ $sed 's/^ *#(open|close).(19|20)..-..-..-..-..-..$/#\1 XXXX-XX-XX-XX-XX-XX/g' From 35827eeb31987a096031a1b4dd782dea64efa2b5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 26 Jul 2018 19:57:36 -0500 Subject: [PATCH 03/24] Add rate-limiting sampling mechanism for weird events The generation of weird events, by default, are now rate-limited according to these tunable options: - Weird::sampling_whitelist - Weird::sampling_threshold - Weird::sampling_rate - Weird::sampling_duration The new get_reporter_stats() BIF also allows one to query the total number of weirds generated (pre-sampling) which the new policy/misc/weird-stats.bro script uses periodically to populate a weird_stats.log. There's also new reporter BIFs to allow generating weirds from the script-layer such that they go through the same, internal rate-limiting/sampling mechanisms: - Reporter::conn_weird - Reporter::flow_weird - Reporter::net_weird Some of the code was adapted from previous work by Johanna Amann. --- NEWS | 15 + doc/script-reference/log-files.rst | 2 + scripts/base/init-bare.bro | 47 +++ scripts/base/protocols/http/main.bro | 2 +- scripts/base/protocols/sip/main.bro | 2 +- scripts/base/protocols/ssl/main.bro | 2 +- scripts/policy/misc/weird-stats.bro | 100 +++++ scripts/policy/protocols/smb/smb1-main.bro | 2 +- scripts/test-all-policy.bro | 1 + src/Conn.cc | 25 ++ src/Conn.h | 14 + src/Func.cc | 1 + src/Reporter.cc | 132 ++++++- src/Reporter.h | 51 ++- src/Timer.cc | 2 + src/Timer.h | 2 + src/reporter.bif | 41 ++ src/stats.bif | 49 +++ .../core.reporter-weird-sampling/output | 351 ++++++++++++++++++ .../btest/Baseline/coverage.find-bro-logs/out | 1 + .../manager-1.weird_stats.log | 13 + .../bro.weird_stats.log | 12 + .../btest/core/reporter-weird-sampling.bro | 55 +++ .../policy/misc/weird-stats-cluster.bro | 93 +++++ .../btest/scripts/policy/misc/weird-stats.bro | 32 ++ 25 files changed, 1037 insertions(+), 10 deletions(-) create mode 100644 scripts/policy/misc/weird-stats.bro create mode 100644 testing/btest/Baseline/core.reporter-weird-sampling/output create mode 100644 testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log create mode 100644 testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log create mode 100644 testing/btest/core/reporter-weird-sampling.bro create mode 100644 testing/btest/scripts/policy/misc/weird-stats-cluster.bro create mode 100644 testing/btest/scripts/policy/misc/weird-stats.bro diff --git a/NEWS b/NEWS index 3cb0fd79b3..a878c9398e 100644 --- a/NEWS +++ b/NEWS @@ -356,6 +356,21 @@ Changed Functionality - The string_to_pattern() built-in (and the now-deprecated merge_pattern() built-in) is no longer restricted to only be called at initialization time. +- "Weird" events are now generally suppressed/sampled by default according to + some tunable parameters: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + Those options can be changed if one needs the previous behavior of + a "net_weird", "flow_weird", or "conn_weird" event being raised for + every single event. Otherwise, there is a new weird_stats.log which + contains concise summaries of weird counts per type per time period + and the original weird.log may not differ much either, except in + the cases where a particular weird type exceeds the sampling threshold. + Removed Functionality --------------------- diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 8f7d87d89a..513ece8864 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -154,6 +154,8 @@ Miscellaneous +----------------------------+---------------------------------------+---------------------------------+ | weird.log | Unexpected network-level activity | :bro:type:`Weird::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| weird-stats.log | Statistics about unexpected activity | :bro:type:`WeirdStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ Bro Diagnostics --------------- diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index c502607cbd..c37f924dfc 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -82,6 +82,13 @@ type addr_vec: vector of addr; ## directly and then remove this alias. type table_string_of_string: table[string] of string; +## A table of counts indexed by strings. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type table_string_of_count: table[string] of count; + ## A set of file analyzer tags. ## ## .. todo:: We need this type definition only for declaring builtin functions @@ -626,6 +633,17 @@ type BrokerStats: record { num_ids_outgoing: count; }; +## Statistics about reporter messages and weirds. +## +## .. bro:see:: get_reporter_stats +type ReporterStats: record { + ## Number of total weirds encountered, before any rate-limiting. + weirds: count; + ## Number of times each individual weird is encountered, before any + ## rate-limiting is applied. + weirds_by_type: table[string] of count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere @@ -4823,6 +4841,35 @@ export { type Cluster::Pool: record {}; } +module Weird; +export { + ## Prevents rate-limiting sampling of any weirds named in the table. + const sampling_whitelist: set[string] &redef; + + ## How many weirds of a given type to tolerate before sampling begins. + ## i.e. this many consecutive weirds of a given type will be allowed to + ## raise events for script-layer handling before being rate-limited. + const sampling_threshold = 25 &redef; + + ## The rate-limiting sampling rate. One out of every of this number of + ## rate-limited weirds of a given type will be allowed to raise events + ## for further script-layer handling. + const sampling_rate = 1000 &redef; + + ## How long a weird of a given type is allowed to keep state/counters in + ## memory. For "net" weirds an expiration timer starts per weird name when + ## first initializing its counter. For "flow" weirds an expiration timer + ## starts once per src/dst IP pair for the first weird of any name. For + ## "conn" weirds, counters and expiration timers are kept for the duration + ## of the connection for each named weird and reset when necessary. e.g. + ## if a "conn" weird by the name of "foo" is seen more than + ## :bro:see:`Weird::sampling_threshold` times, then an expiration timer + ## begins for "foo" and upon triggering will reset the counter for "foo" + ## and unthrottle its rate-limiting until it once again exceeds the + ## threshold. + const sampling_duration = 10min &redef; +} + module GLOBAL; ## Seed for hashes computed internally for probabilistic data structures. Using diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 51a89a33b9..ef949df4e3 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string, c$http$uri = unescaped_URI; if ( method !in http_methods ) - event conn_weird("unknown_HTTP_method", c, method); + Reporter::conn_weird("unknown_HTTP_method", c, method); } event http_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index f629049928..09d9609481 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version: c$sip$uri = original_URI; if ( method !in sip_methods ) - event conn_weird("unknown_SIP_method", c, method); + Reporter::conn_weird("unknown_SIP_method", c, method); } event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 6ac4260537..85aedac063 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) { c$ssl$server_name = names[0]; if ( |names| > 1 ) - event conn_weird("SSL_many_server_names", c, cat(names)); + Reporter::conn_weird("SSL_many_server_names", c, cat(names)); } } diff --git a/scripts/policy/misc/weird-stats.bro b/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..d08ce9381d --- /dev/null +++ b/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,100 @@ +##! Log weird statistics. + +@load base/frameworks/sumstats +@load base/frameworks/cluster + +module WeirdStats; + +export { + redef enum Log::ID += { LOG }; + + ## How often stats are reported. + const weird_stat_interval = 15min &redef; + + type Info: record { + ## Timestamp for the measurement. + ts: time &log; + ## Name of the weird. + name: string &log; + ## Number of times weird was seen since the last stats interval. + num_seen: count &log; + }; + + global log_weird_stats: event(rec: Info); +} + +global this_epoch_weirds: table[string] of double; +global last_epoch_weirds: table[string] of double; + +function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result) + { + this_epoch_weirds[key$str]=result["weirds.encountered"]$sum; + } + +function weird_epoch_finished(ts: time) + { + for ( n in this_epoch_weirds ) + { + local last_count: double = 0.0; + + if ( n in last_epoch_weirds ) + last_count = last_epoch_weirds[n]; + + local num_seen: double = this_epoch_weirds[n] - last_count; + + if ( num_seen > 0.0 ) + Log::write(LOG, Info($ts = ts, $name = n, + $num_seen = double_to_count(num_seen))); + } + + last_epoch_weirds = this_epoch_weirds; + this_epoch_weirds = table(); + } + +event bro_init() &priority=5 + { + Log::create_stream(WeirdStats::LOG, + [$columns = Info, $ev = log_weird_stats, + $path="weird_stats"]); + local r1 = SumStats::Reducer($stream = "weirds.encountered", + $apply = set(SumStats::SUM)); + SumStats::create([$name = "weirds.statistics", + $epoch = weird_stat_interval, $reducers = set(r1), + $epoch_result = weird_epoch_results, + $epoch_finished = weird_epoch_finished]); + } + +module SumStats; + +function observe_weird_stats() + { + local rs = get_reporter_stats(); + + for ( n in rs$weirds_by_type ) + SumStats::observe("weirds.encountered", SumStats::Key($str = n), + SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0)); + } + +@if ( Cluster::is_enabled() ) + +# I'm not sure if this is a hack or not: the manager will generate this +# event at the end of its epoch so workers can handle it just in time to +# generate the necessary stats. Alternative may be workers generating the +# stats individually/proactively in their own finish_epoch, but that may be +# less synchronized? +event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10 + { + observe_weird_stats(); + } + +@else + +event SumStats::finish_epoch(ss: SumStat) &priority=10 + { + if ( ss$name != "weirds.statistics" ) + return; + + observe_weird_stats(); + } + +@endif diff --git a/scripts/policy/protocols/smb/smb1-main.bro b/scripts/policy/protocols/smb/smb1-main.bro index 6b23fe91db..3e0e7da3ed 100644 --- a/scripts/policy/protocols/smb/smb1-main.bro +++ b/scripts/policy/protocols/smb/smb1-main.bro @@ -316,7 +316,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count) if ( ! f?$uuid ) { # TODO: figure out why this is happening. - event conn_weird("smb_pipe_request_missing_uuid", c, ""); + Reporter::conn_weird("smb_pipe_request_missing_uuid", c, ""); return; } local arg = fmt("%s: %s", diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 3389942fe5..cf4a35399f 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -54,6 +54,7 @@ @load misc/profiling.bro @load misc/scan.bro @load misc/stats.bro +@load misc/weird-stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @load protocols/conn/known-services.bro diff --git a/src/Conn.cc b/src/Conn.cc index 1edecde0b9..719d55d3b0 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -1027,3 +1027,28 @@ void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label) else saw_first_resp_packet = 1; } + +bool Connection::PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration) + { + auto& state = weird_state[name]; + ++state.count; + + if ( state.count < threshold ) + return true; + + if ( state.count == threshold ) + state.sampling_start_time = network_time; + else + { + if ( network_time > state.sampling_start_time + duration ) + { + state.sampling_start_time = 0; + state.count = 1; + return true; + } + } + + auto num_above_threshold = state.count - threshold; + return num_above_threshold % rate == 0; + } diff --git a/src/Conn.h b/src/Conn.h index db0cb2fe55..ee73ab24d5 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -5,6 +5,9 @@ #include +#include +#include + #include "Dict.h" #include "Val.h" #include "Timer.h" @@ -264,6 +267,9 @@ public: uint32 GetOrigFlowLabel() { return orig_flow_label; } uint32 GetRespFlowLabel() { return resp_flow_label; } + bool PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration); + protected: Connection() { persistent = 0; } @@ -328,6 +334,14 @@ protected: analyzer::pia::PIA* primary_PIA; Bro::UID uid; // Globally unique connection ID. + + struct WeirdState { + WeirdState() { count = 0; sampling_start_time = 0; } + uint64 count = 0; + double sampling_start_time = 0; + }; + + std::unordered_map weird_state; }; class ConnectionTimer : public Timer { diff --git a/src/Func.cc b/src/Func.cc index 703cc04d29..015829e2a1 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -722,6 +722,7 @@ void init_builtin_funcs() FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); BrokerStats = internal_type("BrokerStats")->AsRecordType(); + ReporterStats = internal_type("ReporterStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/Reporter.cc b/src/Reporter.cc index eb89a29d30..59e877bb7c 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -10,6 +10,7 @@ #include "NetVar.h" #include "Net.h" #include "Conn.h" +#include "Timer.h" #include "plugin/Plugin.h" #include "plugin/Manager.h" @@ -36,6 +37,11 @@ Reporter::Reporter() warnings_to_stderr = true; errors_to_stderr = true; + weird_count = 0; + weird_sampling_rate = 0; + weird_sampling_duration = 0; + weird_sampling_threshold = 0; + openlog("bro", 0, LOG_LOCAL5); } @@ -49,6 +55,24 @@ void Reporter::InitOptions() info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool(); warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool(); errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool(); + weird_sampling_rate = internal_const_val("Weird::sampling_rate")->AsCount(); + weird_sampling_threshold = internal_const_val("Weird::sampling_threshold")->AsCount(); + weird_sampling_duration = internal_const_val("Weird::sampling_duration")->AsInterval(); + auto wl_val = internal_const_val("Weird::sampling_whitelist")->AsTableVal(); + auto wl_table = wl_val->AsTable(); + + HashKey* k; + IterCookie* c = wl_table->InitForIteration(); + TableEntryVal* v; + + while ( (v = wl_table->NextEntry(k, c)) ) + { + auto index = wl_val->RecoverIndex(k); + string key = index->Index(0)->AsString()->CheckString(); + weird_sampling_whitelist.emplace(move(key)); + Unref(index); + delete k; + } } void Reporter::Info(const char* fmt, ...) @@ -221,23 +245,121 @@ void Reporter::WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const cha delete vl; } +void Reporter::UpdateWeirdStats(const char* name) + { + ++weird_count; + ++weird_count_by_type[name]; + } + +class NetWeirdTimer : public Timer { +public: + NetWeirdTimer(double t, const char* name, double timeout) + : Timer(t + timeout, TIMER_NET_WEIRD_EXPIRE), weird_name(name) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetNetWeird(weird_name); } + + std::string weird_name; +}; + +class FlowWeirdTimer : public Timer { +public: + using IPPair = std::pair; + + FlowWeirdTimer(double t, IPPair p, double timeout) + : Timer(t + timeout, TIMER_FLOW_WEIRD_EXPIRE), endpoints(p) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetFlowWeird(endpoints.first, endpoints.second); } + + IPPair endpoints; +}; + +void Reporter::ResetNetWeird(const std::string& name) + { + net_weird_state.erase(name); + } + +void Reporter::ResetFlowWeird(const IPAddr& orig, const IPAddr& resp) + { + flow_weird_state.erase(std::make_pair(orig, resp)); + } + +bool Reporter::PermitNetWeird(const char* name) + { + auto& count = net_weird_state[name]; + ++count; + + if ( count == 1 ) + timer_mgr->Add(new NetWeirdTimer(network_time, name, + weird_sampling_duration)); + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + +bool Reporter::PermitFlowWeird(const char* name, + const IPAddr& orig, const IPAddr& resp) + { + auto endpoints = std::make_pair(orig, resp); + auto& map = flow_weird_state[endpoints]; + + if ( map.empty() ) + timer_mgr->Add(new FlowWeirdTimer(network_time, endpoints, + weird_sampling_duration)); + + auto& count = map[name]; + ++count; + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + void Reporter::Weird(const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitNetWeird(name) ) + return; + } + WeirdHelper(net_weird, 0, 0, "%s", name); } void Reporter::Weird(Connection* conn, const char* name, const char* addl) { - WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); - } + UpdateWeirdStats(name); -void Reporter::Weird(Val* conn_val, const char* name, const char* addl) - { - WeirdHelper(conn_weird, conn_val, addl, "%s", name); + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! conn->PermitWeird(name, weird_sampling_threshold, + weird_sampling_rate, weird_sampling_duration) ) + return; + } + + WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); } void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitFlowWeird(name, orig, resp) ) + return; + } + WeirdFlowHelper(orig, resp, "%s", name); } diff --git a/src/Reporter.h b/src/Reporter.h index 85b9a483a2..f5090ee91e 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -7,6 +7,10 @@ #include #include +#include +#include +#include +#include #include "util.h" #include "EventHandler.h" @@ -36,6 +40,11 @@ protected: class Reporter { public: + using IPPair = std::pair; + using WeirdCountMap = std::unordered_map; + using WeirdFlowMap = std::map; + using WeirdSet = std::unordered_set; + Reporter(); ~Reporter(); @@ -76,7 +85,6 @@ public: // that may lead to incorrectly processing a connnection. void Weird(const char* name); // Raises net_weird(). void Weird(Connection* conn, const char* name, const char* addl = ""); // Raises conn_weird(). - void Weird(Val* conn_val, const char* name, const char* addl = ""); // Raises conn_weird(). void Weird(const IPAddr& orig, const IPAddr& resp, const char* name); // Raises flow_weird(). // Syslog a message. This methods does nothing if we're running @@ -120,6 +128,30 @@ public: // Signals that we're done processing an error handler event. void EndErrorHandler() { --in_error_handler; } + /** + * Reset/cleanup state tracking for a "net" weird. + */ + void ResetNetWeird(const std::string& name); + + /** + * Reset/cleanup state tracking for a "flow" weird. + */ + void ResetFlowWeird(const IPAddr& orig, const IPAddr& resp); + + /** + * Return the total number of weirds generated (counts weirds before + * any rate-limiting occurs). + */ + uint64 GetWeirdCount() const + { return weird_count; } + + /** + * Return number of weirds generated per weird type/name (counts weirds + * before any rate-limiting occurs). + */ + const WeirdCountMap& GetWeirdsByType() const + { return weird_count_by_type; } + private: void DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, @@ -129,6 +161,11 @@ private: // contain format specifiers void WeirdHelper(EventHandlerPtr event, Val* conn_val, const char* addl, const char* fmt_name, ...) __attribute__((format(printf, 5, 6)));; void WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const char* fmt_name, ...) __attribute__((format(printf, 4, 5)));; + void UpdateWeirdStats(const char* name); + inline bool WeirdOnSamplingWhiteList(const char* name) + { return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); } + bool PermitNetWeird(const char* name); + bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r); int errors; bool via_events; @@ -138,6 +175,18 @@ private: bool errors_to_stderr; std::list > locations; + + uint64 weird_count; + WeirdCountMap weird_count_by_type; + + WeirdCountMap net_weird_state; + WeirdFlowMap flow_weird_state; + + WeirdSet weird_sampling_whitelist; + uint64 weird_sampling_threshold; + uint64 weird_sampling_rate; + double weird_sampling_duration; + }; extern Reporter* reporter; diff --git a/src/Timer.cc b/src/Timer.cc index 4fe998606b..101733028c 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -18,12 +18,14 @@ const char* TimerNames[] = { "ConnectionStatusUpdateTimer", "DNSExpireTimer", "FileAnalysisInactivityTimer", + "FlowWeirdTimer", "FragTimer", "IncrementalSendTimer", "IncrementalWriteTimer", "InterconnTimer", "IPTunnelInactivityTimer", "NetbiosExpireTimer", + "NetWeirdTimer", "NetworkTimer", "NTPExpireTimer", "ProfileTimer", diff --git a/src/Timer.h b/src/Timer.h index ea410e5c7b..8d6de857a0 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -23,12 +23,14 @@ enum TimerType { TIMER_CONN_STATUS_UPDATE, TIMER_DNS_EXPIRE, TIMER_FILE_ANALYSIS_INACTIVITY, + TIMER_FLOW_WEIRD_EXPIRE, TIMER_FRAG, TIMER_INCREMENTAL_SEND, TIMER_INCREMENTAL_WRITE, TIMER_INTERCONN, TIMER_IP_TUNNEL_INACTIVITY, TIMER_NB_EXPIRE, + TIMER_NET_WEIRD_EXPIRE, TIMER_NETWORK, TIMER_NTP_EXPIRE, TIMER_PROFILE, diff --git a/src/reporter.bif b/src/reporter.bif index 3cdefc4da4..a6463f6226 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -71,3 +71,44 @@ function Reporter::fatal%(msg: string%): bool reporter->PopLocation(); return new Val(1, TYPE_BOOL); %} + +## Generates a "net" weird. +## +## name: the name of the weird. +## +## Returns: Always true. +function Reporter::net_weird%(name: string%): bool + %{ + reporter->Weird(name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "flow" weird. +## +## name: the name of the weird. +## +## orig: the originator host associated with the weird. +## +## resp: the responder host associated with the weird. +## +## Returns: Always true. +function Reporter::flow_weird%(name: string, orig: addr, resp: addr%): bool + %{ + reporter->Weird(orig->AsAddr(), resp->AsAddr(), name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "conn" weird. +## +## name: the name of the weird. +## +## c: the connection associated with the weird. +## +## addl: additional information to accompany the weird. +## +## Returns: Always true. +function Reporter::conn_weird%(name: string, c: connection, addl: string &default=""%): bool + %{ + reporter->Weird(c, name->CheckString(), addl->CheckString()); + return new Val(1, TYPE_BOOL); + %} diff --git a/src/stats.bif b/src/stats.bif index 2a525a1790..e3ce6f0245 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -16,6 +16,7 @@ RecordType* ThreadStats; RecordType* TimerStats; RecordType* FileAnalysisStats; RecordType* BrokerStats; +RecordType* ReporterStats; %%} ## Returns packet capture statistics. Statistics include the number of @@ -35,6 +36,7 @@ RecordType* BrokerStats; ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_net_stats%(%): NetStats %{ uint64 recv = 0; @@ -83,6 +85,7 @@ function get_net_stats%(%): NetStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_conn_stats%(%): ConnStats %{ RecordVal* r = new RecordVal(ConnStats); @@ -133,6 +136,7 @@ function get_conn_stats%(%): ConnStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_proc_stats%(%): ProcStats %{ struct rusage ru; @@ -189,6 +193,7 @@ function get_proc_stats%(%): ProcStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -215,6 +220,7 @@ function get_event_stats%(%): EventStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -243,6 +249,7 @@ function get_reassembler_stats%(%): ReassemblerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -276,6 +283,7 @@ function get_dns_stats%(%): DNSStats ## get_reassembler_stats ## get_thread_stats ## get_broker_stats +## get_reporter_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -303,6 +311,7 @@ function get_timer_stats%(%): TimerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -330,6 +339,7 @@ function get_file_analysis_stats%(%): FileAnalysisStats ## get_reassembler_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -355,6 +365,7 @@ function get_thread_stats%(%): ThreadStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_gap_stats%(%): GapStats %{ RecordVal* r = new RecordVal(GapStats); @@ -386,6 +397,7 @@ function get_gap_stats%(%): GapStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_matcher_stats%(%): MatcherStats %{ RecordVal* r = new RecordVal(MatcherStats); @@ -423,6 +435,7 @@ function get_matcher_stats%(%): MatcherStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_broker_stats%(%): BrokerStats %{ RecordVal* r = new RecordVal(BrokerStats); @@ -441,3 +454,39 @@ function get_broker_stats%(%): BrokerStats return r; %} + +## Returns statistics about reporter messages and weirds. +## +## Returns: A record with reporter statistics. +## +## .. bro:see:: get_conn_stats +## get_dns_stats +## get_event_stats +## get_file_analysis_stats +## get_gap_stats +## get_matcher_stats +## get_net_stats +## get_proc_stats +## get_reassembler_stats +## get_thread_stats +## get_timer_stats +## get_broker_stats +function get_reporter_stats%(%): ReporterStats + %{ + RecordVal* r = new RecordVal(ReporterStats); + int n = 0; + + TableVal* weirds_by_type = new TableVal(internal_type("table_string_of_count")->AsTableType()); + + for ( auto& kv : reporter->GetWeirdsByType() ) + { + Val* weird = new StringVal(kv.first); + weirds_by_type->Assign(weird, new Val(kv.second, TYPE_COUNT)); + Unref(weird); + } + + r->Assign(n++, new Val(reporter->GetWeirdCount(), TYPE_COUNT)); + r->Assign(n++, weirds_by_type); + + return r; + %} diff --git a/testing/btest/Baseline/core.reporter-weird-sampling/output b/testing/btest/Baseline/core.reporter-weird-sampling/output new file mode 100644 index 0000000000..b9be6227dd --- /dev/null +++ b/testing/btest/Baseline/core.reporter-weird-sampling/output @@ -0,0 +1,351 @@ +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out index 70a635b24f..cac0156707 100644 --- a/testing/btest/Baseline/coverage.find-bro-logs/out +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -55,4 +55,5 @@ traceroute tunnel unified2 weird +weird_stats x509 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log new file mode 100644 index 0000000000..001da38e49 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log @@ -0,0 +1,13 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-26-23-11-27 +#fields ts name num_seen +#types time string count +1532646687.827249 weird3 1 +1532646687.827249 weird2 1000 +1532646687.827249 weird1 2000 +1532646692.877464 weird1 2 +#close 2018-07-26-23-11-34 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log new file mode 100644 index 0000000000..e44145ed86 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-27-00-20-35 +#fields ts name num_seen +#types time string count +1532650834.978616 my_weird 1000 +1532650840.011592 my_weird 2000 +1532650845.043367 my_weird 10 +#close 2018-07-27-00-20-47 diff --git a/testing/btest/core/reporter-weird-sampling.bro b/testing/btest/core/reporter-weird-sampling.bro new file mode 100644 index 0000000000..d9d99681c4 --- /dev/null +++ b/testing/btest/core/reporter-weird-sampling.bro @@ -0,0 +1,55 @@ +# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output +# @TEST-EXEC: btest-diff output + +redef Weird::sampling_duration = 5sec; +redef Weird::sampling_threshold = 10; +redef Weird::sampling_rate = 10; +redef Weird::sampling_whitelist = set("whitelisted_net_weird", + "whitelisted_flow_weird", + "whitelisted_conn_weird"); + +event conn_weird(name: string, c: connection, addl: string) + { + print "conn_weird", name; + } + +event flow_weird(name: string, src: addr, dst: addr) + { + print "flow_weird", name; + } + +event net_weird(name: string) + { + print "net_weird", name; + } + +event gen_weirds(c: connection) + { + local num = 30; + + while ( num != 0 ) + { + Reporter::net_weird("my_net_weird"); + Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("my_conn_weird", c); + + Reporter::net_weird("whitelisted_net_weird"); + Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("whitelisted_conn_weird", c); + --num; + } + } + +global did_one_connection = F; + +event new_connection(c: connection) + { + if ( did_one_connection ) + return; + + did_one_connection = T; + event gen_weirds(c); # should permit 10 + 2 of each "my" weird + schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird + schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird + # Total of 27 "my" weirds of each type and 90 of each "whitelisted" type + } diff --git a/testing/btest/scripts/policy/misc/weird-stats-cluster.bro b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro new file mode 100644 index 0000000000..492b682c4d --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro @@ -0,0 +1,93 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 + +# @TEST-EXEC: btest-diff manager-1/weird_stats.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"], +}; +@TEST-END-FILE + +@load misc/weird-stats + +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + +redef Log::enable_local_logging = T; +redef Log::default_rotation_interval = 0secs; +redef WeirdStats::weird_stat_interval = 5secs; + +event terminate_me() + { + terminate(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event ready_again() + { + Reporter::net_weird("weird1"); + + if ( Cluster::node == "worker-2" ) + { + schedule 5secs { terminate_me() }; + } + } + +event ready_for_data() + { + local n = 0; + + if ( Cluster::node == "worker-1" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + ++n; + } + + Reporter::net_weird("weird3"); + } + else if ( Cluster::node == "worker-2" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + Reporter::net_weird("weird2"); + ++n; + } + } + + schedule 5secs { ready_again() }; + } + + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +event bro_init() + { + Broker::auto_publish(Cluster::worker_topic, ready_for_data); + } + +global peer_count = 0; + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peer_count; + + if ( peer_count == 2 ) + event ready_for_data(); + } + +@endif diff --git a/testing/btest/scripts/policy/misc/weird-stats.bro b/testing/btest/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..b26fce8e47 --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,32 @@ +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff bro/weird_stats.log + +@load misc/weird-stats.bro + +redef exit_only_after_terminate = T; +redef WeirdStats::weird_stat_interval = 5sec; + +event die() + { + terminate(); + } + +event gen_weirds(n: count, done: bool &default = F) + { + while ( n != 0 ) + { + Reporter::net_weird("my_weird"); + --n; + } + + if ( done ) + schedule 5sec { die() }; + } + +event bro_init() + { + event gen_weirds(1000); + schedule 7.5sec { gen_weirds(2000) } ; + schedule 12.5sec { gen_weirds(10, T) } ; + } From 0e2dbfe7c31912187bf57db7a98c8d7affc7ecf9 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 30 Jul 2018 14:12:25 -0500 Subject: [PATCH 04/24] Fix typos and formatting issues in config framework docs --- scripts/base/frameworks/config/input.bro | 4 ++-- scripts/base/frameworks/config/main.bro | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/base/frameworks/config/input.bro b/scripts/base/frameworks/config/input.bro index aaef2e59b1..7c1f37567b 100644 --- a/scripts/base/frameworks/config/input.bro +++ b/scripts/base/frameworks/config/input.bro @@ -60,8 +60,8 @@ event InputConfig::new_value(name: string, source: string, id: string, value: an function read_config(filename: string) { - # Only read the configuration on the manager. The other nodes are being fed from - # the manager. + # Only read the configuration on the manager. The other nodes are being fed + # from the manager. if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER ) return; diff --git a/scripts/base/frameworks/config/main.bro b/scripts/base/frameworks/config/main.bro index b83da42785..b11e67659a 100644 --- a/scripts/base/frameworks/config/main.bro +++ b/scripts/base/frameworks/config/main.bro @@ -28,15 +28,15 @@ export { ## record as it is sent on to the logging framework. global log_config: event(rec: Info); - ## Broker topic for announcing new configuration value. Sending new_value, - ## peers can send configuration changes that will be distributed accross + ## Broker topic for announcing new configuration values. Sending new_value, + ## peers can send configuration changes that will be distributed across ## the entire cluster. const change_topic = "bro/config/change"; ## This function is the config framework layer around the lower-level ## :bro:see:`Option::set` call. Config::set_value will set the configuration ## value for all nodes in the cluster, no matter where it was called. Note - ## that `bro:see:`Option::set` does not distribute configuration changes + ## that :bro:see:`Option::set` does not distribute configuration changes ## to other nodes. ## ## ID: The ID of the option to update. @@ -45,7 +45,7 @@ export { ## ## location: Optional parameter detailing where this change originated from. ## - ## Returns: true on success, false when an error ocured. + ## Returns: true on success, false when an error occurs. global set_value: function(ID: string, val: any, location: string &default = "" &optional): bool; } @@ -73,7 +73,7 @@ event Config::cluster_set_option(ID: string, val: any, location: string) function set_value(ID: string, val: any, location: string &default = "" &optional): bool { local cache_val: any; - # first cache value in case setting it succeeds and we have to store it. + # First cache value in case setting it succeeds and we have to store it. if ( Cluster::local_node_type() == Cluster::MANAGER ) cache_val = copy(val); # First try setting it locally - abort if not possible. @@ -151,8 +151,8 @@ event bro_init() &priority=10 # Limit logging to the manager - everyone else just feeds off it. @if ( !Cluster::is_enabled() || Cluster::local_node_type() == Cluster::MANAGER ) - # Iterate over all existing options and add ourselves as change handlers with - # a low priority so that we can log the changes. + # Iterate over all existing options and add ourselves as change handlers + # with a low priority so that we can log the changes. local gids = global_ids(); for ( i in gids ) { From 407d6461f56a31112ef03e700e8c2c02ee15ac45 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 30 Jul 2018 14:27:47 -0500 Subject: [PATCH 05/24] Update config framework doc for clusterization changes --- doc/frameworks/configuration.rst | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/doc/frameworks/configuration.rst b/doc/frameworks/configuration.rst index efb5182301..e4be6be1ec 100644 --- a/doc/frameworks/configuration.rst +++ b/doc/frameworks/configuration.rst @@ -11,7 +11,7 @@ Bro includes a "configuration framework" that allows updating script options dynamically at runtime. This functionality consists of several components: an "option" declaration, the ability to specify input files to enable changing the value of options at -runtime, a couple of built-in functions, and a log file "config.log" +runtime, a couple of functions, and a log file "config.log" which contains information about every change to option values. @@ -55,7 +55,8 @@ The "option" keyword allows variables to be declared as configuration options. The rules regarding options can be thought of as being in between global variables and constants. Like global variables, options cannot be declared inside a function, hook, or event handler. Like constants, options must be -initialized when declared. The value of an option can change at runtime, +initialized when declared (the type can often be inferred from the initializer +but may need to be specified). The value of an option can change at runtime, but options cannot be assigned a new value using normal assignments. @@ -71,16 +72,20 @@ The format for these files looks like this: [option name][tab/spaces][new value] Configuration files can be specified by adding them to Config::config_files. +Note that in a cluster configuration, only the manager node attempts to read +the specified configuration files. + For example, simply add something like this to local.bro: .. code:: bro redef Config::config_files += { "/path/to/config.dat" }; -The specified configuration file will then be monitored continuously for changes, -so that writing ``TestModule::enable_feature T`` into that file will -automatically update the option's value accordingly. Here is an example -configuration file:: +The specified configuration file will then be monitored continuously for +changes, so that writing ``TestModule::enable_feature T`` into that file will +automatically update the option's value accordingly (in a cluster +configuration, the change will be sent from the manager to all other nodes in +the cluster). Here is an example configuration file:: TestModule::my_networks 10.0.12.0/24,192.168.17.0/24 TestModule::enable_feature T @@ -94,10 +99,12 @@ for configuration files: the files need no header lines and either tabs or spaces are accepted as separators. If you inspect the configuration framework scripts, you will notice that the -scripts simply catch events from the input framework and then a built-in -function :bro:see:`Option::set` is called to set an option to the new value. -If you want to change an option yourself during runtime, you can -call Option::set directly from a script. +scripts simply catch events from the input framework and then a +function :bro:see:`Config::set_value` is called to set an option to the new +value. If you want to change an option yourself during runtime, you can +call Config::set_value directly from a script (in a cluster configuration, +this only needs to happen on the manager, as the change will be automatically +sent to all other nodes in the cluster). The log file "config.log" contains information about each configuration change that occurs during runtime. From 6044983666727415bd874340975bb524682318ce Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 31 Jul 2018 13:31:59 -0500 Subject: [PATCH 06/24] Improve handling of empty lines in several text protocol analyzers --- src/analyzer/protocol/finger/Finger.cc | 3 +++ src/analyzer/protocol/ftp/FTP.cc | 4 ++++ src/analyzer/protocol/ident/Ident.cc | 3 +++ src/analyzer/protocol/pop3/POP3.cc | 3 +++ src/analyzer/protocol/smtp/SMTP.cc | 4 +++- 5 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/analyzer/protocol/finger/Finger.cc b/src/analyzer/protocol/finger/Finger.cc index e1be27e795..0aeb544a8a 100644 --- a/src/analyzer/protocol/finger/Finger.cc +++ b/src/analyzer/protocol/finger/Finger.cc @@ -41,6 +41,9 @@ void Finger_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig const char* line = (const char*) data; const char* end_of_line = line + length; + if ( length == 0 ) + return; + if ( is_orig ) { diff --git a/src/analyzer/protocol/ftp/FTP.cc b/src/analyzer/protocol/ftp/FTP.cc index 70d1be5777..b062f5aa7e 100644 --- a/src/analyzer/protocol/ftp/FTP.cc +++ b/src/analyzer/protocol/ftp/FTP.cc @@ -69,6 +69,10 @@ void FTP_Analyzer::DeliverStream(int length, const u_char* data, bool orig) const char* line = (const char*) data; const char* end_of_line = line + length; + if ( length == 0 ) + // Could emit "ftp empty request/reply" weird, but maybe not worth it. + return; + val_list* vl = new val_list; vl->append(BuildConnVal()); diff --git a/src/analyzer/protocol/ident/Ident.cc b/src/analyzer/protocol/ident/Ident.cc index 27eafb5426..423028968c 100644 --- a/src/analyzer/protocol/ident/Ident.cc +++ b/src/analyzer/protocol/ident/Ident.cc @@ -56,6 +56,9 @@ void Ident_Analyzer::DeliverStream(int length, const u_char* data, bool is_orig) if ( TCP() ) s = is_orig ? TCP()->Orig() : TCP()->Resp(); + if ( length == 0 ) + return; + if ( is_orig ) { if ( ! ident_request ) diff --git a/src/analyzer/protocol/pop3/POP3.cc b/src/analyzer/protocol/pop3/POP3.cc index b7d6aa0dcb..4f5e5819a6 100644 --- a/src/analyzer/protocol/pop3/POP3.cc +++ b/src/analyzer/protocol/pop3/POP3.cc @@ -132,6 +132,9 @@ static string trim_whitespace(const char* in) void POP3_Analyzer::ProcessRequest(int length, const char* line) { + if ( length == 0 ) + return; + if ( waitingForAuthentication ) { ++authLines; diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 8296f83cb3..3b8383eaa0 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -269,7 +269,9 @@ void SMTP_Analyzer::ProcessLine(int length, const char* line, bool orig) if ( smtp_request ) { int data_len = end_of_line - line; - RequestEvent(cmd_len, cmd, data_len, line); + + if ( cmd_len > 0 || data_len > 0 ) + RequestEvent(cmd_len, cmd, data_len, line); } if ( cmd_code != SMTP_CMD_END_OF_DATA ) From 8928189878a7e906c0db1d984aaeb383fe4f9d6d Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 31 Jul 2018 14:53:02 -0500 Subject: [PATCH 07/24] Update NEWS for config framework clusterization changes --- NEWS | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/NEWS b/NEWS index 3cb0fd79b3..c8ec3d281f 100644 --- a/NEWS +++ b/NEWS @@ -114,7 +114,11 @@ New Functionality - Option variables: The new "option" keyword allows variables to be declared as runtime options. Such variables cannot be changed using normal assignments. Instead, they can be changed using the - new function Option::set. + new function Config::set_value. This function will automatically + apply the change to all nodes in a cluster. Note that options can also + be changed using the new function Option::set, but this function will + not send the change to any other nodes, so Config::set_value should + typically be used instead of Option::set. It is possible to "subscribe" to an option through Option::set_change_handler, which will trigger a handler callback From 41a5ffa132179ce9c2025a89a8be7b0a19543b6d Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 1 Aug 2018 16:01:11 -0500 Subject: [PATCH 08/24] Improve install/setup instructions for libmaxminddb Improved install instructions for libmaxminddb and location database. Improved the explanation of how Bro finds the location database files, fixed some reST formatting, and fixed a broken link. --- doc/frameworks/geoip.rst | 56 ++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/doc/frameworks/geoip.rst b/doc/frameworks/geoip.rst index d826aabff6..06829bfcd5 100644 --- a/doc/frameworks/geoip.rst +++ b/doc/frameworks/geoip.rst @@ -10,8 +10,8 @@ GeoLocation During the process of creating policy scripts the need may arise to find the geographic location for an IP address. Bro had support for the `GeoIP library `__ at the - policy script level from release 1.3 to 2.5.X to account for this - need. Starting with release 2.6 GeoIP support requires `libmaxminddb + policy script level from release 1.3 to 2.5.x to account for this + need. Starting with release 2.6, GeoIP support requires `libmaxminddb `__. To use this functionality, you need to first install the libmaxminddb software, and then install the GeoLite2 city database before building @@ -19,17 +19,11 @@ GeoLocation .. contents:: -Install libGeoIP ----------------- +Install libmaxminddb +-------------------- Before building Bro, you need to install libmaxminddb. -* FreeBSD: - - .. console:: - - sudo pkg install libmaxminddb - * RPM/RedHat-based Linux: .. console:: @@ -42,12 +36,17 @@ Before building Bro, you need to install libmaxminddb. sudo apt-get install libmaxminddb-dev +* FreeBSD: + + .. console:: + + sudo pkg install libmaxminddb + * Mac OS X: You need to install from your preferred package management system - (e.g. MacPorts, Fink, or Homebrew). The name of the package that you need - may be libmaxminddb, maxminddb, or libmaxminddb-dev, depending on which - package management system you are using. + (e.g. Homebrew, MacPorts, or Fink). For Homebrew, the name of the package + that you need is libmaxminddb. GeoLite2-City Database Installation @@ -64,8 +63,8 @@ the GeoLite2 city binary database: wget http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz tar zxf GeoLite2-City.tar.gz -Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be renamed -and put in the GeoIP database directory. This directory should already exist +Next, the file "GeoLite2-City_YYYYMMDD/GeoLite2-City.mmdb" needs to be moved +to the GeoIP database directory. This directory might already exist and will vary depending on which platform and package you are using. For FreeBSD, use ``/usr/local/share/GeoIP``. For Linux, use ``/usr/share/GeoIP`` or ``/var/lib/GeoIP`` (choose whichever one already exists). @@ -88,22 +87,23 @@ functionality works by running a command like this: If you see an error message similar to "Failed to open GeoIP location database", then you may need to either rename or move your GeoIP -location database file. Bro looks for location database files in the -following order by default: +location database file. If the :bro:see:`mmdb_dir` value is set to a +directory pathname (it is not set by default), then Bro looks for location +database files in that directory. If none are found or if mmdb_dir is not set, +then Bro looks for location database files in the following order: - /usr/share/GeoIP/GeoLite2-City.mmdb - /var/lib/GeoIP/GeoLite2-City.mmdb - /usr/local/share/GeoIP/GeoLite2-City.mmdb - /usr/local/var/GeoIP/GeoLite2-City.mmdb - /usr/share/GeoIP/GeoLite2-Country.mmdb - /var/lib/GeoIP/GeoLite2-Country.mmdb - /usr/local/share/GeoIP/GeoLite2-Country.mmdb - /usr/local/var/GeoIP/GeoLite2-Country.mmdb +* /usr/share/GeoIP/GeoLite2-City.mmdb +* /var/lib/GeoIP/GeoLite2-City.mmdb +* /usr/local/share/GeoIP/GeoLite2-City.mmdb +* /usr/local/var/GeoIP/GeoLite2-City.mmdb +* /usr/share/GeoIP/GeoLite2-Country.mmdb +* /var/lib/GeoIP/GeoLite2-Country.mmdb +* /usr/local/share/GeoIP/GeoLite2-Country.mmdb +* /usr/local/var/GeoIP/GeoLite2-Country.mmdb If you see an error message similar to "Bro was not configured for GeoIP -support", then you either need to rebuild Bro and make sure it is linked -against libmaxminddb or else set the :bro:see:`mmdb_dir`` value -correctly. Normally, if libmaxminddb is installed correctly then it +support", then you need to rebuild Bro and make sure it is linked +against libmaxminddb. Normally, if libmaxminddb is installed correctly then it should automatically be found when building Bro. If this doesn't happen, then you may need to specify the path to the libmaxminddb installation (e.g. ``./configure --with-geoip=``). From 1f7e112879b3dd574c6840cc887f1c44833d6ea8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 13 Aug 2018 16:30:44 -0500 Subject: [PATCH 09/24] Fix SMTP command string comparisons --- src/analyzer/protocol/smtp/SMTP.cc | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 3b8383eaa0..546b39e6bd 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -807,12 +807,22 @@ void SMTP_Analyzer::UpdateState(const int cmd_code, const int reply_code, bool o #endif } +static bool istrequal(const char* s, const char* cmd, int s_len) + { + auto cmd_len = strlen(cmd); + + if ( cmd_len != s_len ) + return false; + + return strncasecmp(s, cmd, s_len) == 0; + } + void SMTP_Analyzer::ProcessExtension(int ext_len, const char* ext) { if ( ! ext ) return; - if ( ! strncasecmp(ext, "PIPELINING", ext_len) ) + if ( istrequal(ext, "PIPELINING", ext_len) ) pipelining = 1; } @@ -822,11 +832,11 @@ int SMTP_Analyzer::ParseCmd(int cmd_len, const char* cmd) return -1; // special case because we cannot define our usual macros with "-" - if ( strncmp(cmd, "X-ANONYMOUSTLS", cmd_len) == 0 ) + if ( istrequal(cmd, "X-ANONYMOUSTLS", cmd_len) ) return SMTP_CMD_X_ANONYMOUSTLS; for ( int code = SMTP_CMD_EHLO; code < SMTP_CMD_LAST; ++code ) - if ( ! strncasecmp(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) + if ( istrequal(cmd, smtp_cmd_word[code - SMTP_CMD_EHLO], cmd_len) ) return code; return -1; From d66a589558bd0337be7e92fcaa9ca5a873e129df Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 13 Aug 2018 16:31:11 -0500 Subject: [PATCH 10/24] Add 'smtp_excessive_pending_cmds' weird --- src/analyzer/protocol/smtp/SMTP.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 546b39e6bd..7077ee8bfb 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -381,7 +381,17 @@ void SMTP_Analyzer::NewCmd(const int cmd_code) if ( first_cmd < 0 ) first_cmd = cmd_code; else + { + auto constexpr max_pending_cmd_q_size = 1000; + + if ( pending_cmd_q.size() == max_pending_cmd_q_size ) + { + Weird("smtp_excessive_pending_cmds"); + pending_cmd_q.clear(); + } + pending_cmd_q.push_back(cmd_code); + } } else first_cmd = cmd_code; From dc0904a7f3e9f48cc45e9eba1e641e51e30d9690 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 15 Aug 2018 10:17:14 -0500 Subject: [PATCH 11/24] Convert some redef-able constants to runtime options --- .../base/frameworks/netcontrol/catch-and-release.bro | 2 +- scripts/base/frameworks/notice/main.bro | 6 +++--- scripts/base/protocols/dhcp/main.bro | 2 +- scripts/base/protocols/ftp/main.bro | 6 +++--- scripts/base/protocols/http/main.bro | 4 ++-- scripts/base/protocols/krb/main.bro | 4 ++-- scripts/base/protocols/sip/main.bro | 4 ++-- .../policy/frameworks/dpd/packet-segment-logging.bro | 2 +- scripts/policy/frameworks/files/detect-MHR.bro | 2 +- scripts/policy/frameworks/software/version-changes.bro | 2 +- scripts/policy/misc/capture-loss.bro | 2 +- scripts/policy/misc/dump-events.bro | 2 +- scripts/policy/misc/stats.bro | 2 +- scripts/policy/protocols/http/header-names.bro | 4 ++-- scripts/policy/protocols/smb/main.bro | 10 +++++----- .../policy/protocols/smtp/detect-suspicious-orig.bro | 4 ++-- scripts/policy/protocols/smtp/entities-excerpt.bro | 2 +- scripts/policy/protocols/ssh/geo-data.bro | 2 +- scripts/policy/protocols/ssl/expiring-certs.bro | 4 ++-- scripts/policy/protocols/ssl/weak-keys.bro | 8 ++++---- 20 files changed, 37 insertions(+), 37 deletions(-) diff --git a/scripts/base/frameworks/netcontrol/catch-and-release.bro b/scripts/base/frameworks/netcontrol/catch-and-release.bro index 57f135de10..0b87d0bf85 100644 --- a/scripts/base/frameworks/netcontrol/catch-and-release.bro +++ b/scripts/base/frameworks/netcontrol/catch-and-release.bro @@ -142,7 +142,7 @@ export { ## If true, catch and release warns if packets of an IP address are still seen after it ## should have been blocked. - const catch_release_warn_blocked_ip_encountered = F &redef; + option catch_release_warn_blocked_ip_encountered = F; ## Time intervals for which subsequent drops of the same IP take ## effect. diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 3d4e354394..21e3d2079c 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -201,14 +201,14 @@ export { ## Address that emails will be from. ## ## Note that this is overridden by the BroControl MailFrom option. - const mail_from = "Big Brother " &redef; + option mail_from = "Big Brother "; ## Reply-to address used in outbound email. - const reply_to = "" &redef; + option reply_to = ""; ## Text string prefixed to the subject of all emails sent out. ## ## Note that this is overridden by the BroControl MailSubjectPrefix ## option. - const mail_subject_prefix = "[Bro]" &redef; + option mail_subject_prefix = "[Bro]"; ## The maximum amount of time a plugin can delay email from being sent. const max_email_delay = 15secs &redef; diff --git a/scripts/base/protocols/dhcp/main.bro b/scripts/base/protocols/dhcp/main.bro index ae102e6085..3d3bf42118 100644 --- a/scripts/base/protocols/dhcp/main.bro +++ b/scripts/base/protocols/dhcp/main.bro @@ -77,7 +77,7 @@ export { ## The maximum amount of time that a transation ID will be watched ## for to try and tie messages together into a single DHCP ## transaction narrative. - const DHCP::max_txid_watch_time = 30secs &redef; + option DHCP::max_txid_watch_time = 30secs; ## This event is used internally to distribute data around clusters ## since DHCP doesn't follow the normal "connection" model used by diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 8a24a9d92f..b3eaf79370 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -17,13 +17,13 @@ export { redef enum Log::ID += { LOG }; ## List of commands that should have their command/response pairs logged. - const logged_commands = { + option logged_commands = { "APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT", "EPSV" - } &redef; + }; ## User IDs that can be considered "anonymous". - const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef; + option guest_ids = { "anonymous", "ftp", "ftpuser", "guest" }; ## This record is to hold a parsed FTP reply code. For example, for the ## 201 status code, the digits would be parsed as: x->2, y->0, z->1. diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 51a89a33b9..5b8b608971 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -109,7 +109,7 @@ export { ## A list of HTTP methods. Other methods will generate a weird. Note ## that the HTTP analyzer will only accept methods consisting solely ## of letters ``[A-Za-z]``. - const http_methods: set[string] = { + option http_methods: set[string] = { "GET", "POST", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE", "CONNECT", # HTTP methods for distributed authoring: @@ -117,7 +117,7 @@ export { "COPY", "MOVE", "LOCK", "UNLOCK", "POLL", "REPORT", "SUBSCRIBE", "BMOVE", "SEARCH" - } &redef; + }; ## Event that can be handled to access the HTTP record as it is sent on ## to the logging framework. diff --git a/scripts/base/protocols/krb/main.bro b/scripts/base/protocols/krb/main.bro index 02abced683..e45ce92cd5 100644 --- a/scripts/base/protocols/krb/main.bro +++ b/scripts/base/protocols/krb/main.bro @@ -48,7 +48,7 @@ export { }; ## The server response error texts which are *not* logged. - const ignored_errors: set[string] = { + option ignored_errors: set[string] = { # This will significantly increase the noisiness of the log. # However, one attack is to iterate over principals, looking # for ones that don't require preauth, and then performn @@ -58,7 +58,7 @@ export { # This is a more specific version of NEEDED_PREAUTH that's used # by Windows AD Kerberos. "Need to use PA-ENC-TIMESTAMP/PA-PK-AS-REQ", - } &redef; + }; ## Event that can be handled to access the KRB record as it is sent on ## to the logging framework. diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index f4dba22876..c2a78e32af 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -80,9 +80,9 @@ export { ## A list of SIP methods. Other methods will generate a weird. Note ## that the SIP analyzer will only accept methods consisting solely ## of letters ``[A-Za-z]``. - const sip_methods: set[string] = { + option sip_methods: set[string] = { "REGISTER", "INVITE", "ACK", "CANCEL", "BYE", "OPTIONS", "NOTIFY", "SUBSCRIBE" - } &redef; + }; ## Event that can be handled to access the SIP record as it is sent on ## to the logging framework. diff --git a/scripts/policy/frameworks/dpd/packet-segment-logging.bro b/scripts/policy/frameworks/dpd/packet-segment-logging.bro index c1d6699352..35a52c3870 100644 --- a/scripts/policy/frameworks/dpd/packet-segment-logging.bro +++ b/scripts/policy/frameworks/dpd/packet-segment-logging.bro @@ -16,7 +16,7 @@ export { }; ## Size of the packet segment to display in the DPD log. - const packet_segment_size: int = 255 &redef; + option packet_segment_size: int = 255; } diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro index 6917212356..760c5b3693 100644 --- a/scripts/policy/frameworks/files/detect-MHR.bro +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -32,7 +32,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/scripts/policy/frameworks/software/version-changes.bro b/scripts/policy/frameworks/software/version-changes.bro index 5494f14bca..215a64d6b7 100644 --- a/scripts/policy/frameworks/software/version-changes.bro +++ b/scripts/policy/frameworks/software/version-changes.bro @@ -19,7 +19,7 @@ export { ## Some software is more interesting when the version changes and this ## is a set of all software that should raise a notice when a different ## version is seen on a host. - const interesting_version_changes: set[string] = { } &redef; + option interesting_version_changes: set[string] = {}; } event Software::version_change(old: Software::Info, new: Software::Info) diff --git a/scripts/policy/misc/capture-loss.bro b/scripts/policy/misc/capture-loss.bro index 648e3d6717..8bd0a61a05 100644 --- a/scripts/policy/misc/capture-loss.bro +++ b/scripts/policy/misc/capture-loss.bro @@ -44,7 +44,7 @@ export { ## when the :bro:enum:`CaptureLoss::Too_Much_Loss` notice should be ## generated. The value is expressed as a double between 0 and 1 with 1 ## being 100%. - const too_much_loss: double = 0.1 &redef; + option too_much_loss: double = 0.1; } event CaptureLoss::take_measurement(last_ts: time, last_acks: count, last_gaps: count) diff --git a/scripts/policy/misc/dump-events.bro b/scripts/policy/misc/dump-events.bro index a0c36753cb..78772b265b 100644 --- a/scripts/policy/misc/dump-events.bro +++ b/scripts/policy/misc/dump-events.bro @@ -7,7 +7,7 @@ module DumpEvents; export { ## If true, include event arguments in output. - const include_args = T &redef; + option include_args = T; ## Only include events matching the given pattern into output. By default, the ## pattern matches all events. diff --git a/scripts/policy/misc/stats.bro b/scripts/policy/misc/stats.bro index c307ece849..0bbf5c8aac 100644 --- a/scripts/policy/misc/stats.bro +++ b/scripts/policy/misc/stats.bro @@ -8,7 +8,7 @@ export { redef enum Log::ID += { LOG }; ## How often stats are reported. - const report_interval = 5min &redef; + option report_interval = 5min; type Info: record { ## Timestamp for the measurement. diff --git a/scripts/policy/protocols/http/header-names.bro b/scripts/policy/protocols/http/header-names.bro index 1b256226dd..9f4e83638c 100644 --- a/scripts/policy/protocols/http/header-names.bro +++ b/scripts/policy/protocols/http/header-names.bro @@ -18,10 +18,10 @@ export { }; ## A boolean value to determine if client header names are to be logged. - const log_client_header_names = T &redef; + option log_client_header_names = T; ## A boolean value to determine if server header names are to be logged. - const log_server_header_names = F &redef; + option log_server_header_names = F; } event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 diff --git a/scripts/policy/protocols/smb/main.bro b/scripts/policy/protocols/smb/main.bro index 51aab775c0..5b87882882 100644 --- a/scripts/policy/protocols/smb/main.bro +++ b/scripts/policy/protocols/smb/main.bro @@ -32,19 +32,19 @@ export { }; ## The file actions which are logged. - const logged_file_actions: set[Action] = { + option logged_file_actions: set[Action] = { FILE_OPEN, FILE_RENAME, FILE_DELETE, PRINT_OPEN, PRINT_CLOSE, - } &redef; + }; ## The server response statuses which are *not* logged. - const ignored_command_statuses: set[string] = { + option ignored_command_statuses: set[string] = { "MORE_PROCESSING_REQUIRED", - } &redef; + }; ## This record is for the smb_files.log type FileInfo: record { @@ -159,7 +159,7 @@ export { ## Optionally write out the SMB commands log. This is ## primarily useful for debugging so is disabled by default. - const write_cmd_log = F &redef; + option write_cmd_log = F; ## Everything below here is used internally in the SMB scripts. diff --git a/scripts/policy/protocols/smtp/detect-suspicious-orig.bro b/scripts/policy/protocols/smtp/detect-suspicious-orig.bro index 6fe37e02a8..a39ec185a6 100644 --- a/scripts/policy/protocols/smtp/detect-suspicious-orig.bro +++ b/scripts/policy/protocols/smtp/detect-suspicious-orig.bro @@ -11,8 +11,8 @@ export { ## Places where it's suspicious for mail to originate from represented ## as all-capital, two character country codes (e.g., US). It requires ## libGeoIP support built in. - const suspicious_origination_countries: set[string] = {} &redef; - const suspicious_origination_networks: set[subnet] = {} &redef; + option suspicious_origination_countries: set[string] = {}; + option suspicious_origination_networks: set[subnet] = {}; } diff --git a/scripts/policy/protocols/smtp/entities-excerpt.bro b/scripts/policy/protocols/smtp/entities-excerpt.bro index 0e596ebdd1..a3c35507ca 100644 --- a/scripts/policy/protocols/smtp/entities-excerpt.bro +++ b/scripts/policy/protocols/smtp/entities-excerpt.bro @@ -14,7 +14,7 @@ export { ## This is the default value for how much of the entity body should be ## included for all MIME entities. The lesser of this value and ## :bro:see:`default_file_bof_buffer_size` will be used. - const default_entity_excerpt_len = 0 &redef; + option default_entity_excerpt_len = 0; } event file_new(f: fa_file) &priority=5 diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro index 7235f24199..c8f73b6ac7 100644 --- a/scripts/policy/protocols/ssh/geo-data.bro +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -21,7 +21,7 @@ export { ## The set of countries for which you'd like to generate notices upon ## successful login. - const watched_countries: set[string] = {"RO"} &redef; + option watched_countries: set[string] = {"RO"}; } function get_location(c: connection): geo_location diff --git a/scripts/policy/protocols/ssl/expiring-certs.bro b/scripts/policy/protocols/ssl/expiring-certs.bro index 04ebeb3c5a..1e806942d7 100644 --- a/scripts/policy/protocols/ssl/expiring-certs.bro +++ b/scripts/policy/protocols/ssl/expiring-certs.bro @@ -27,11 +27,11 @@ export { ## notices will be suppressed by the notice framework for 1 day after ## a particular certificate has had a notice generated. ## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS - const notify_certs_expiration = LOCAL_HOSTS &redef; + option notify_certs_expiration = LOCAL_HOSTS; ## The time before a certificate is going to expire that you would like ## to start receiving :bro:enum:`SSL::Certificate_Expires_Soon` notices. - const notify_when_cert_expiring_in = 30days &redef; + option notify_when_cert_expiring_in = 30days; } event ssl_established(c: connection) &priority=3 diff --git a/scripts/policy/protocols/ssl/weak-keys.bro b/scripts/policy/protocols/ssl/weak-keys.bro index a8859b2e59..afebb035a3 100644 --- a/scripts/policy/protocols/ssl/weak-keys.bro +++ b/scripts/policy/protocols/ssl/weak-keys.bro @@ -21,24 +21,24 @@ export { ## keys/ciphers/protocol_versions. By default, these notices will be suppressed ## by the notice framework for 1 day after a particular host has had a notice ## generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS - const notify_weak_keys = LOCAL_HOSTS &redef; + option notify_weak_keys = LOCAL_HOSTS; ## The minimal key length in bits that is considered to be safe. Any shorter ## (non-EC) key lengths will trigger a notice. - const notify_minimal_key_length = 2048 &redef; + option notify_minimal_key_length = 2048; ## Warn if the DH key length is smaller than the certificate key length. This is ## potentially unsafe because it gives a wrong impression of safety due to the ## certificate key length. However, it is very common and cannot be avoided in some ## settings (e.g. with old jave clients). - const notify_dh_length_shorter_cert_length = T &redef; + option notify_dh_length_shorter_cert_length = T; ## Warn if a server negotiates a SSL session with a protocol version smaller than ## the specified version. By default, the minimal version is TLSv10 because SSLv2 ## and v3 have serious security issued. ## See https://tools.ietf.org/html/draft-thomson-sslv3-diediedie-00 ## To disable, set to SSLv20 - const tls_minimum_version = TLSv10 &redef; + option tls_minimum_version = TLSv10; ## Warn if a server negotiates an unsafe cipher suite. By default, we only warn when ## encountering old export cipher suites, or RC4 (see RFC7465). From d7be90c3ca2d32498c6ec24fb796a39490f31584 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 15 Aug 2018 10:30:09 -0500 Subject: [PATCH 12/24] Remove unused redef-able constants --- scripts/base/protocols/dns/main.bro | 5 ----- scripts/base/utils/exec.bro | 3 --- 2 files changed, 8 deletions(-) diff --git a/scripts/base/protocols/dns/main.bro b/scripts/base/protocols/dns/main.bro index 127a06b5a0..c98f0b2201 100644 --- a/scripts/base/protocols/dns/main.bro +++ b/scripts/base/protocols/dns/main.bro @@ -113,11 +113,6 @@ export { ## DNS message query/transaction ID. type PendingMessages: table[count] of Queue::Queue; - ## The amount of time that DNS queries or replies for a given - ## query/transaction ID are allowed to be queued while waiting for - ## a matching reply or query. - const pending_msg_expiry_interval = 2min &redef; - ## Give up trying to match pending DNS queries or replies for a given ## query/transaction ID once this number of unmatched queries or replies ## is reached (this shouldn't happen unless either the DNS server/resolver diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro index 61488a1249..5079396bbc 100644 --- a/scripts/base/utils/exec.bro +++ b/scripts/base/utils/exec.bro @@ -42,9 +42,6 @@ export { ## Returns: A record representing the full results from the ## external program execution. global run: function(cmd: Command): Result; - - ## The system directory for temporary files. - const tmp_dir = "/tmp" &redef; } # Indexed by command uid. From ccfca956e9c2347a2c0a6e6be77be43320213b00 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 16 Aug 2018 11:31:26 -0500 Subject: [PATCH 13/24] Rearrange some lines on the "Log Files" documentation page Moved config.log out of the "Network Protocols" section. Moved broker.log so that it appears in alphabetical order. --- doc/script-reference/log-files.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 8f7d87d89a..712d86c26c 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -14,8 +14,6 @@ Network Protocols +============================+=======================================+=================================+ | conn.log | TCP/UDP/ICMP connections | :bro:type:`Conn::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| config.log | Configuration option changes | :bro:type:`Config::Info` | -+----------------------------+---------------------------------------+---------------------------------+ | dce_rpc.log | Distributed Computing Environment/RPC | :bro:type:`DCE_RPC::Info` | +----------------------------+---------------------------------------+---------------------------------+ | dhcp.log | DHCP leases | :bro:type:`DHCP::Info` | @@ -161,12 +159,14 @@ Bro Diagnostics +----------------------------+---------------------------------------+---------------------------------+ | Log File | Description | Field Descriptions | +============================+=======================================+=================================+ +| broker.log | Peering status events between Bro or | :bro:type:`Broker::Info` | +| | Broker-enabled processes | | ++----------------------------+---------------------------------------+---------------------------------+ | capture_loss.log | Packet loss rate | :bro:type:`CaptureLoss::Info` | +----------------------------+---------------------------------------+---------------------------------+ | cluster.log | Bro cluster messages | :bro:type:`Cluster::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| broker.log | Peering status events between Bro or | :bro:type:`Broker::Info` | -| | Broker-enabled processes | | +| config.log | Configuration option changes | :bro:type:`Config::Info` | +----------------------------+---------------------------------------+---------------------------------+ | loaded_scripts.log | Shows all scripts loaded by Bro | :bro:type:`LoadedScripts::Info` | +----------------------------+---------------------------------------+---------------------------------+ From 6ef98cdb77431273179fd645bc92632d6cc299ec Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 16 Aug 2018 13:32:46 -0500 Subject: [PATCH 14/24] Improvements to the config framework documentation Add documentation of using redef to redefine initial value of options. Mention caveats for changing the value of specific data types. Show an example of how to use the Config::set_value() function. Other small improvements to the examples and text. --- doc/frameworks/configuration.rst | 85 ++++++++++++++++++++++++-------- 1 file changed, 65 insertions(+), 20 deletions(-) diff --git a/doc/frameworks/configuration.rst b/doc/frameworks/configuration.rst index e4be6be1ec..23a384a280 100644 --- a/doc/frameworks/configuration.rst +++ b/doc/frameworks/configuration.rst @@ -50,6 +50,8 @@ The "option" keyword allows variables to be declared as configuration options. option my_networks: set[subnet] = {}; option enable_feature = F; option hostname = "testsystem"; + option timeout = 1min; + option my_ports: vector of port = {}; } The rules regarding options can be thought of as being in between global @@ -59,6 +61,17 @@ initialized when declared (the type can often be inferred from the initializer but may need to be specified). The value of an option can change at runtime, but options cannot be assigned a new value using normal assignments. +The initial value of an option can be redefined with a :bro:keyword:`redef` +declaration just like for global variables and constants. The only difference +being that there is no need to specify the :bro:attr:`&redef` attribute in +the declaration of an option. For example, given the above option +declarations, here are some possible redefs: + +.. code:: bro + + redef TestModule::enable_feature = T; + redef TestModule::my_networks += { 10.1.0.0/16, 10.2.0.0/16 }; + Changing options ---------------- @@ -71,9 +84,9 @@ The format for these files looks like this: [option name][tab/spaces][new value] -Configuration files can be specified by adding them to Config::config_files. -Note that in a cluster configuration, only the manager node attempts to read -the specified configuration files. +Configuration files can be specified by adding them +to :bro:id:`Config::config_files`. Note that in a cluster configuration, +only the manager node attempts to read the specified configuration files. For example, simply add something like this to local.bro: @@ -90,6 +103,13 @@ the cluster). Here is an example configuration file:: TestModule::my_networks 10.0.12.0/24,192.168.17.0/24 TestModule::enable_feature T TestModule::hostname host-1 + TestModule::timeout 50.5 + TestModule::my_ports 80/tcp,53/udp + +Note that as seen in the above example, for options of +type :bro:type:`interval`, the numeric value in the config file +is interpreted as seconds and there cannot be any time units +(such as sec, min, etc.). Internally, the configuration framework uses the Bro input framework with a type of input reader specifically for reading config files. Users @@ -106,8 +126,27 @@ call Config::set_value directly from a script (in a cluster configuration, this only needs to happen on the manager, as the change will be automatically sent to all other nodes in the cluster). -The log file "config.log" contains information about each configuration -change that occurs during runtime. +Note that some data types (such as pattern, table, and record) are not +supported by the config input reader. In that case you would need to use +the Config::set_value function to change the value of such an option as +shown in the following example. + +.. code:: bro + + module TestModule; + + export { + option host_port: table[addr] of port = {}; + } + + event bro_init() { + local t: table[addr] of port = { [10.0.0.2] = 123/tcp }; + Config::set_value("TestModule::host_port", t); + } + +Regardless of whether an option change is triggered by a config file or by +the Config::set_value function, the change is always logged to the +log file "config.log". Change handlers @@ -121,7 +160,11 @@ accordingly): .. code:: bro - option testaddr = 127.0.0.1; + module TestModule; + + export { + option testaddr = 127.0.0.1; + } # Note: the data type of 2nd parameter and return type must match function change_addr(ID: string, new_value: addr): addr @@ -132,24 +175,26 @@ accordingly): event bro_init() { - Option::set_change_handler("testaddr", change_addr); + Option::set_change_handler("TestModule::testaddr", change_addr); } -Each time the specified option value is changed, the change handler -function will be called before the change is performed. The value returned -by the change handler is the value finally assigned to the option. This -allows, for example, checking of values to reject invalid input (the original -value can be returned to reject the change). - -A change handler can optionally have a third argument, which is the location -string (this is normally the pathname of the configuration file that triggered +Immediately before the specified option value is changed, the change handler +function will be called. The value returned by the change handler is the +value finally assigned to the option. This allows, for example, checking of +values to reject invalid input (the original value can be returned to reject the change). -It is also possible to chain together multiple change handlers. In this -case, the value returned by the first change handler is the "new value" seen -by the next change handler, and so on. The built-in function -:bro:see:`Option::set_change_handler` takes an optional third argument -that can specify a priority for the handlers. +It is possible define multiple change handlers for a single option. In +this case, the change handlers are chained together: the value returned by the +first change handler is the "new value" seen by the next change handler, and +so on. The built-in function :bro:see:`Option::set_change_handler` takes an +optional third argument that can specify a priority for the handlers. + +A change handler function can optionally have a third argument of type +string. When a config file triggers a change, then the third argument is +the pathname of the config file. When the Config::set_value function triggers +a change, then the third argument of the change handler is the value passed +to the optional third argument of the Config::set_value function. Note that change handlers are also used internally by the configuration framework. If you look at the script level source code of From 4613347a95848fe6380f5a5648e2e46bbb3e56aa Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 16 Aug 2018 14:22:08 -0500 Subject: [PATCH 15/24] Update documentation of "option" and "redef" declarations Add documentation of using "redef" on a runtime option. Also mention how to change an option's value at runtime. --- doc/script-reference/statements.rst | 36 ++++++++++++++++++----------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst index 9e061d4df7..6c7e54395a 100644 --- a/doc/script-reference/statements.rst +++ b/doc/script-reference/statements.rst @@ -190,7 +190,11 @@ all loaded Bro scripts. option hostname = "host-1"; option peers: set[addr] = {}; - The value of an option cannot be changed by an assignment statement. + The initial value can be redefined with a :bro:keyword:`redef`. + + The value of an option cannot be changed by an assignment statement, but + it can be changed by either the :bro:id:`Config::set_value` function or + by changing a config file specified in :bro:id:`Config::config_files`. The scope of an option is global. @@ -215,26 +219,30 @@ all loaded Bro scripts. .. bro:keyword:: redef - There are three ways that "redef" can be used: to change the value of - a global variable (but only if it has the :bro:attr:`&redef` attribute), - to extend a record type or enum type, or to specify - a new event handler body that replaces all those that were previously - defined. + There are several ways that "redef" can be used: to redefine the initial + value of a global variable or runtime option, to extend a record type or + enum type, or to specify a new event handler body that replaces all those + that were previously defined. - If you're using "redef" to change a global variable (defined using either - :bro:keyword:`const` or :bro:keyword:`global`), then the variable that you - want to change must have the :bro:attr:`&redef` attribute. If the variable - you're changing is a table, set, or pattern, you can use ``+=`` to add - new elements, or you can use ``=`` to specify a new value (all previous - contents of the object are removed). If the variable you're changing is a - set or table, then you can use the ``-=`` operator to remove the - specified elements (nothing happens for specified elements that don't + If you're using "redef" to redefine the initial value of a global variable + (defined using either :bro:keyword:`const` or :bro:keyword:`global`), then + the variable that you want to change must have the :bro:attr:`&redef` + attribute. You can use "redef" to redefine the initial value of a + runtime option (defined using :bro:keyword:`option`) even if it doesn't + have the :bro:attr:`&redef` attribute. + + If the variable you're changing is a table, set, or pattern, you can + use ``+=`` to add new elements, or you can use ``=`` to specify a new value + (all previous contents of the object are removed). If the variable you're + changing is a set or table, then you can use the ``-=`` operator to remove + the specified elements (nothing happens for specified elements that don't exist). If the variable you are changing is not a table, set, or pattern, then you must use the ``=`` operator. Examples:: redef pi = 3.14; + redef set_of_ports += { 22/tcp, 53/udp }; If you're using "redef" to extend a record or enum, then you must use the ``+=`` assignment operator. From f40e317c0d7de1baa71536624a46c4fcb319a82f Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 16 Aug 2018 15:41:18 -0500 Subject: [PATCH 16/24] Update install instructions for python-ipaddress --- doc/install/install.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/install/install.rst b/doc/install/install.rst index cc8d81b14f..cdea7d9fac 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -60,6 +60,9 @@ To install the required dependencies, you can use: sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev + If your system uses Python 2.7, then you will also need to install the + "python-ipaddress" package. + * FreeBSD: Most required dependencies should come with a minimal FreeBSD install From c941c565a698744fb54a96aee3933e1865bdb57a Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 16 Aug 2018 15:45:58 -0500 Subject: [PATCH 17/24] Replace references to libgeoip in the documentation Replace references to the old libgeoip library with "libmaxminddb" or "GeoIP support". --- doc/install/install.rst | 6 +++--- scripts/base/frameworks/notice/actions/add-geodata.bro | 2 +- scripts/base/utils/geoip-distance.bro | 2 +- scripts/policy/protocols/smtp/detect-suspicious-orig.bro | 4 ++-- scripts/site/local.bro | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/doc/install/install.rst b/doc/install/install.rst index cdea7d9fac..dee3ba0d0c 100644 --- a/doc/install/install.rst +++ b/doc/install/install.rst @@ -117,7 +117,7 @@ Optional Dependencies Bro can make use of some optional libraries and tools if they are found at build time: - * LibGeoIP (for geolocating IP addresses) + * libmaxminddb (for geolocating IP addresses) * sendmail (enables Bro and BroControl to send mail) * curl (used by a Bro script that implements active HTTP) * gperftools (tcmalloc is used to improve memory and CPU usage) @@ -125,9 +125,9 @@ build time: * PF_RING (Linux only, see :doc:`Cluster Configuration <../configuration/index>`) * ipsumdump (for trace-summary; http://www.cs.ucla.edu/~kohler/ipsumdump) -LibGeoIP is probably the most interesting and can be installed +Geolocation is probably the most interesting and can be installed on most platforms by following the instructions for :ref:`installing -libGeoIP and the GeoIP database +the GeoIP library and database `. diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro index c762be3b86..ab9957327a 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.bro +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -19,7 +19,7 @@ export { }; redef record Info += { - ## If libGeoIP support is built in, notices can have geographic + ## If GeoIP support is built in, notices can have geographic ## information attached to them. remote_location: geo_location &log &optional; }; diff --git a/scripts/base/utils/geoip-distance.bro b/scripts/base/utils/geoip-distance.bro index 3bcf93f828..8d3149cb03 100644 --- a/scripts/base/utils/geoip-distance.bro +++ b/scripts/base/utils/geoip-distance.bro @@ -1,7 +1,7 @@ ##! Functions to calculate distance between two locations, based on GeoIP data. ## Returns the distance between two IP addresses using the haversine formula, -## based on GeoIP database locations. Requires Bro to be built with libgeoip. +## based on GeoIP database locations. Requires Bro to be built with GeoIP. ## ## a1: First IP address. ## diff --git a/scripts/policy/protocols/smtp/detect-suspicious-orig.bro b/scripts/policy/protocols/smtp/detect-suspicious-orig.bro index 6fe37e02a8..9c7d16a658 100644 --- a/scripts/policy/protocols/smtp/detect-suspicious-orig.bro +++ b/scripts/policy/protocols/smtp/detect-suspicious-orig.bro @@ -9,8 +9,8 @@ export { }; ## Places where it's suspicious for mail to originate from represented - ## as all-capital, two character country codes (e.g., US). It requires - ## libGeoIP support built in. + ## as all-capital, two character country codes (e.g., US). It requires + ## Bro to be built with GeoIP support. const suspicious_origination_countries: set[string] = {} &redef; const suspicious_origination_networks: set[subnet] = {} &redef; diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 5064d6a330..be6ddd8591 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -66,7 +66,7 @@ # certificate notary service; see http://notary.icsi.berkeley.edu . # @load protocols/ssl/notary -# If you have libGeoIP support built in, do some geographic detections and +# If you have GeoIP support built in, do some geographic detections and # logging for SSH traffic. @load protocols/ssh/geo-data # Detect hosts doing SSH bruteforce attacks. From 8fe300a47c5f01162e48c61234ca1bc5bf23f513 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 16 Aug 2018 21:57:04 -0500 Subject: [PATCH 18/24] Update the operators documentation Added documentation for some new operators and improve documentation of the "in" operator. Also corrected a few typos in the docs. --- doc/script-reference/operators.rst | 29 ++++++++++++++++++++++++++++- doc/script-reference/types.rst | 29 ++++++++++++++++++----------- 2 files changed, 46 insertions(+), 12 deletions(-) diff --git a/doc/script-reference/operators.rst b/doc/script-reference/operators.rst index 8f64617494..b8a34050aa 100644 --- a/doc/script-reference/operators.rst +++ b/doc/script-reference/operators.rst @@ -10,6 +10,11 @@ Relational operators The relational operators evaluate to type :bro:type:`bool`. +In addition to numeric operands, the relational operators also work with +operands of type :bro:type:`interval`, :bro:type:`time`, :bro:type:`string`, +:bro:type:`port`, :bro:type:`addr`, and :bro:type:`set`. + + +------------------------------+--------------+ | Name | Syntax | +==============================+==============+ @@ -104,12 +109,27 @@ only. | Bitwise complement | ~ *a* | +------------------------------+-------------+ +Set operators +------------- + ++------------------------------+-------------+ +| Name | Syntax | ++==============================+=============+ +| Set intersection | *s1* & *s2* | ++------------------------------+-------------+ +| Set union | *s1* | *s2* | ++------------------------------+-------------+ +| Set difference | *s1* - *s2* | ++------------------------------+-------------+ Assignment operators -------------------- The assignment operators evaluate to the result of the assignment. +The "+=" operator can also be used to append an element to the end of a +vector. For example, ``v += e`` is equivalent to ``v[|v|] = e``. + +------------------------------+-------------+ | Name | Syntax | +==============================+=============+ @@ -215,7 +235,14 @@ Other operators | Name | Syntax | Notes | +================================+===================+========================+ | Membership test | *a* in *b* |Evaluates to type | -| | |:bro:type:`bool`. Do not| +| | |:bro:type:`bool`. Works | +| | |with :bro:type:`string`,| +| | |:bro:type:`pattern`, | +| | |:bro:type:`subnet`, | +| | |:bro:type:`set`, | +| | |:bro:type:`table`, or | +| | |:bro:type:`vector` | +| | |operands. Do not | | | |confuse this use of "in"| | | |with that used in a | | | |:bro:keyword:`for` | diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index cfb47270ff..1c7a177ebe 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -253,10 +253,10 @@ Here is a more detailed description of each type: When specifying a pattern, you can add a final ``i`` specifier to mark it as case-insensitive. For example, ``/foo|bar/i`` will match - a "foo", "Foo", "BaR", etc. + "foo", "Foo", "BaR", etc. You can also introduce a case-insensitive sub-pattern by enclosing it - in ``(?i:````)``. So, for example, ``/foo|(?i:bar)/`` will + in ``(?i:)``. So, for example, ``/foo|(?i:bar)/`` will match "foo" and "BaR", but *not* "Foo". For both ways of specifying case-insensitivity, characters enclosed @@ -545,13 +545,14 @@ Here is a more detailed description of each type: |s| You can compute the union, intersection, or difference of two sets - using the ``|``, ``&``, and ``-`` operators. You can compare - sets for equality (they have exactly the same elements) using ``==``. - The ``<`` operator returns ``T`` if the lefthand operand is a proper - subset of the righthand operand. Similarly, ``<=`` returns ``T`` - if the lefthand operator is a subset (not necessarily proper, i.e., - it may be equal to the righthand operand). The operators ``!=``, ``>`` - and ``>=`` provide the expected complementary operations. + using the ``|``, ``&``, and ``-`` operators. + + You can compare sets for equality (they have exactly the same elements) + using ``==``. The ``<`` operator returns ``T`` if the lefthand operand + is a proper subset of the righthand operand. Similarly, ``<=`` + returns ``T`` if the lefthand operator is a subset (not necessarily proper, + i.e., it may be equal to the righthand operand). The operators ``!=``, + ``>`` and ``>=`` provide the expected complementary operations. See the :bro:keyword:`for` statement for info on how to iterate over the elements in a set. @@ -601,8 +602,9 @@ Here is a more detailed description of each type: v[3] = "four"; - The number of elements in a vector can be obtained by placing the vector - identifier between vertical pipe characters: + The size of a vector (this is one greater than the highest index value, and + is normally equal to the number of elements in the vector) can be obtained + by placing the vector identifier between vertical pipe characters: .. code:: bro @@ -622,6 +624,11 @@ Here is a more detailed description of each type: v[|v|] = e; + The "in" operator can be used to check if a value has been assigned at a + specified index value in the vector. For example, if a vector has size 4, + then the expression ``3 in v`` would yield true and ``4 in v`` would yield + false. + Vectors of integral types (``int`` or ``count``) support the pre-increment (``++``) and pre-decrement operators (``--``), which will increment or decrement each element in the vector. From ab2f745edbc851d90f681017c6cad75bb992d005 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 17 Aug 2018 11:33:19 -0500 Subject: [PATCH 19/24] Fix some typos and improve formatting in NEWS --- NEWS | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/NEWS b/NEWS index 4c1ebc6baa..341e419597 100644 --- a/NEWS +++ b/NEWS @@ -166,7 +166,7 @@ New Functionality } The specified file will now be monitored continuously for changes, so - that writing "testbool T" into /path/to/config.dat will + that writing "TestConfig::testbool T" into /path/to/config.dat will automatically update the option's value accordingly. The configuration framework creates a config.log that shows all @@ -224,7 +224,6 @@ New Functionality ssl_server_curve event, ssl_server_curve is now marked as deprecated. - Functions for retrieving files by their ID have been added: - Files::file_exists, Files::lookup_File - New functions in the logging API: Log::get_filter_names, Log::enable_stream @@ -241,10 +240,10 @@ New Functionality mount_proc_umnt, mount_proc_umnt_all, mount_proc_not_implemented, mount_reply_status. -- Added new NFS events: nfs_proc_symlink, nfs_proc_link, nfs_proc_sattr +- Added new NFS events: nfs_proc_symlink, nfs_proc_link, nfs_proc_sattr. - Added new SMB events: smb1_transaction_secondary_request, - smb1_transaction2_secondary_request, smb1_transaction_response + smb1_transaction2_secondary_request, smb1_transaction_response. - Bro can now decrypt Kerberos tickets, and retrieve the authentication from them, given a suitable keytab file. @@ -261,20 +260,18 @@ New Functionality - You can now specify that a pattern matches in a case-insensitive fashion by adding 'i' to the end of its specification. So for example - /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar". Characters - enclosed in quotes however keep their casing, so /"fOO"/i in "xFoObar" - yields F, though it yields T for "xfOObar". + /fOO/i == "Foo" yields T, as does /fOO/i in "xFoObar". You can achieve the same functionality for a subpattern enclosed in parentheses by adding "?i:" to the open parenthesis. So for example - "/foo|(?i:bar)/" will match "BaR", but not "FoO". + /foo|(?i:bar)/ will match "BaR", but not "FoO". For both ways of specifying case-insensitivity, characters enclosed in - double quotes maintain their case-sensitivity. So for example /"foo"/i - will not match "Foo", but it will match "foo". + double quotes remain case-sensitive. So for example /"foo"/i will not + match "Foo", but it will match "foo". - "make install" now installs Bro's include headers (and more) into - --prefix so that compiling plugins does no longer need access to a + --prefix so that compiling plugins no longer needs access to a source/build tree. For OS distributions, this also facilitates creating "bro-devel" packages providing all files necessary to build plugins. @@ -314,7 +311,7 @@ New Functionality of non-equality, proper superset, and superset-or-equal. - An expression of the form "v += e" will append the value of the expression - "e" to the end of the vector "v" (of course assuming type-compatbility). + "e" to the end of the vector "v" (of course assuming type-compatibility). Changed Functionality --------------------- @@ -334,9 +331,9 @@ Changed Functionality script since it's generally less relevant now with the updated log. - Removed the base/protocols/dhcp/utils.bro script and thus the - 'reverse_ip' function. + "reverse_ip" function. - - Replaced all DHCP events with the single 'dhcp_message' event. + - Replaced all DHCP events with the single "dhcp_message" event. The list of removed events includes: - dhcp_discover @@ -372,9 +369,9 @@ Changed Functionality redef SOCKS::default_capture_password = T; - The DNS base scripts no longer generate some noisy and annoying - weirds (dns_unmatched_msg, dns_unmatched_msg_quantity, dns_unmatched_reply) + weirds (dns_unmatched_msg, dns_unmatched_msg_quantity, dns_unmatched_reply). -- The 'tunnel_parents' field of conn.log is now marked &optional, so, for +- The "tunnel_parents" field of conn.log is now marked &optional, so, for the default configuration of logs, this field will show "-" instead of "(empty)" for connections that lack any tunneling. @@ -390,7 +387,7 @@ Changed Functionality - event ssl_server_signature now has an additional argument "signature_and_hashalgorithm". -- The "dnp3_header_block" event no longer has the "start" parameter +- The "dnp3_header_block" event no longer has the "start" parameter. - The string_to_pattern() built-in (and the now-deprecated merge_pattern() built-in) is no longer restricted to only be called at initialization time. @@ -413,11 +410,11 @@ Removed Functionality https://github.com/bro/packages for a list of Bro packages currently available. -- BroControl: The option 'IPv6Comm' and 'ZoneID' options are no longer - available (though Broker should be able to handle IPv6 automatically). - - The "ocsp_request" event no longer has "requestorName" parameter. +- BroControl: The "IPv6Comm" and "ZoneID" options are no longer + available (though Broker should be able to handle IPv6 automatically). + Deprecated Functionality ------------------------ @@ -432,10 +429,6 @@ Deprecated Functionality as BiFs like send_id(). Use Broker data stores and the new configuration framework instead. -- BroControl: The 'update' command is deprecated and scheduled for - removal with the next Bro release. Bro's new configuration framework - is taking its place. - - Mixing of scalars and vectors, such as "v + e" yielding a vector corresponding to the vector v with the scalar e added to each of its elements, has been deprecated. @@ -446,6 +439,10 @@ Deprecated Functionality - The undocumented feature of using "&&" and "||" operators for patterns has been deprecated. +- BroControl: The "update" command is deprecated and scheduled for + removal with the next Bro release. Bro's new configuration framework + is taking its place. + Bro 2.5.1 ========= From bd24421734a9cd26d6888fc7cb1cdb372a948b7c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 17 Aug 2018 15:10:03 -0500 Subject: [PATCH 20/24] BIT-466: add redef += support to vectors --- CHANGES | 4 ++++ NEWS | 3 +++ VERSION | 2 +- src/ID.cc | 5 ++-- src/Val.cc | 23 +++++++++++++++++++ src/Val.h | 4 ++++ .../btest/Baseline/language.redef-vector/out | 3 +++ testing/btest/language/redef-vector.bro | 18 +++++++++++++++ 8 files changed, 59 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/language.redef-vector/out create mode 100644 testing/btest/language/redef-vector.bro diff --git a/CHANGES b/CHANGES index c51cd00bbf..355b58f8fc 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.5-852 | 2018-08-17 15:15:55 -0500 + + * BIT-466: add redef += support to vectors (Jon Siwek, Corelight) + 2.5-850 | 2018-08-17 11:12:53 -0500 * BIT-1815: move SMB::write_cmd_log functionality into policy/ script diff --git a/NEWS b/NEWS index e5f5b28054..bb9ea619c8 100644 --- a/NEWS +++ b/NEWS @@ -314,6 +314,9 @@ New Functionality - An expression of the form "v += e" will append the value of the expression "e" to the end of the vector "v" (of course assuming type-compatbility). + "redef v += { a, b, c }" will similarly extend a vector previously declared + with &redef by appending the result of expressions "a", "b", and "c" to + the vector at initialization-time. Changed Functionality --------------------- diff --git a/VERSION b/VERSION index 3e949de53a..6aa99abe08 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-850 +2.5-852 diff --git a/src/ID.cc b/src/ID.cc index a68abb6264..9e4eb7d245 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -140,10 +140,11 @@ void ID::SetVal(Val* v, init_class c) } if ( type->Tag() != TYPE_TABLE && - (type->Tag() != TYPE_PATTERN || c == INIT_REMOVE) ) + (type->Tag() != TYPE_PATTERN || c == INIT_REMOVE) && + (type->Tag() != TYPE_VECTOR || c == INIT_REMOVE) ) { if ( c == INIT_EXTRA ) - Error("+= initializer only applies to tables, sets and patterns", v); + Error("+= initializer only applies to tables, sets, vectors and patterns", v); else Error("-= initializer only applies to tables and sets", v); } diff --git a/src/Val.cc b/src/Val.cc index 7879d282b2..71d38e3c63 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3321,6 +3321,29 @@ bool VectorVal::AssignRepeat(unsigned int index, unsigned int how_many, return true; } +int VectorVal::AddTo(Val* val, int /* is_first_init */) const + { + if ( val->Type()->Tag() != TYPE_VECTOR ) + { + val->Error("not a vector"); + return 0; + } + + VectorVal* v = val->AsVectorVal(); + + if ( ! same_type(type, v->Type()) ) + { + type->Error("vector type clash", v->Type()); + return 0; + } + + auto last_idx = v->Size(); + + for ( auto i = 0u; i < Size(); ++i ) + v->Assign(last_idx++, Lookup(i)->Ref()); + + return 1; + } Val* VectorVal::Lookup(unsigned int index) const { diff --git a/src/Val.h b/src/Val.h index bb18dceb4f..f189668b28 100644 --- a/src/Val.h +++ b/src/Val.h @@ -1046,6 +1046,10 @@ public: bool AssignRepeat(unsigned int index, unsigned int how_many, Val* element); + // Add this value to the given value (if appropriate). + // Returns true if succcessful. + int AddTo(Val* v, int is_first_init) const override; + // Returns nil if no element was at that value. // Lookup does NOT grow the vector to this size. // The Val* variant assumes that the index Val* has been type-checked. diff --git a/testing/btest/Baseline/language.redef-vector/out b/testing/btest/Baseline/language.redef-vector/out new file mode 100644 index 0000000000..73a5ef460d --- /dev/null +++ b/testing/btest/Baseline/language.redef-vector/out @@ -0,0 +1,3 @@ +[testing, blah, foo, foo, testing] +[one, two, three] +[a, b, c, one, two, three, a, b, c, abc, d] diff --git a/testing/btest/language/redef-vector.bro b/testing/btest/language/redef-vector.bro new file mode 100644 index 0000000000..26dc2109ba --- /dev/null +++ b/testing/btest/language/redef-vector.bro @@ -0,0 +1,18 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +const foo: vector of string &redef; +redef foo += { "testing", "blah", "foo", "foo", "testing" }; + +const bar: vector of string = vector() &redef; +redef bar += { "one", "two", "three" }; + +const baz: vector of string = vector("a", "b", "c") &redef; +redef baz += { "one", "two", "three" }; +redef baz += { "a", "b", "c" }; +const d = "d"; +redef baz += { "a" + "b" + "c", d }; + +print foo; +print bar; +print baz; From fa46c6a16a1354ab8d9eabfa82bf6c4b63e5aa40 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 17 Aug 2018 15:16:25 -0500 Subject: [PATCH 21/24] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index e0f9f6504d..2727afff78 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit e0f9f6504db9285a48e0be490abddf959999a404 +Subproject commit 2727afff785201168b33b0c7448c886643f10e08 From 491251351759e61271ad1a930907b4d031182c92 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 20 Aug 2018 12:45:32 -0500 Subject: [PATCH 22/24] Improve diff-remove-abspath canonifier: collapse '/' sequences --- CHANGES | 9 +++++++++ VERSION | 2 +- testing/scripts/diff-remove-abspath | 7 +++++++ 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index dc4947efe0..0eb7336ac2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,13 @@ +2.5-875 | 2018-08-20 12:45:32 -0500 + + * Improve diff-remove-abspath canonifier: collapse '/' sequences + (Jon Siwek, Corelight) + + * Remove unused redef-able constants (Daniel Thayer) + + * Convert some redef-able constants to runtime options (Daniel Thayer) + 2.5-870 | 2018-08-17 17:07:57 -0500 * Documentation improvements (Daniel Thayer) diff --git a/VERSION b/VERSION index 03a9771c0e..f77e8fcc00 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-870 +2.5-875 diff --git a/testing/scripts/diff-remove-abspath b/testing/scripts/diff-remove-abspath index 361ad3fa6d..a981206e83 100755 --- a/testing/scripts/diff-remove-abspath +++ b/testing/scripts/diff-remove-abspath @@ -2,4 +2,11 @@ # # Replace absolute paths with the basename. +if [ `uname` == "Linux" ]; then + sed="sed -r" +else + sed="sed -E" +fi + +$sed 's#/+#/#g' | \ sed 's#/\([^/]\{1,\}/\)\{1,\}\([^/]\{1,\}\)#<...>/\2#g' From 052a5b4d8498f8615770b16c0aa6257e76fc2129 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 20 Aug 2018 12:57:00 -0500 Subject: [PATCH 23/24] Remove the node-specific local-*.bro scripts --- scripts/CMakeLists.txt | 16 ---------------- scripts/site/local-logger.bro | 1 - scripts/site/local-manager.bro | 1 - scripts/site/local-proxy.bro | 1 - scripts/site/local-worker.bro | 1 - 5 files changed, 20 deletions(-) delete mode 100644 scripts/site/local-logger.bro delete mode 100644 scripts/site/local-manager.bro delete mode 100644 scripts/site/local-proxy.bro delete mode 100644 scripts/site/local-worker.bro diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index fb20ba6fdb..96c682871a 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -14,19 +14,3 @@ InstallPackageConfigFile( ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro ${BRO_SCRIPT_INSTALL_PATH}/site local.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-manager.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-logger.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-logger.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-proxy.bro) -InstallPackageConfigFile( - ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro - ${BRO_SCRIPT_INSTALL_PATH}/site - local-worker.bro) diff --git a/scripts/site/local-logger.bro b/scripts/site/local-logger.bro deleted file mode 100644 index 0642e86ce3..0000000000 --- a/scripts/site/local-logger.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the logger if Bro is running as a cluster. diff --git a/scripts/site/local-manager.bro b/scripts/site/local-manager.bro deleted file mode 100644 index 5e6005f21e..0000000000 --- a/scripts/site/local-manager.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the manager if Bro is running as a cluster. diff --git a/scripts/site/local-proxy.bro b/scripts/site/local-proxy.bro deleted file mode 100644 index 478ba6d048..0000000000 --- a/scripts/site/local-proxy.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the proxies if Bro is running as a cluster. diff --git a/scripts/site/local-worker.bro b/scripts/site/local-worker.bro deleted file mode 100644 index b2a100e135..0000000000 --- a/scripts/site/local-worker.bro +++ /dev/null @@ -1 +0,0 @@ -##! Local site policy loaded only by the workers if Bro is running as a cluster. \ No newline at end of file From ee0bbdad34f08189ce8f4ce6c65dfe3427bec878 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 20 Aug 2018 15:38:44 -0500 Subject: [PATCH 24/24] Fix outdated documentation test baselines --- CHANGES | 40 +++++++++++++++++++ VERSION | 2 +- .../output | 2 +- .../output | 2 +- ...licy_frameworks_files_detect-MHR_bro.btest | 2 +- ...cy_frameworks_files_detect-MHR_bro@3.btest | 2 +- 6 files changed, 45 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index 6b4c44d20e..b1c90ff1ab 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,44 @@ +2.5-884 | 2018-08-20 15:39:21 -0500 + + * Fix outdated documentation test baselines (Jon Siwek, Corelight) + + * Add 'smtp_excessive_pending_cmds' weird (Jon Siwek, Corelight) + + * Fix SMTP command string comparisons (Jon Siwek, Corelight) + + * Improve handling of empty lines in several text protocol analyzers + (Jon Siwek, Corelight) + + * Add rate-limiting sampling mechanism for weird events + + The generation of weird events, by default, are now rate-limited + according to these tunable options: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + The new get_reporter_stats() BIF also allows one to query the + total number of weirds generated (pre-sampling) which the new + policy/misc/weird-stats.bro script uses periodically to populate + a weird_stats.log. + + There's also new reporter BIFs to allow generating weirds from the + script-layer such that they go through the same, internal + rate-limiting/sampling mechanisms: + + - Reporter::conn_weird + - Reporter::flow_weird + - Reporter::net_weird + + Some of the code was adapted from previous work by Johanna Amann. + (Jon Siwek, Corelight) + + * Teach timestamp canonifier about timestamps before ~2001 + (Jon Siwek, Corelight) + 2.5-877 | 2018-08-20 14:58:58 -0500 * Remove the node-specific local-*.bro scripts (Daniel Thayer) diff --git a/VERSION b/VERSION index fea70a1577..4285fbbeff 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-877 +2.5-884 diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output index 03ba9cb3cd..8efd4099ad 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output @@ -36,7 +36,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output index 30c6b1040e..f1d91b8392 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output @@ -27,5 +27,5 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest index 03ba9cb3cd..8efd4099ad 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest @@ -36,7 +36,7 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; } function do_mhr_lookup(hash: string, fi: Notice::FileInfo) diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest index 30c6b1040e..f1d91b8392 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest @@ -27,5 +27,5 @@ export { ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold ## allows you to require a minimum detection rate. - const notice_threshold = 10 &redef; + option notice_threshold = 10; }