From 35827eeb31987a096031a1b4dd782dea64efa2b5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 26 Jul 2018 19:57:36 -0500 Subject: [PATCH] Add rate-limiting sampling mechanism for weird events The generation of weird events, by default, are now rate-limited according to these tunable options: - Weird::sampling_whitelist - Weird::sampling_threshold - Weird::sampling_rate - Weird::sampling_duration The new get_reporter_stats() BIF also allows one to query the total number of weirds generated (pre-sampling) which the new policy/misc/weird-stats.bro script uses periodically to populate a weird_stats.log. There's also new reporter BIFs to allow generating weirds from the script-layer such that they go through the same, internal rate-limiting/sampling mechanisms: - Reporter::conn_weird - Reporter::flow_weird - Reporter::net_weird Some of the code was adapted from previous work by Johanna Amann. --- NEWS | 15 + doc/script-reference/log-files.rst | 2 + scripts/base/init-bare.bro | 47 +++ scripts/base/protocols/http/main.bro | 2 +- scripts/base/protocols/sip/main.bro | 2 +- scripts/base/protocols/ssl/main.bro | 2 +- scripts/policy/misc/weird-stats.bro | 100 +++++ scripts/policy/protocols/smb/smb1-main.bro | 2 +- scripts/test-all-policy.bro | 1 + src/Conn.cc | 25 ++ src/Conn.h | 14 + src/Func.cc | 1 + src/Reporter.cc | 132 ++++++- src/Reporter.h | 51 ++- src/Timer.cc | 2 + src/Timer.h | 2 + src/reporter.bif | 41 ++ src/stats.bif | 49 +++ .../core.reporter-weird-sampling/output | 351 ++++++++++++++++++ .../btest/Baseline/coverage.find-bro-logs/out | 1 + .../manager-1.weird_stats.log | 13 + .../bro.weird_stats.log | 12 + .../btest/core/reporter-weird-sampling.bro | 55 +++ .../policy/misc/weird-stats-cluster.bro | 93 +++++ .../btest/scripts/policy/misc/weird-stats.bro | 32 ++ 25 files changed, 1037 insertions(+), 10 deletions(-) create mode 100644 scripts/policy/misc/weird-stats.bro create mode 100644 testing/btest/Baseline/core.reporter-weird-sampling/output create mode 100644 testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log create mode 100644 testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log create mode 100644 testing/btest/core/reporter-weird-sampling.bro create mode 100644 testing/btest/scripts/policy/misc/weird-stats-cluster.bro create mode 100644 testing/btest/scripts/policy/misc/weird-stats.bro diff --git a/NEWS b/NEWS index 3cb0fd79b3..a878c9398e 100644 --- a/NEWS +++ b/NEWS @@ -356,6 +356,21 @@ Changed Functionality - The string_to_pattern() built-in (and the now-deprecated merge_pattern() built-in) is no longer restricted to only be called at initialization time. +- "Weird" events are now generally suppressed/sampled by default according to + some tunable parameters: + + - Weird::sampling_whitelist + - Weird::sampling_threshold + - Weird::sampling_rate + - Weird::sampling_duration + + Those options can be changed if one needs the previous behavior of + a "net_weird", "flow_weird", or "conn_weird" event being raised for + every single event. Otherwise, there is a new weird_stats.log which + contains concise summaries of weird counts per type per time period + and the original weird.log may not differ much either, except in + the cases where a particular weird type exceeds the sampling threshold. + Removed Functionality --------------------- diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 8f7d87d89a..513ece8864 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -154,6 +154,8 @@ Miscellaneous +----------------------------+---------------------------------------+---------------------------------+ | weird.log | Unexpected network-level activity | :bro:type:`Weird::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| weird-stats.log | Statistics about unexpected activity | :bro:type:`WeirdStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ Bro Diagnostics --------------- diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index c502607cbd..c37f924dfc 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -82,6 +82,13 @@ type addr_vec: vector of addr; ## directly and then remove this alias. type table_string_of_string: table[string] of string; +## A table of counts indexed by strings. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type table_string_of_count: table[string] of count; + ## A set of file analyzer tags. ## ## .. todo:: We need this type definition only for declaring builtin functions @@ -626,6 +633,17 @@ type BrokerStats: record { num_ids_outgoing: count; }; +## Statistics about reporter messages and weirds. +## +## .. bro:see:: get_reporter_stats +type ReporterStats: record { + ## Number of total weirds encountered, before any rate-limiting. + weirds: count; + ## Number of times each individual weird is encountered, before any + ## rate-limiting is applied. + weirds_by_type: table[string] of count; +}; + ## Deprecated. ## ## .. todo:: Remove. It's still declared internally but doesn't seem used anywhere @@ -4823,6 +4841,35 @@ export { type Cluster::Pool: record {}; } +module Weird; +export { + ## Prevents rate-limiting sampling of any weirds named in the table. + const sampling_whitelist: set[string] &redef; + + ## How many weirds of a given type to tolerate before sampling begins. + ## i.e. this many consecutive weirds of a given type will be allowed to + ## raise events for script-layer handling before being rate-limited. + const sampling_threshold = 25 &redef; + + ## The rate-limiting sampling rate. One out of every of this number of + ## rate-limited weirds of a given type will be allowed to raise events + ## for further script-layer handling. + const sampling_rate = 1000 &redef; + + ## How long a weird of a given type is allowed to keep state/counters in + ## memory. For "net" weirds an expiration timer starts per weird name when + ## first initializing its counter. For "flow" weirds an expiration timer + ## starts once per src/dst IP pair for the first weird of any name. For + ## "conn" weirds, counters and expiration timers are kept for the duration + ## of the connection for each named weird and reset when necessary. e.g. + ## if a "conn" weird by the name of "foo" is seen more than + ## :bro:see:`Weird::sampling_threshold` times, then an expiration timer + ## begins for "foo" and upon triggering will reset the counter for "foo" + ## and unthrottle its rate-limiting until it once again exceeds the + ## threshold. + const sampling_duration = 10min &redef; +} + module GLOBAL; ## Seed for hashes computed internally for probabilistic data structures. Using diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 51a89a33b9..ef949df4e3 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string, c$http$uri = unescaped_URI; if ( method !in http_methods ) - event conn_weird("unknown_HTTP_method", c, method); + Reporter::conn_weird("unknown_HTTP_method", c, method); } event http_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/sip/main.bro b/scripts/base/protocols/sip/main.bro index f629049928..09d9609481 100644 --- a/scripts/base/protocols/sip/main.bro +++ b/scripts/base/protocols/sip/main.bro @@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version: c$sip$uri = original_URI; if ( method !in sip_methods ) - event conn_weird("unknown_SIP_method", c, method); + Reporter::conn_weird("unknown_SIP_method", c, method); } event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5 diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 6ac4260537..85aedac063 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) { c$ssl$server_name = names[0]; if ( |names| > 1 ) - event conn_weird("SSL_many_server_names", c, cat(names)); + Reporter::conn_weird("SSL_many_server_names", c, cat(names)); } } diff --git a/scripts/policy/misc/weird-stats.bro b/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..d08ce9381d --- /dev/null +++ b/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,100 @@ +##! Log weird statistics. + +@load base/frameworks/sumstats +@load base/frameworks/cluster + +module WeirdStats; + +export { + redef enum Log::ID += { LOG }; + + ## How often stats are reported. + const weird_stat_interval = 15min &redef; + + type Info: record { + ## Timestamp for the measurement. + ts: time &log; + ## Name of the weird. + name: string &log; + ## Number of times weird was seen since the last stats interval. + num_seen: count &log; + }; + + global log_weird_stats: event(rec: Info); +} + +global this_epoch_weirds: table[string] of double; +global last_epoch_weirds: table[string] of double; + +function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result) + { + this_epoch_weirds[key$str]=result["weirds.encountered"]$sum; + } + +function weird_epoch_finished(ts: time) + { + for ( n in this_epoch_weirds ) + { + local last_count: double = 0.0; + + if ( n in last_epoch_weirds ) + last_count = last_epoch_weirds[n]; + + local num_seen: double = this_epoch_weirds[n] - last_count; + + if ( num_seen > 0.0 ) + Log::write(LOG, Info($ts = ts, $name = n, + $num_seen = double_to_count(num_seen))); + } + + last_epoch_weirds = this_epoch_weirds; + this_epoch_weirds = table(); + } + +event bro_init() &priority=5 + { + Log::create_stream(WeirdStats::LOG, + [$columns = Info, $ev = log_weird_stats, + $path="weird_stats"]); + local r1 = SumStats::Reducer($stream = "weirds.encountered", + $apply = set(SumStats::SUM)); + SumStats::create([$name = "weirds.statistics", + $epoch = weird_stat_interval, $reducers = set(r1), + $epoch_result = weird_epoch_results, + $epoch_finished = weird_epoch_finished]); + } + +module SumStats; + +function observe_weird_stats() + { + local rs = get_reporter_stats(); + + for ( n in rs$weirds_by_type ) + SumStats::observe("weirds.encountered", SumStats::Key($str = n), + SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0)); + } + +@if ( Cluster::is_enabled() ) + +# I'm not sure if this is a hack or not: the manager will generate this +# event at the end of its epoch so workers can handle it just in time to +# generate the necessary stats. Alternative may be workers generating the +# stats individually/proactively in their own finish_epoch, but that may be +# less synchronized? +event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10 + { + observe_weird_stats(); + } + +@else + +event SumStats::finish_epoch(ss: SumStat) &priority=10 + { + if ( ss$name != "weirds.statistics" ) + return; + + observe_weird_stats(); + } + +@endif diff --git a/scripts/policy/protocols/smb/smb1-main.bro b/scripts/policy/protocols/smb/smb1-main.bro index 6b23fe91db..3e0e7da3ed 100644 --- a/scripts/policy/protocols/smb/smb1-main.bro +++ b/scripts/policy/protocols/smb/smb1-main.bro @@ -316,7 +316,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count) if ( ! f?$uuid ) { # TODO: figure out why this is happening. - event conn_weird("smb_pipe_request_missing_uuid", c, ""); + Reporter::conn_weird("smb_pipe_request_missing_uuid", c, ""); return; } local arg = fmt("%s: %s", diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 3389942fe5..cf4a35399f 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -54,6 +54,7 @@ @load misc/profiling.bro @load misc/scan.bro @load misc/stats.bro +@load misc/weird-stats.bro @load misc/trim-trace-file.bro @load protocols/conn/known-hosts.bro @load protocols/conn/known-services.bro diff --git a/src/Conn.cc b/src/Conn.cc index 1edecde0b9..719d55d3b0 100644 --- a/src/Conn.cc +++ b/src/Conn.cc @@ -1027,3 +1027,28 @@ void Connection::CheckFlowLabel(bool is_orig, uint32 flow_label) else saw_first_resp_packet = 1; } + +bool Connection::PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration) + { + auto& state = weird_state[name]; + ++state.count; + + if ( state.count < threshold ) + return true; + + if ( state.count == threshold ) + state.sampling_start_time = network_time; + else + { + if ( network_time > state.sampling_start_time + duration ) + { + state.sampling_start_time = 0; + state.count = 1; + return true; + } + } + + auto num_above_threshold = state.count - threshold; + return num_above_threshold % rate == 0; + } diff --git a/src/Conn.h b/src/Conn.h index db0cb2fe55..ee73ab24d5 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -5,6 +5,9 @@ #include +#include +#include + #include "Dict.h" #include "Val.h" #include "Timer.h" @@ -264,6 +267,9 @@ public: uint32 GetOrigFlowLabel() { return orig_flow_label; } uint32 GetRespFlowLabel() { return resp_flow_label; } + bool PermitWeird(const char* name, uint64 threshold, uint64 rate, + double duration); + protected: Connection() { persistent = 0; } @@ -328,6 +334,14 @@ protected: analyzer::pia::PIA* primary_PIA; Bro::UID uid; // Globally unique connection ID. + + struct WeirdState { + WeirdState() { count = 0; sampling_start_time = 0; } + uint64 count = 0; + double sampling_start_time = 0; + }; + + std::unordered_map weird_state; }; class ConnectionTimer : public Timer { diff --git a/src/Func.cc b/src/Func.cc index 703cc04d29..015829e2a1 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -722,6 +722,7 @@ void init_builtin_funcs() FileAnalysisStats = internal_type("FileAnalysisStats")->AsRecordType(); ThreadStats = internal_type("ThreadStats")->AsRecordType(); BrokerStats = internal_type("BrokerStats")->AsRecordType(); + ReporterStats = internal_type("ReporterStats")->AsRecordType(); var_sizes = internal_type("var_sizes")->AsTableType(); diff --git a/src/Reporter.cc b/src/Reporter.cc index eb89a29d30..59e877bb7c 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -10,6 +10,7 @@ #include "NetVar.h" #include "Net.h" #include "Conn.h" +#include "Timer.h" #include "plugin/Plugin.h" #include "plugin/Manager.h" @@ -36,6 +37,11 @@ Reporter::Reporter() warnings_to_stderr = true; errors_to_stderr = true; + weird_count = 0; + weird_sampling_rate = 0; + weird_sampling_duration = 0; + weird_sampling_threshold = 0; + openlog("bro", 0, LOG_LOCAL5); } @@ -49,6 +55,24 @@ void Reporter::InitOptions() info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool(); warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool(); errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool(); + weird_sampling_rate = internal_const_val("Weird::sampling_rate")->AsCount(); + weird_sampling_threshold = internal_const_val("Weird::sampling_threshold")->AsCount(); + weird_sampling_duration = internal_const_val("Weird::sampling_duration")->AsInterval(); + auto wl_val = internal_const_val("Weird::sampling_whitelist")->AsTableVal(); + auto wl_table = wl_val->AsTable(); + + HashKey* k; + IterCookie* c = wl_table->InitForIteration(); + TableEntryVal* v; + + while ( (v = wl_table->NextEntry(k, c)) ) + { + auto index = wl_val->RecoverIndex(k); + string key = index->Index(0)->AsString()->CheckString(); + weird_sampling_whitelist.emplace(move(key)); + Unref(index); + delete k; + } } void Reporter::Info(const char* fmt, ...) @@ -221,23 +245,121 @@ void Reporter::WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const cha delete vl; } +void Reporter::UpdateWeirdStats(const char* name) + { + ++weird_count; + ++weird_count_by_type[name]; + } + +class NetWeirdTimer : public Timer { +public: + NetWeirdTimer(double t, const char* name, double timeout) + : Timer(t + timeout, TIMER_NET_WEIRD_EXPIRE), weird_name(name) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetNetWeird(weird_name); } + + std::string weird_name; +}; + +class FlowWeirdTimer : public Timer { +public: + using IPPair = std::pair; + + FlowWeirdTimer(double t, IPPair p, double timeout) + : Timer(t + timeout, TIMER_FLOW_WEIRD_EXPIRE), endpoints(p) + {} + + void Dispatch(double t, int is_expire) override + { reporter->ResetFlowWeird(endpoints.first, endpoints.second); } + + IPPair endpoints; +}; + +void Reporter::ResetNetWeird(const std::string& name) + { + net_weird_state.erase(name); + } + +void Reporter::ResetFlowWeird(const IPAddr& orig, const IPAddr& resp) + { + flow_weird_state.erase(std::make_pair(orig, resp)); + } + +bool Reporter::PermitNetWeird(const char* name) + { + auto& count = net_weird_state[name]; + ++count; + + if ( count == 1 ) + timer_mgr->Add(new NetWeirdTimer(network_time, name, + weird_sampling_duration)); + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + +bool Reporter::PermitFlowWeird(const char* name, + const IPAddr& orig, const IPAddr& resp) + { + auto endpoints = std::make_pair(orig, resp); + auto& map = flow_weird_state[endpoints]; + + if ( map.empty() ) + timer_mgr->Add(new FlowWeirdTimer(network_time, endpoints, + weird_sampling_duration)); + + auto& count = map[name]; + ++count; + + if ( count < weird_sampling_threshold ) + return true; + + auto num_above_threshold = count - weird_sampling_threshold; + return num_above_threshold % weird_sampling_rate == 0; + } + void Reporter::Weird(const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitNetWeird(name) ) + return; + } + WeirdHelper(net_weird, 0, 0, "%s", name); } void Reporter::Weird(Connection* conn, const char* name, const char* addl) { - WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); - } + UpdateWeirdStats(name); -void Reporter::Weird(Val* conn_val, const char* name, const char* addl) - { - WeirdHelper(conn_weird, conn_val, addl, "%s", name); + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! conn->PermitWeird(name, weird_sampling_threshold, + weird_sampling_rate, weird_sampling_duration) ) + return; + } + + WeirdHelper(conn_weird, conn->BuildConnVal(), addl, "%s", name); } void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name) { + UpdateWeirdStats(name); + + if ( ! WeirdOnSamplingWhiteList(name) ) + { + if ( ! PermitFlowWeird(name, orig, resp) ) + return; + } + WeirdFlowHelper(orig, resp, "%s", name); } diff --git a/src/Reporter.h b/src/Reporter.h index 85b9a483a2..f5090ee91e 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -7,6 +7,10 @@ #include #include +#include +#include +#include +#include #include "util.h" #include "EventHandler.h" @@ -36,6 +40,11 @@ protected: class Reporter { public: + using IPPair = std::pair; + using WeirdCountMap = std::unordered_map; + using WeirdFlowMap = std::map; + using WeirdSet = std::unordered_set; + Reporter(); ~Reporter(); @@ -76,7 +85,6 @@ public: // that may lead to incorrectly processing a connnection. void Weird(const char* name); // Raises net_weird(). void Weird(Connection* conn, const char* name, const char* addl = ""); // Raises conn_weird(). - void Weird(Val* conn_val, const char* name, const char* addl = ""); // Raises conn_weird(). void Weird(const IPAddr& orig, const IPAddr& resp, const char* name); // Raises flow_weird(). // Syslog a message. This methods does nothing if we're running @@ -120,6 +128,30 @@ public: // Signals that we're done processing an error handler event. void EndErrorHandler() { --in_error_handler; } + /** + * Reset/cleanup state tracking for a "net" weird. + */ + void ResetNetWeird(const std::string& name); + + /** + * Reset/cleanup state tracking for a "flow" weird. + */ + void ResetFlowWeird(const IPAddr& orig, const IPAddr& resp); + + /** + * Return the total number of weirds generated (counts weirds before + * any rate-limiting occurs). + */ + uint64 GetWeirdCount() const + { return weird_count; } + + /** + * Return number of weirds generated per weird type/name (counts weirds + * before any rate-limiting occurs). + */ + const WeirdCountMap& GetWeirdsByType() const + { return weird_count_by_type; } + private: void DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, @@ -129,6 +161,11 @@ private: // contain format specifiers void WeirdHelper(EventHandlerPtr event, Val* conn_val, const char* addl, const char* fmt_name, ...) __attribute__((format(printf, 5, 6)));; void WeirdFlowHelper(const IPAddr& orig, const IPAddr& resp, const char* fmt_name, ...) __attribute__((format(printf, 4, 5)));; + void UpdateWeirdStats(const char* name); + inline bool WeirdOnSamplingWhiteList(const char* name) + { return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); } + bool PermitNetWeird(const char* name); + bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r); int errors; bool via_events; @@ -138,6 +175,18 @@ private: bool errors_to_stderr; std::list > locations; + + uint64 weird_count; + WeirdCountMap weird_count_by_type; + + WeirdCountMap net_weird_state; + WeirdFlowMap flow_weird_state; + + WeirdSet weird_sampling_whitelist; + uint64 weird_sampling_threshold; + uint64 weird_sampling_rate; + double weird_sampling_duration; + }; extern Reporter* reporter; diff --git a/src/Timer.cc b/src/Timer.cc index 4fe998606b..101733028c 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -18,12 +18,14 @@ const char* TimerNames[] = { "ConnectionStatusUpdateTimer", "DNSExpireTimer", "FileAnalysisInactivityTimer", + "FlowWeirdTimer", "FragTimer", "IncrementalSendTimer", "IncrementalWriteTimer", "InterconnTimer", "IPTunnelInactivityTimer", "NetbiosExpireTimer", + "NetWeirdTimer", "NetworkTimer", "NTPExpireTimer", "ProfileTimer", diff --git a/src/Timer.h b/src/Timer.h index ea410e5c7b..8d6de857a0 100644 --- a/src/Timer.h +++ b/src/Timer.h @@ -23,12 +23,14 @@ enum TimerType { TIMER_CONN_STATUS_UPDATE, TIMER_DNS_EXPIRE, TIMER_FILE_ANALYSIS_INACTIVITY, + TIMER_FLOW_WEIRD_EXPIRE, TIMER_FRAG, TIMER_INCREMENTAL_SEND, TIMER_INCREMENTAL_WRITE, TIMER_INTERCONN, TIMER_IP_TUNNEL_INACTIVITY, TIMER_NB_EXPIRE, + TIMER_NET_WEIRD_EXPIRE, TIMER_NETWORK, TIMER_NTP_EXPIRE, TIMER_PROFILE, diff --git a/src/reporter.bif b/src/reporter.bif index 3cdefc4da4..a6463f6226 100644 --- a/src/reporter.bif +++ b/src/reporter.bif @@ -71,3 +71,44 @@ function Reporter::fatal%(msg: string%): bool reporter->PopLocation(); return new Val(1, TYPE_BOOL); %} + +## Generates a "net" weird. +## +## name: the name of the weird. +## +## Returns: Always true. +function Reporter::net_weird%(name: string%): bool + %{ + reporter->Weird(name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "flow" weird. +## +## name: the name of the weird. +## +## orig: the originator host associated with the weird. +## +## resp: the responder host associated with the weird. +## +## Returns: Always true. +function Reporter::flow_weird%(name: string, orig: addr, resp: addr%): bool + %{ + reporter->Weird(orig->AsAddr(), resp->AsAddr(), name->CheckString()); + return new Val(1, TYPE_BOOL); + %} + +## Generates a "conn" weird. +## +## name: the name of the weird. +## +## c: the connection associated with the weird. +## +## addl: additional information to accompany the weird. +## +## Returns: Always true. +function Reporter::conn_weird%(name: string, c: connection, addl: string &default=""%): bool + %{ + reporter->Weird(c, name->CheckString(), addl->CheckString()); + return new Val(1, TYPE_BOOL); + %} diff --git a/src/stats.bif b/src/stats.bif index 2a525a1790..e3ce6f0245 100644 --- a/src/stats.bif +++ b/src/stats.bif @@ -16,6 +16,7 @@ RecordType* ThreadStats; RecordType* TimerStats; RecordType* FileAnalysisStats; RecordType* BrokerStats; +RecordType* ReporterStats; %%} ## Returns packet capture statistics. Statistics include the number of @@ -35,6 +36,7 @@ RecordType* BrokerStats; ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_net_stats%(%): NetStats %{ uint64 recv = 0; @@ -83,6 +85,7 @@ function get_net_stats%(%): NetStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_conn_stats%(%): ConnStats %{ RecordVal* r = new RecordVal(ConnStats); @@ -133,6 +136,7 @@ function get_conn_stats%(%): ConnStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_proc_stats%(%): ProcStats %{ struct rusage ru; @@ -189,6 +193,7 @@ function get_proc_stats%(%): ProcStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_event_stats%(%): EventStats %{ RecordVal* r = new RecordVal(EventStats); @@ -215,6 +220,7 @@ function get_event_stats%(%): EventStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_reassembler_stats%(%): ReassemblerStats %{ RecordVal* r = new RecordVal(ReassemblerStats); @@ -243,6 +249,7 @@ function get_reassembler_stats%(%): ReassemblerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_dns_stats%(%): DNSStats %{ RecordVal* r = new RecordVal(DNSStats); @@ -276,6 +283,7 @@ function get_dns_stats%(%): DNSStats ## get_reassembler_stats ## get_thread_stats ## get_broker_stats +## get_reporter_stats function get_timer_stats%(%): TimerStats %{ RecordVal* r = new RecordVal(TimerStats); @@ -303,6 +311,7 @@ function get_timer_stats%(%): TimerStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_file_analysis_stats%(%): FileAnalysisStats %{ RecordVal* r = new RecordVal(FileAnalysisStats); @@ -330,6 +339,7 @@ function get_file_analysis_stats%(%): FileAnalysisStats ## get_reassembler_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_thread_stats%(%): ThreadStats %{ RecordVal* r = new RecordVal(ThreadStats); @@ -355,6 +365,7 @@ function get_thread_stats%(%): ThreadStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_gap_stats%(%): GapStats %{ RecordVal* r = new RecordVal(GapStats); @@ -386,6 +397,7 @@ function get_gap_stats%(%): GapStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_matcher_stats%(%): MatcherStats %{ RecordVal* r = new RecordVal(MatcherStats); @@ -423,6 +435,7 @@ function get_matcher_stats%(%): MatcherStats ## get_thread_stats ## get_timer_stats ## get_broker_stats +## get_reporter_stats function get_broker_stats%(%): BrokerStats %{ RecordVal* r = new RecordVal(BrokerStats); @@ -441,3 +454,39 @@ function get_broker_stats%(%): BrokerStats return r; %} + +## Returns statistics about reporter messages and weirds. +## +## Returns: A record with reporter statistics. +## +## .. bro:see:: get_conn_stats +## get_dns_stats +## get_event_stats +## get_file_analysis_stats +## get_gap_stats +## get_matcher_stats +## get_net_stats +## get_proc_stats +## get_reassembler_stats +## get_thread_stats +## get_timer_stats +## get_broker_stats +function get_reporter_stats%(%): ReporterStats + %{ + RecordVal* r = new RecordVal(ReporterStats); + int n = 0; + + TableVal* weirds_by_type = new TableVal(internal_type("table_string_of_count")->AsTableType()); + + for ( auto& kv : reporter->GetWeirdsByType() ) + { + Val* weird = new StringVal(kv.first); + weirds_by_type->Assign(weird, new Val(kv.second, TYPE_COUNT)); + Unref(weird); + } + + r->Assign(n++, new Val(reporter->GetWeirdCount(), TYPE_COUNT)); + r->Assign(n++, weirds_by_type); + + return r; + %} diff --git a/testing/btest/Baseline/core.reporter-weird-sampling/output b/testing/btest/Baseline/core.reporter-weird-sampling/output new file mode 100644 index 0000000000..b9be6227dd --- /dev/null +++ b/testing/btest/Baseline/core.reporter-weird-sampling/output @@ -0,0 +1,351 @@ +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird +net_weird, my_net_weird +flow_weird, my_flow_weird +conn_weird, my_conn_weird +net_weird, whitelisted_net_weird +flow_weird, whitelisted_flow_weird +conn_weird, whitelisted_conn_weird diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out index 70a635b24f..cac0156707 100644 --- a/testing/btest/Baseline/coverage.find-bro-logs/out +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -55,4 +55,5 @@ traceroute tunnel unified2 weird +weird_stats x509 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log new file mode 100644 index 0000000000..001da38e49 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats-cluster/manager-1.weird_stats.log @@ -0,0 +1,13 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-26-23-11-27 +#fields ts name num_seen +#types time string count +1532646687.827249 weird3 1 +1532646687.827249 weird2 1000 +1532646687.827249 weird1 2000 +1532646692.877464 weird1 2 +#close 2018-07-26-23-11-34 diff --git a/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log new file mode 100644 index 0000000000..e44145ed86 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.misc.weird-stats/bro.weird_stats.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird_stats +#open 2018-07-27-00-20-35 +#fields ts name num_seen +#types time string count +1532650834.978616 my_weird 1000 +1532650840.011592 my_weird 2000 +1532650845.043367 my_weird 10 +#close 2018-07-27-00-20-47 diff --git a/testing/btest/core/reporter-weird-sampling.bro b/testing/btest/core/reporter-weird-sampling.bro new file mode 100644 index 0000000000..d9d99681c4 --- /dev/null +++ b/testing/btest/core/reporter-weird-sampling.bro @@ -0,0 +1,55 @@ +# @TEST-EXEC: bro -b -r $TRACES/http/bro.org.pcap %INPUT >output +# @TEST-EXEC: btest-diff output + +redef Weird::sampling_duration = 5sec; +redef Weird::sampling_threshold = 10; +redef Weird::sampling_rate = 10; +redef Weird::sampling_whitelist = set("whitelisted_net_weird", + "whitelisted_flow_weird", + "whitelisted_conn_weird"); + +event conn_weird(name: string, c: connection, addl: string) + { + print "conn_weird", name; + } + +event flow_weird(name: string, src: addr, dst: addr) + { + print "flow_weird", name; + } + +event net_weird(name: string) + { + print "net_weird", name; + } + +event gen_weirds(c: connection) + { + local num = 30; + + while ( num != 0 ) + { + Reporter::net_weird("my_net_weird"); + Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("my_conn_weird", c); + + Reporter::net_weird("whitelisted_net_weird"); + Reporter::flow_weird("whitelisted_flow_weird", c$id$orig_h, c$id$resp_h); + Reporter::conn_weird("whitelisted_conn_weird", c); + --num; + } + } + +global did_one_connection = F; + +event new_connection(c: connection) + { + if ( did_one_connection ) + return; + + did_one_connection = T; + event gen_weirds(c); # should permit 10 + 2 of each "my" weird + schedule 2sec { gen_weirds(c) }; # should permit 3 of each "my" weird + schedule 7sec { gen_weirds(c) }; # should permit 10 + 2 of each "my" weird + # Total of 27 "my" weirds of each type and 90 of each "whitelisted" type + } diff --git a/testing/btest/scripts/policy/misc/weird-stats-cluster.bro b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro new file mode 100644 index 0000000000..492b682c4d --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats-cluster.bro @@ -0,0 +1,93 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 + +# @TEST-EXEC: btest-diff manager-1/weird_stats.log + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"], +}; +@TEST-END-FILE + +@load misc/weird-stats + +redef Cluster::retry_interval = 1sec; +redef Broker::default_listen_retry = 1sec; +redef Broker::default_connect_retry = 1sec; + +redef Log::enable_local_logging = T; +redef Log::default_rotation_interval = 0secs; +redef WeirdStats::weird_stat_interval = 5secs; + +event terminate_me() + { + terminate(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + terminate(); + } + +event ready_again() + { + Reporter::net_weird("weird1"); + + if ( Cluster::node == "worker-2" ) + { + schedule 5secs { terminate_me() }; + } + } + +event ready_for_data() + { + local n = 0; + + if ( Cluster::node == "worker-1" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + ++n; + } + + Reporter::net_weird("weird3"); + } + else if ( Cluster::node == "worker-2" ) + { + while ( n < 1000 ) + { + Reporter::net_weird("weird1"); + Reporter::net_weird("weird2"); + ++n; + } + } + + schedule 5secs { ready_again() }; + } + + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +event bro_init() + { + Broker::auto_publish(Cluster::worker_topic, ready_for_data); + } + +global peer_count = 0; + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + ++peer_count; + + if ( peer_count == 2 ) + event ready_for_data(); + } + +@endif diff --git a/testing/btest/scripts/policy/misc/weird-stats.bro b/testing/btest/scripts/policy/misc/weird-stats.bro new file mode 100644 index 0000000000..b26fce8e47 --- /dev/null +++ b/testing/btest/scripts/policy/misc/weird-stats.bro @@ -0,0 +1,32 @@ +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff bro/weird_stats.log + +@load misc/weird-stats.bro + +redef exit_only_after_terminate = T; +redef WeirdStats::weird_stat_interval = 5sec; + +event die() + { + terminate(); + } + +event gen_weirds(n: count, done: bool &default = F) + { + while ( n != 0 ) + { + Reporter::net_weird("my_weird"); + --n; + } + + if ( done ) + schedule 5sec { die() }; + } + +event bro_init() + { + event gen_weirds(1000); + schedule 7.5sec { gen_weirds(2000) } ; + schedule 12.5sec { gen_weirds(10, T) } ; + }