Merge remote-tracking branch 'origin/topic/robin/gh-623-sampling'

- Merge adjustments:
  - Minor whitespace/style tweaks
  - Fixed portability of the btest due to differences in `uniq -c`
    output format

* origin/topic/robin/gh-623-sampling:
  Extend weird sampling with option to track selected weirds globally.

CHANGES

@@ -1,4 +1,11 @@
 
+3.3.0-dev.252 | 2020-09-08 17:04:19 -0700
+
+  * GH-623: Extend weird sampling with option to track selected weirds globally. (Robin Sommer, Corelight)
+
+    The new set "sampling_global_list" lists weirds to rate-limit
+    globally instead of per connection/flow.
+
 3.3.0-dev.249 | 2020-09-04 18:30:19 -0700
 
   * Exclude installing "zeek -> ." include dir symlink (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-3.3.0-dev.249
+3.3.0-dev.252

scripts/base/init-bare.zeek

@@ -5263,6 +5263,9 @@ export {
 	## Prevents rate-limiting sampling of any weirds named in the table.
 	option sampling_whitelist: set[string] = {};
 
+	## Rate-limits weird names in the table globally instead of per connection/flow.
+	option sampling_global_list: set[string] = {};
+
 	## How many weirds of a given type to tolerate before sampling begins.
 	## I.e. this many consecutive weirds of a given type will be allowed to
 	## raise events for script-layer handling before being rate-limited.

src/Reporter.cc

@@ -72,7 +72,10 @@ void Reporter::InitOptions()
 	weird_sampling_rate = id::find_val("Weird::sampling_rate")->AsCount();
 	weird_sampling_threshold = id::find_val("Weird::sampling_threshold")->AsCount();
 	weird_sampling_duration = id::find_val("Weird::sampling_duration")->AsInterval();
-	auto wl_val = id::find_val("Weird::sampling_whitelist")->AsTableVal();
+
+	auto init_weird_set = [](WeirdSet* set, const char* name)
+		{
+		auto wl_val = id::find_val(name)->AsTableVal();
 		auto wl_table = wl_val->AsTable();
 
 		detail::HashKey* k;
@@ -83,9 +86,13 @@ void Reporter::InitOptions()
 			{
 			auto index = wl_val->RecreateIndex(*k);
 			std::string key = index->Idx(0)->AsString()->CheckString();
-		weird_sampling_whitelist.emplace(move(key));
+			set->emplace(move(key));
 			delete k;
 			}
+		};
+
+	init_weird_set(&weird_sampling_whitelist, "Weird::sampling_whitelist");
+	init_weird_set(&weird_sampling_global_list, "Weird::sampling_global_list");
 	}
 
 void Reporter::Info(const char* fmt, ...)
@@ -307,6 +314,18 @@ void Reporter::ResetExpiredConnWeird(const ConnTuple& id)
 	expired_conn_weird_state.erase(id);
 	}
 
+Reporter::PermitWeird Reporter::CheckGlobalWeirdLists(const char* name)
+	{
+	if ( WeirdOnSamplingWhiteList(name) )
+		return PermitWeird::Allow;
+
+	if ( WeirdOnGlobalList(name) )
+		// We track weirds on the global list through the "net_weird" table.
+		return PermitNetWeird(name) ? PermitWeird::Allow : PermitWeird::Deny;
+
+	return PermitWeird::Unknown;
+	}
+
 bool Reporter::PermitNetWeird(const char* name)
 	{
 	auto& count = net_weird_state[name];
@@ -395,8 +414,12 @@ void Reporter::Weird(file_analysis::File* f, const char* name, const char* addl)
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
+	switch ( CheckGlobalWeirdLists(name) ) {
-		{
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! f->PermitWeird(name, weird_sampling_threshold,
 		                      weird_sampling_rate, weird_sampling_duration) )
 			return;
@@ -410,8 +433,12 @@ void Reporter::Weird(Connection* conn, const char* name, const char* addl)
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
+	switch ( CheckGlobalWeirdLists(name) ) {
-		{
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! conn->PermitWeird(name, weird_sampling_threshold,
 		                         weird_sampling_rate, weird_sampling_duration) )
 			return;
@@ -426,8 +453,12 @@ void Reporter::Weird(RecordValPtr conn_id, StringValPtr uid,
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
+	switch ( CheckGlobalWeirdLists(name) ) {
-		{
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! PermitExpiredConnWeird(name, *conn_id) )
 			return;
 	}
@@ -441,8 +472,12 @@ void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name, c
 	{
 	UpdateWeirdStats(name);
 
-	if ( ! WeirdOnSamplingWhiteList(name) )
+	switch ( CheckGlobalWeirdLists(name) ) {
-		{
+	case PermitWeird::Allow:
+		break;
+	case PermitWeird::Deny:
+		return;
+	case PermitWeird::Unknown:
 		if ( ! PermitFlowWeird(name, orig, resp) )
 			 return;
 	}

src/Reporter.h

@@ -190,6 +190,24 @@ public:
 		this->weird_sampling_whitelist = weird_sampling_whitelist;
 		}
 
+	/**
+	 * Gets the weird sampling global list.
+	 */
+	WeirdSet GetWeirdSamplingGlobalList() const
+		{
+		return weird_sampling_global_list;
+		}
+
+	/**
+	 * Sets the weird sampling global list.
+	 *
+	 * @param weird_sampling_global list New weird sampling global list.
+	 */
+	void SetWeirdSamplingGlobalList(const WeirdSet& weird_sampling_global_list)
+		{
+		this->weird_sampling_global_list = weird_sampling_global_list;
+		}
+
 	/**
 	 * Gets the current weird sampling threshold.
 	 *
@@ -269,10 +287,15 @@ private:
 	void UpdateWeirdStats(const char* name);
 	inline bool WeirdOnSamplingWhiteList(const char* name)
 		{ return weird_sampling_whitelist.find(name) != weird_sampling_whitelist.end(); }
+	inline bool WeirdOnGlobalList(const char* name)
+		{ return weird_sampling_global_list.find(name) != weird_sampling_global_list.end(); }
 	bool PermitNetWeird(const char* name);
 	bool PermitFlowWeird(const char* name, const IPAddr& o, const IPAddr& r);
 	bool PermitExpiredConnWeird(const char* name, const RecordVal& conn_id);
 
+	enum class PermitWeird { Allow, Deny, Unknown };
+	PermitWeird CheckGlobalWeirdLists(const char* name);
+
 	bool EmitToStderr(bool flag)
 		{ return flag || ! after_zeek_init; }
 
@@ -294,6 +317,7 @@ private:
 	WeirdConnTupleMap expired_conn_weird_state;
 
 	WeirdSet weird_sampling_whitelist;
+	WeirdSet weird_sampling_global_list;
 	uint64_t weird_sampling_threshold;
 	uint64_t weird_sampling_rate;
 	double weird_sampling_duration;

src/reporter.bif

@@ -198,6 +198,46 @@ function Reporter::set_weird_sampling_whitelist%(weird_sampling_whitelist: strin
 	return zeek::val_mgr->True();
 	%}
 
+## Gets the weird sampling global list
+##
+## Returns: Current weird sampling global list
+function Reporter::get_weird_sampling_global_list%(%): string_set
+	%{
+	auto set = zeek::make_intrusive<zeek::TableVal>(zeek::id::string_set);
+	for ( auto el : reporter->GetWeirdSamplingGlobalList() )
+		{
+		auto idx = zeek::make_intrusive<zeek::StringVal>(el);
+		set->Assign(std::move(idx), nullptr);
+		}
+	return set;
+	%}
+
+## Sets the weird sampling global list
+##
+## global_list: New weird sampling rate.
+##
+## Returns: Always true.
+function Reporter::set_weird_sampling_global_list%(weird_sampling_global_list: string_set%) : bool
+	%{
+	auto wl_val = weird_sampling_global_list->AsTableVal();
+	auto wl_table = wl_val->AsTable();
+	std::unordered_set<std::string> global_list_set;
+
+	zeek::detail::HashKey* k;
+	IterCookie* c = wl_table->InitForIteration();
+	TableEntryVal* v;
+
+	while ( (v = wl_table->NextEntry(k, c)) )
+		{
+		auto index = wl_val->RecreateIndex(*k);
+		string key = index->Idx(0)->AsString()->CheckString();
+		global_list_set.emplace(move(key));
+		delete k;
+		}
+	reporter->SetWeirdSamplingGlobalList(global_list_set);
+	return zeek::val_mgr->True();
+	%}
+
 ## Gets the current weird sampling threshold
 ##
 ## Returns: current weird sampling threshold.

testing/btest/Baseline/core.reporter-weird-sampling-global/output

@@ -0,0 +1,6 @@
+10 conn_weird, global_listed_conn_weird
+20 conn_weird, my_conn_weird
+10 flow_weird, global_listed_flow_weird
+20 flow_weird, my_flow_weird
+10 net_weird, global_listed_net_weird
+10 net_weird, my_net_weird

testing/btest/Baseline/plugins.hooks/output

@@ -282,7 +282,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Broker::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, <frame>, (Config::LOG)) -> <no result>
@@ -463,7 +463,7 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])) -> <no result>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])) -> <no result>
-0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
+0.000000   MetaHookPost  CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T])) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::check_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(NetControl::init, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, <frame>, ()) -> <no result>
@@ -535,6 +535,7 @@
 0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::ignore_hosts, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_duration, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_duration, Config::weird_option_change_interval{ if (Weird::sampling_duration == Config::ID) { Reporter::set_weird_sampling_duration(Config::new_value)}return (Config::new_value)}, 5)) -> <no result>
+0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_global_list, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_rate, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_rate, Config::weird_option_change_count{ if (Weird::sampling_threshold == Config::ID) { Reporter::set_weird_sampling_threshold(Config::new_value)}elseif (Weird::sampling_rate == Config::ID) { Reporter::set_weird_sampling_rate(Config::new_value)}return (Config::new_value)}, 5)) -> <no result>
 0.000000   MetaHookPost  CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_threshold, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)) -> <no result>
@@ -1207,7 +1208,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, <frame>, (PacketFilter::LOG, [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Broker::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, <frame>, (Config::LOG))
@@ -1388,7 +1389,7 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, <frame>, (mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql]))
-0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, <frame>, (PacketFilter::LOG, [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(NetControl::check_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(NetControl::init, <null>, ())
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, <frame>, ())
@@ -1460,6 +1461,7 @@
 0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::ignore_hosts, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
 0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_duration, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
 0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_duration, Config::weird_option_change_interval{ if (Weird::sampling_duration == Config::ID) { Reporter::set_weird_sampling_duration(Config::new_value)}return (Config::new_value)}, 5))
+0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_global_list, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
 0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_rate, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
 0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_rate, Config::weird_option_change_count{ if (Weird::sampling_threshold == Config::ID) { Reporter::set_weird_sampling_threshold(Config::new_value)}elseif (Weird::sampling_rate == Config::ID) { Reporter::set_weird_sampling_rate(Config::new_value)}return (Config::new_value)}, 5))
 0.000000   MetaHookPre   CallFunction(Option::set_change_handler, <frame>, (Weird::sampling_threshold, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100))
@@ -2131,7 +2133,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Broker::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Config::LOG)
@@ -2312,7 +2314,7 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=Weird::Info, ev=Weird::log_weird, path=weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=X509::Info, ev=X509::log_x509, path=x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=MySQL::Info, ev=MySQL::log_mysql, path=mysql])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction NetControl::check_plugins()
 0.000000 | HookCallFunction NetControl::init()
 0.000000 | HookCallFunction Notice::want_pp()
@@ -2384,6 +2386,7 @@
 0.000000 | HookCallFunction Option::set_change_handler(Weird::ignore_hosts, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_duration, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_duration, Config::weird_option_change_interval{ if (Weird::sampling_duration == Config::ID) { Reporter::set_weird_sampling_duration(Config::new_value)}return (Config::new_value)}, 5)
+0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_global_list, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_rate, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_rate, Config::weird_option_change_count{ if (Weird::sampling_threshold == Config::ID) { Reporter::set_weird_sampling_threshold(Config::new_value)}elseif (Weird::sampling_rate == Config::ID) { Reporter::set_weird_sampling_rate(Config::new_value)}return (Config::new_value)}, 5)
 0.000000 | HookCallFunction Option::set_change_handler(Weird::sampling_threshold, Config::config_option_changed{ Config::log = (coerce [$ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value)] to Config::Info)if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, Config::log)return (Config::new_value)}, -100)
@@ -2768,7 +2771,7 @@
 0.000000 | HookLoadFile  base<...>/xmpp
 0.000000 | HookLoadFile  base<...>/zeek.bif.zeek
 0.000000 | HookLogInit   packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)}
-0.000000 | HookLogWrite  packet_filter [ts=1598558690.596616, node=zeek, filter=ip or not ip, init=T, success=T]
+0.000000 | HookLogWrite  packet_filter [ts=1599463847.746249, node=zeek, filter=ip or not ip, init=T, success=T]
 0.000000 | HookQueueEvent NetControl::init()
 0.000000 | HookQueueEvent filter_change_tracking()
 0.000000 | HookQueueEvent zeek_init()

testing/btest/core/reporter-weird-sampling-global.zeek

@@ -0,0 +1,59 @@
+# @TEST-EXEC: zeek -C -b -r $TRACES/wlanmon.pcap %INPUT | sort | uniq -c | awk '{print $1, $2, $3}' >output
+# @TEST-EXEC: btest-diff output
+
+# The sampling functionality itself is already tested through other tests.
+# Here, we just set the parameters so that we see a difference between global
+# and non-global, using some smooth numbers not affected by rate or timing.
+redef Weird::sampling_duration = 300days;
+redef Weird::sampling_threshold = 10;
+redef Weird::sampling_rate = 0;
+redef Weird::sampling_global_list = set("global_listed_net_weird",
+                                        "global_listed_flow_weird",
+                                        "global_listed_conn_weird");
+
+event conn_weird(name: string, c: connection, addl: string)
+	{
+	print "conn_weird", name;
+	}
+
+event flow_weird(name: string, src: addr, dst: addr, addl: string)
+	{
+	print "flow_weird", name;
+	}
+
+event net_weird(name: string, addl: string)
+	{
+	print "net_weird", name;
+	}
+
+event gen_weirds(c: connection)
+	{
+	local num = 30;
+
+	while ( num != 0 )
+		{
+		Reporter::net_weird("my_net_weird");
+		Reporter::flow_weird("my_flow_weird", c$id$orig_h, c$id$resp_h);
+		Reporter::conn_weird("my_conn_weird", c);
+
+		Reporter::net_weird("global_listed_net_weird");
+		Reporter::flow_weird("global_listed_flow_weird", c$id$orig_h, c$id$resp_h);
+		Reporter::conn_weird("global_listed_conn_weird", c);
+		--num;
+		}
+	}
+
+global flows: set[addr, addr];
+
+event new_connection(c: connection)
+	{
+	if ( [c$id$orig_h, c$id$resp_h] in flows )
+		return;
+
+	add flows[c$id$orig_h, c$id$resp_h];
+
+	if ( |flows| > 2 )
+		return;
+
+	event gen_weirds(c);
+	}