Merge remote-tracking branch 'origin/topic/jsiwek/empty-lines'

* origin/topic/jsiwek/empty-lines:
  Add 'smtp_excessive_pending_cmds' weird
  Fix SMTP command string comparisons
  Improve handling of empty lines in several text protocol analyzers
  Add rate-limiting sampling mechanism for weird events
  Teach timestamp canonifier about timestamps before ~2001

scripts/base/init-bare.bro

@@ -82,6 +82,13 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A table of counts indexed by strings.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type table_string_of_count: table[string] of count;
+
 ## A set of file analyzer tags.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -626,6 +633,17 @@ type BrokerStats: record {
 	num_ids_outgoing: count;
 };
 
+## Statistics about reporter messages and weirds.
+##
+## .. bro:see:: get_reporter_stats
+type ReporterStats: record {
+	## Number of total weirds encountered, before any rate-limiting.
+	weirds: count;
+	## Number of times each individual weird is encountered, before any
+	## rate-limiting is applied.
+	weirds_by_type:	table[string] of count;
+};
+
 ## Deprecated.
 ##
 ## .. todo:: Remove. It's still declared internally but doesn't seem  used anywhere
@@ -4826,6 +4844,35 @@ export {
 	type Cluster::Pool: record {};
 }
 
+module Weird;
+export {
+	## Prevents rate-limiting sampling of any weirds named in the table.
+	const sampling_whitelist: set[string] &redef;
+
+	## How many weirds of a given type to tolerate before sampling begins.
+	## i.e. this many consecutive weirds of a given type will be allowed to
+	## raise events for script-layer handling before being rate-limited.
+	const sampling_threshold = 25 &redef;
+
+	## The rate-limiting sampling rate.  One out of every of this number of
+	## rate-limited weirds of a given type will be allowed to raise events
+	## for further script-layer handling.
+	const sampling_rate = 1000 &redef;
+
+	## How long a weird of a given type is allowed to keep state/counters in
+	## memory.  For "net" weirds an expiration timer starts per weird name when
+	## first initializing its counter.  For "flow" weirds an expiration timer
+	## starts once per src/dst IP pair for the first weird of any name.  For
+	## "conn" weirds, counters and expiration timers are kept for the duration
+	## of the connection for each named weird and reset when necessary.  e.g.
+	## if a "conn" weird by the name of "foo" is seen more than
+	## :bro:see:`Weird::sampling_threshold` times, then an expiration timer
+	## begins for "foo" and upon triggering will reset the counter for "foo"
+	## and unthrottle its rate-limiting until it once again exceeds the
+	## threshold.
+	const sampling_duration = 10min &redef;
+}
+
 module GLOBAL;
 
 ## Seed for hashes computed internally for probabilistic data structures. Using

scripts/base/protocols/http/main.bro

@@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string,
 	c$http$uri = unescaped_URI;
 
 	if ( method !in http_methods )
-		event conn_weird("unknown_HTTP_method", c, method);
+		Reporter::conn_weird("unknown_HTTP_method", c, method);
 	}
 
 event http_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/sip/main.bro

@@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version:
 	c$sip$uri = original_URI;
 
 	if ( method !in sip_methods )
-		event conn_weird("unknown_SIP_method", c, method);
+		Reporter::conn_weird("unknown_SIP_method", c, method);
 	}
 
 event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/smb/smb1-main.bro

@@ -309,7 +309,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count)
 	if ( ! f?$uuid )
 		{
 		# TODO: figure out why this is happening.
-		event conn_weird("smb_pipe_request_missing_uuid", c, "");
+		Reporter::conn_weird("smb_pipe_request_missing_uuid", c, "");
 		return;
 		}
 	local arg = fmt("%s: %s",

scripts/base/protocols/ssl/main.bro

@@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		{
 		c$ssl$server_name = names[0];
 		if ( |names| > 1 )
-			event conn_weird("SSL_many_server_names", c, cat(names));
+			Reporter::conn_weird("SSL_many_server_names", c, cat(names));
 		}
 	}
 

scripts/policy/misc/weird-stats.bro

@@ -0,0 +1,100 @@
+##! Log weird statistics.
+
+@load base/frameworks/sumstats
+@load base/frameworks/cluster
+
+module WeirdStats;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	## How often stats are reported.
+	const weird_stat_interval = 15min &redef;
+
+	type Info: record {
+		## Timestamp for the measurement.
+		ts: time &log;
+		## Name of the weird.
+		name: string &log;
+		## Number of times weird was seen since the last stats interval.
+		num_seen: count &log;
+	};
+
+	global log_weird_stats: event(rec: Info);
+}
+
+global this_epoch_weirds: table[string] of double;
+global last_epoch_weirds: table[string] of double;
+
+function weird_epoch_results(ts: time, key: SumStats::Key, result: SumStats::Result)
+	{
+	this_epoch_weirds[key$str]=result["weirds.encountered"]$sum;
+	}
+
+function weird_epoch_finished(ts: time)
+	{
+	for ( n in this_epoch_weirds )
+		{
+		local last_count: double = 0.0;
+
+		if ( n in last_epoch_weirds )
+			last_count = last_epoch_weirds[n];
+
+		local num_seen: double = this_epoch_weirds[n] - last_count;
+
+		if ( num_seen > 0.0 )
+			Log::write(LOG, Info($ts = ts, $name = n,
+			                     $num_seen = double_to_count(num_seen)));
+		}
+
+	last_epoch_weirds = this_epoch_weirds;
+	this_epoch_weirds = table();
+	}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(WeirdStats::LOG,
+	                   [$columns = Info, $ev = log_weird_stats,
+	                    $path="weird_stats"]);
+	local r1 = SumStats::Reducer($stream = "weirds.encountered",
+	                             $apply = set(SumStats::SUM));
+	SumStats::create([$name = "weirds.statistics",
+	                  $epoch = weird_stat_interval, $reducers = set(r1),
+	                  $epoch_result = weird_epoch_results,
+	                  $epoch_finished = weird_epoch_finished]);
+	}
+
+module SumStats;
+
+function observe_weird_stats()
+	{
+	local rs = get_reporter_stats();
+
+	for ( n in rs$weirds_by_type )
+		SumStats::observe("weirds.encountered", SumStats::Key($str = n),
+		                  SumStats::Observation($dbl=rs$weirds_by_type[n]+0.0));
+	}
+
+@if ( Cluster::is_enabled() )
+
+# I'm not sure if this is a hack or not: the manager will generate this
+# event at the end of its epoch so workers can handle it just in time to
+# generate the necessary stats.  Alternative may be workers generating the
+# stats individually/proactively in their own finish_epoch, but that may be
+# less synchronized?
+event SumStats::cluster_ss_request(uid: string, ss_name: string, cleanup: bool) &priority=10
+	{
+	observe_weird_stats();
+	}
+
+@else
+
+event SumStats::finish_epoch(ss: SumStat) &priority=10
+	{
+	if ( ss$name != "weirds.statistics" )
+		return;
+
+	observe_weird_stats();
+	}
+
+@endif

scripts/test-all-policy.bro

@@ -54,6 +54,7 @@
 @load misc/profiling.bro
 @load misc/scan.bro
 @load misc/stats.bro
+@load misc/weird-stats.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro