Merge remote-tracking branch 'origin/master' into topic/johanna/tls-more-data

scripts/base/frameworks/config/input.bro

@@ -60,8 +60,8 @@ event InputConfig::new_value(name: string, source: string, id: string, value: an
 
 function read_config(filename: string)
 	{
-	# Only read the configuration on the manager. The other nodes are being fed from
-	# the manager.
+	# Only read the configuration on the manager. The other nodes are being fed
+	# from the manager.
 	if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER )
 		return;
 

scripts/base/frameworks/config/main.bro

@@ -28,15 +28,15 @@ export {
 	## record as it is sent on to the logging framework.
 	global log_config: event(rec: Info);
 
-	## Broker topic for announcing new configuration value. Sending new_value,
-	## peers can send configuration changes that will be distributed accross
+	## Broker topic for announcing new configuration values. Sending new_value,
+	## peers can send configuration changes that will be distributed across
 	## the entire cluster.
 	const change_topic = "bro/config/change";
 
 	## This function is the config framework layer around the lower-level
 	## :bro:see:`Option::set` call. Config::set_value will set the configuration
 	## value for all nodes in the cluster, no matter where it was called. Note
-	## that `bro:see:`Option::set` does not distribute configuration changes
+	## that :bro:see:`Option::set` does not distribute configuration changes
 	## to other nodes.
 	##
 	## ID: The ID of the option to update.
@@ -45,7 +45,7 @@ export {
 	##
 	## location: Optional parameter detailing where this change originated from.
 	##
-	## Returns: true on success, false when an error ocured.
+	## Returns: true on success, false when an error occurs.
 	global set_value: function(ID: string, val: any, location: string &default = "" &optional): bool;
 }
 
@@ -73,7 +73,7 @@ event Config::cluster_set_option(ID: string, val: any, location: string)
 function set_value(ID: string, val: any, location: string &default = "" &optional): bool
 	{
 	local cache_val: any;
-	# first cache value in case setting it succeeds and we have to store it.
+	# First cache value in case setting it succeeds and we have to store it.
 	if ( Cluster::local_node_type() == Cluster::MANAGER )
 		cache_val = copy(val);
 	# First try setting it locally - abort if not possible.
@@ -151,8 +151,8 @@ event bro_init() &priority=10
 
 	# Limit logging to the manager - everyone else just feeds off it.
 @if ( !Cluster::is_enabled() || Cluster::local_node_type() == Cluster::MANAGER )
-	# Iterate over all existing options and add ourselves as change handlers with
-	# a low priority so that we can log the changes.
+	# Iterate over all existing options and add ourselves as change handlers
+	# with a low priority so that we can log the changes.
 	local gids = global_ids();
 	for ( i in gids )
 		{

scripts/base/frameworks/netcontrol/catch-and-release.bro

@@ -142,7 +142,7 @@ export {
 
 	## If true, catch and release warns if packets of an IP address are still seen after it
 	## should have been blocked.
-	const catch_release_warn_blocked_ip_encountered = F &redef;
+	option catch_release_warn_blocked_ip_encountered = F;
 
 	## Time intervals for which subsequent drops of the same IP take
 	## effect.

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -19,7 +19,7 @@ export {
 	};
 	
 	redef record Info += {
-		## If libGeoIP support is built in, notices can have geographic
+		## If GeoIP support is built in, notices can have geographic
 		## information attached to them.
 		remote_location: geo_location  &log &optional;
 	};

scripts/base/frameworks/notice/main.bro

@@ -201,14 +201,14 @@ export {
 	## Address that emails will be from.
 	##
 	## Note that this is overridden by the BroControl MailFrom option.
-	const mail_from           = "Big Brother <bro@localhost>" &redef;
+	option mail_from           = "Big Brother <bro@localhost>";
 	## Reply-to address used in outbound email.
-	const reply_to            = "" &redef;
+	option reply_to            = "";
 	## Text string prefixed to the subject of all emails sent out.
 	##
 	## Note that this is overridden by the BroControl MailSubjectPrefix
 	## option.
-	const mail_subject_prefix = "[Bro]" &redef;
+	option mail_subject_prefix = "[Bro]";
 	## The maximum amount of time a plugin can delay email from being sent.
 	const max_email_delay     = 15secs &redef;
 

scripts/base/init-bare.bro

@@ -82,6 +82,13 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A table of counts indexed by strings.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type table_string_of_count: table[string] of count;
+
 ## A set of file analyzer tags.
 ##
 ## .. todo:: We need this type definition only for declaring builtin functions
@@ -626,6 +633,17 @@ type BrokerStats: record {
 	num_ids_outgoing: count;
 };
 
+## Statistics about reporter messages and weirds.
+##
+## .. bro:see:: get_reporter_stats
+type ReporterStats: record {
+	## Number of total weirds encountered, before any rate-limiting.
+	weirds: count;
+	## Number of times each individual weird is encountered, before any
+	## rate-limiting is applied.
+	weirds_by_type:	table[string] of count;
+};
+
 ## Deprecated.
 ##
 ## .. todo:: Remove. It's still declared internally but doesn't seem  used anywhere
@@ -4826,6 +4844,35 @@ export {
 	type Cluster::Pool: record {};
 }
 
+module Weird;
+export {
+	## Prevents rate-limiting sampling of any weirds named in the table.
+	const sampling_whitelist: set[string] &redef;
+
+	## How many weirds of a given type to tolerate before sampling begins.
+	## i.e. this many consecutive weirds of a given type will be allowed to
+	## raise events for script-layer handling before being rate-limited.
+	const sampling_threshold = 25 &redef;
+
+	## The rate-limiting sampling rate.  One out of every of this number of
+	## rate-limited weirds of a given type will be allowed to raise events
+	## for further script-layer handling.
+	const sampling_rate = 1000 &redef;
+
+	## How long a weird of a given type is allowed to keep state/counters in
+	## memory.  For "net" weirds an expiration timer starts per weird name when
+	## first initializing its counter.  For "flow" weirds an expiration timer
+	## starts once per src/dst IP pair for the first weird of any name.  For
+	## "conn" weirds, counters and expiration timers are kept for the duration
+	## of the connection for each named weird and reset when necessary.  e.g.
+	## if a "conn" weird by the name of "foo" is seen more than
+	## :bro:see:`Weird::sampling_threshold` times, then an expiration timer
+	## begins for "foo" and upon triggering will reset the counter for "foo"
+	## and unthrottle its rate-limiting until it once again exceeds the
+	## threshold.
+	const sampling_duration = 10min &redef;
+}
+
 module GLOBAL;
 
 ## Seed for hashes computed internally for probabilistic data structures. Using

scripts/base/protocols/dhcp/main.bro

@@ -84,7 +84,7 @@ export {
 	## The maximum amount of time that a transation ID will be watched
 	## for to try and tie messages together into a single DHCP
 	## transaction narrative.
-	const DHCP::max_txid_watch_time = 30secs &redef;
+	option DHCP::max_txid_watch_time = 30secs;
 
 	## This event is used internally to distribute data around clusters
 	## since DHCP doesn't follow the normal "connection" model used by

scripts/base/protocols/dns/main.bro

@@ -113,11 +113,6 @@ export {
 	## DNS message query/transaction ID.
 	type PendingMessages: table[count] of Queue::Queue;
 
-	## The amount of time that DNS queries or replies for a given
-	## query/transaction ID are allowed to be queued while waiting for
-	## a matching reply or query.
-	const pending_msg_expiry_interval = 2min &redef;
-
 	## Give up trying to match pending DNS queries or replies for a given
 	## query/transaction ID once this number of unmatched queries or replies
 	## is reached (this shouldn't happen unless either the DNS server/resolver

scripts/base/protocols/ftp/main.bro

@@ -17,13 +17,13 @@ export {
 	redef enum Log::ID += { LOG };
 
 	## List of commands that should have their command/response pairs logged.
-	const logged_commands = {
+	option logged_commands = {
 		"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT", "PORT", "PASV", "EPRT",
 		"EPSV"
-	} &redef;
+	};
 
 	## User IDs that can be considered "anonymous".
-	const guest_ids = { "anonymous", "ftp", "ftpuser", "guest" } &redef;
+	option guest_ids = { "anonymous", "ftp", "ftpuser", "guest" };
 
 	## This record is to hold a parsed FTP reply code.  For example, for the
 	## 201 status code, the digits would be parsed as: x->2, y->0, z->1.

scripts/base/protocols/http/main.bro

@@ -109,7 +109,7 @@ export {
 	## A list of HTTP methods. Other methods will generate a weird. Note
 	## that the HTTP analyzer will only accept methods consisting solely
 	## of letters ``[A-Za-z]``.
-	const http_methods: set[string] = {
+	option http_methods: set[string] = {
 		"GET", "POST", "HEAD", "OPTIONS",
 		"PUT", "DELETE", "TRACE", "CONNECT",
 		# HTTP methods for distributed authoring:
@@ -117,7 +117,7 @@ export {
 		"COPY", "MOVE", "LOCK", "UNLOCK",
 		"POLL", "REPORT", "SUBSCRIBE", "BMOVE",
 		"SEARCH"
-	} &redef;
+	};
 
 	## Event that can be handled to access the HTTP record as it is sent on
 	## to the logging framework.
@@ -199,7 +199,7 @@ event http_request(c: connection, method: string, original_URI: string,
 	c$http$uri = unescaped_URI;
 
 	if ( method !in http_methods )
-		event conn_weird("unknown_HTTP_method", c, method);
+		Reporter::conn_weird("unknown_HTTP_method", c, method);
 	}
 
 event http_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/krb/main.bro

@@ -48,7 +48,7 @@ export {
 	};
 
 	## The server response error texts which are *not* logged.
-	const ignored_errors: set[string] = {
+	option ignored_errors: set[string] = {
 		# This will significantly increase the noisiness of the log.
 		# However, one attack is to iterate over principals, looking
 		# for ones that don't require preauth, and then performn
@@ -58,7 +58,7 @@ export {
 		# This is a more specific version of NEEDED_PREAUTH that's used
 		# by Windows AD Kerberos.
 		"Need to use PA-ENC-TIMESTAMP/PA-PK-AS-REQ",
-	} &redef;
+	};
 
 	## Event that can be handled to access the KRB record as it is sent on
 	## to the logging framework.

scripts/base/protocols/sip/main.bro

@@ -80,9 +80,9 @@ export {
 	## A list of SIP methods. Other methods will generate a weird. Note
 	## that the SIP analyzer will only accept methods consisting solely
 	## of letters ``[A-Za-z]``.
-	const sip_methods: set[string] = {
+	option sip_methods: set[string] = {
 		"REGISTER", "INVITE", "ACK", "CANCEL", "BYE", "OPTIONS", "NOTIFY", "SUBSCRIBE"
-	} &redef;
+	};
 
 	## Event that can be handled to access the SIP record as it is sent on
 	## to the logging framework.
@@ -168,7 +168,7 @@ event sip_request(c: connection, method: string, original_URI: string, version:
 	c$sip$uri = original_URI;
 
 	if ( method !in sip_methods )
-		event conn_weird("unknown_SIP_method", c, method);
+		Reporter::conn_weird("unknown_SIP_method", c, method);
 	}
 
 event sip_reply(c: connection, version: string, code: count, reason: string) &priority=5

scripts/base/protocols/smb/main.bro

@@ -33,14 +33,14 @@ export {
 	};
 
 	## The file actions which are logged.
-	const logged_file_actions: set[Action] = {
+	option logged_file_actions: set[Action] = {
 		FILE_OPEN,
 		FILE_RENAME,
 		FILE_DELETE,
 
 		PRINT_OPEN,
 		PRINT_CLOSE,
-	} &redef;
+	};
 
 	## This record is for the smb_files.log
 	type FileInfo: record {

scripts/base/protocols/smb/smb1-main.bro

@@ -309,7 +309,7 @@ event smb_pipe_request(c: connection, hdr: SMB1::Header, op_num: count)
 	if ( ! f?$uuid )
 		{
 		# TODO: figure out why this is happening.
-		event conn_weird("smb_pipe_request_missing_uuid", c, "");
+		Reporter::conn_weird("smb_pipe_request_missing_uuid", c, "");
 		return;
 		}
 	local arg = fmt("%s: %s",

scripts/base/protocols/ssl/main.bro

@@ -263,7 +263,7 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		{
 		c$ssl$server_name = names[0];
 		if ( |names| > 1 )
-			event conn_weird("SSL_many_server_names", c, cat(names));
+			Reporter::conn_weird("SSL_many_server_names", c, cat(names));
 		}
 	}
 

scripts/base/utils/exec.bro

@@ -42,9 +42,6 @@ export {
 	## Returns: A record representing the full results from the
 	##          external program execution.
 	global run: function(cmd: Command): Result;
-
-	## The system directory for temporary files.
-	const tmp_dir = "/tmp" &redef;
 }
 
 # Indexed by command uid.

scripts/base/utils/geoip-distance.bro

@@ -1,7 +1,7 @@
 ##! Functions to calculate distance between two locations, based on GeoIP data.
 
 ## Returns the distance between two IP addresses using the haversine formula,
-## based on GeoIP database locations.  Requires Bro to be built with libgeoip.
+## based on GeoIP database locations.  Requires Bro to be built with GeoIP.
 ##
 ## a1: First IP address.
 ##