Merge remote-tracking branch 'origin/master' into topic/seth/metrics-merge

scripts/base/frameworks/input/main.bro

@@ -11,6 +11,24 @@ export {
 	## The default reader mode used. Defaults to `MANUAL`.
 	const default_mode = MANUAL &redef;
 
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;
+
 	## Flag that controls if the input framework accepts records
 	## that contain types that are not supported (at the moment
 	## file and function). If true, the input framework will
@@ -115,7 +133,7 @@ export {
 	global add_event: function(description: Input::EventDescription) : bool;
 
 	## Remove a input stream. Returns true on success and false if the named stream was
-	## not found. 
+	## not found.
 	##
 	## id: string value identifying the stream to be removed
 	global remove: function(id: string) : bool;

scripts/base/frameworks/input/readers/ascii.bro

@@ -7,15 +7,15 @@ module InputAscii;
 export {
 	## Separator between fields.
 	## Please note that the separator has to be exactly one character long
-	const separator = "\t" &redef;
+	const separator = Input::separator &redef;
 
 	## Separator between set elements.
 	## Please note that the separator has to be exactly one character long
-	const set_separator = "," &redef;
+	const set_separator = Input::set_separator &redef;
 
 	## String to use for empty fields.
-	const empty_field = "(empty)" &redef;
+	const empty_field = Input::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Input::unset_field &redef;
 }

scripts/base/frameworks/logging/main.bro

@@ -17,6 +17,23 @@ export {
 	## anything else.
 	const default_writer = WRITER_ASCII &redef;
 
+	## Default separator between fields for logwriters.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious. 
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;	
+
 	## Type defining the content of a logging stream.
 	type Stream: record {
 		## A record type defining the log's columns.

scripts/base/frameworks/logging/writers/ascii.bro

@@ -28,17 +28,17 @@ export {
 	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
-	const separator = "\t" &redef;
+	const separator = Log::separator &redef;
 
 	## Separator between set elements.
-	const set_separator = "," &redef;
+	const set_separator = Log::set_separator &redef;
 
 	## String to use for empty fields. This should be different from
         ## *unset_field* to make the output non-ambigious. 
-	const empty_field = "(empty)" &redef;
+	const empty_field = Log::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Log::unset_field &redef;
 }
 
 # Default function to postprocess a rotated ASCII log file. It moves the rotated

scripts/base/frameworks/reporter/main.bro

@@ -1,10 +1,15 @@
 ##! This framework is intended to create an output and filtering path for
 ##! internal messages/warnings/errors.  It should typically be loaded to
-##! avoid Bro spewing internal messages to standard error and instead log
-##! them to a file in a standard way.  Note that this framework deals with
-##! the handling of internally-generated reporter messages, for the
-##! interface into actually creating reporter messages from the scripting
-##! layer, use the built-in functions in :doc:`/scripts/base/reporter.bif`.
+##! log such messages to a file in a standard way.  For the options to
+##! toggle whether messages are additionally written to STDERR, see
+##! :bro:see:`Reporter::info_to_stderr`,
+##! :bro:see:`Reporter::warnings_to_stderr`, and
+##! :bro:see:`Reporter::errors_to_stderr`.
+##!
+##! Note that this framework deals with the handling of internally generated
+##! reporter messages, for the interface in to actually creating interface
+##! into actually creating reporter messages from the scripting layer, use
+##! the built-in functions in :doc:`/scripts/base/reporter.bif`.
 
 module Reporter;
 
@@ -36,26 +41,11 @@ export {
 		## Not all reporter messages will have locations in them though.
 		location: string &log &optional;
 	};
-
-	## Tunable for sending reporter warning messages to STDERR.  The option to
-	## turn it off is presented here in case Bro is being run by some
-	## external harness and shouldn't output anything to the console.
-	const warnings_to_stderr = T &redef;
-
-	## Tunable for sending reporter error messages to STDERR.  The option to
-	## turn it off is presented here in case Bro is being run by some
-	## external harness and shouldn't output anything to the console.
-	const errors_to_stderr = T &redef;
 }
 
-global stderr: file;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(Reporter::LOG, [$columns=Info]);
-
-	if ( errors_to_stderr || warnings_to_stderr )
-		stderr = open("/dev/stderr");
 	}
 
 event reporter_info(t: time, msg: string, location: string) &priority=-5
@@ -65,26 +55,10 @@ event reporter_info(t: time, msg: string, location: string) &priority=-5
 
 event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
-	if ( warnings_to_stderr )
-		{
-		if ( t > double_to_time(0.0) )
-			print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
-		else
-			print stderr, fmt("WARNING: %s (%s)", msg, location);
-		}
-
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
 event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
-	if ( errors_to_stderr )
-		{
-		if ( t > double_to_time(0.0) )
-			print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
-		else
-			print stderr, fmt("ERROR: %s (%s)", msg, location);
-		}
-
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}

scripts/base/frameworks/signatures/main.bro

@@ -148,7 +148,7 @@ function has_signature_matched(id: string, orig: addr, resp: addr): bool
 event sig_summary(orig: addr, id: string, msg: string)
 	{
 	NOTICE([$note=Signature_Summary, $src=orig,
-	        $filename=id, $msg=fmt("%s: %s", orig, msg),
+	        $msg=fmt("%s: %s", orig, msg),
 	        $n=count_per_orig[orig,id] ]);
 	}
 
@@ -209,7 +209,6 @@ event signature_match(state: signature_state, msg: string, data: string)
 			{
 			NOTICE([$note=Count_Signature, $conn=state$conn,
 				   $msg=msg,
-				   $filename=sig_id,
 				   $n=count_per_resp[dst,sig_id],
 				   $sub=fmt("%d matches of signature %s on host %s",
 						count_per_resp[dst,sig_id],
@@ -240,7 +239,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 	if ( notice )
 		NOTICE([$note=Sensitive_Signature,
 		        $conn=state$conn, $src=src_addr,
-		        $dst=dst_addr, $filename=sig_id, $msg=fmt("%s: %s", src_addr, msg),
+		        $dst=dst_addr, $msg=fmt("%s: %s", src_addr, msg),
 		        $sub=data]);
 	
 	if ( action == SIG_FILE_BUT_NO_SCAN || action == SIG_SUMMARY )
@@ -274,7 +273,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 		     $src_addr=orig, $sig_id=sig_id, $event_msg=msg,
 		     $host_count=hcount, $sub_msg=horz_scan_msg]);
 
-		NOTICE([$note=Multiple_Sig_Responders, $src=orig, $filename=sig_id,
+		NOTICE([$note=Multiple_Sig_Responders, $src=orig,
 			$msg=msg, $n=hcount, $sub=horz_scan_msg]);
 
 		last_hthresh[orig] = hcount;
@@ -295,7 +294,6 @@ event signature_match(state: signature_state, msg: string, data: string)
 			 $sub_msg=vert_scan_msg]);
 
 		NOTICE([$note=Multiple_Signatures, $src=orig, $dst=resp,
-			$filename=sig_id,
 			$msg=fmt("%s different signatures triggered", vcount),
 			$n=vcount, $sub=vert_scan_msg]);
 

scripts/base/init-bare.bro

@@ -240,7 +240,7 @@ export {
 		## The 4-tuple of the encapsulating "connection". In case of an IP-in-IP
 		## tunnel the ports will be set to 0. The direction (i.e., orig and
 		## resp) are set according to the first tunneled packet seen
-		## and not according to the side that established the tunnel. 
+		## and not according to the side that established the tunnel.
 		cid: conn_id;
 		## The type of tunnel.
 		tunnel_type: Tunnel::Type;
@@ -2507,7 +2507,7 @@ type ModbusHeaders: record {
 
 module SOCKS;
 export {
-	## This record is for a SOCKS client or server to provide either a 
+	## This record is for a SOCKS client or server to provide either a
 	## name or an address to represent a desired or established connection.
 	type Address: record {
 		host: addr   &optional;
@@ -2608,6 +2608,15 @@ const gap_report_freq = 1.0 sec &redef;
 ## .. bro:see:: content_gap gap_report partial_connection
 const report_gaps_for_partial = F &redef;
 
+## Flag to prevent Bro from exiting automatically when input is exhausted.
+## Normally Bro terminates when all packets sources have gone dry
+## and  communication isn't enabled. If this flag is set, Bro's main loop will
+## instead keep idleing until :bro:see::`terminate` is explicitly called.
+##
+## This is mainly for testing purposes when termination behaviour needs to be
+## controlled for reproducing results.
+const exit_only_after_terminate = F &redef;
+
 ## The CA certificate file to authorize remote Bros/Broccolis.
 ##
 ## .. bro:see:: ssl_private_key ssl_passphrase
@@ -2857,6 +2866,25 @@ export {
 } # end export
 module GLOBAL;
 
+module Reporter;
+export {
+	## Tunable for sending reporter info messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const info_to_stderr = T &redef;
+
+	## Tunable for sending reporter warning messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
+
+	## Tunable for sending reporter error messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const errors_to_stderr = T &redef;
+}
+module GLOBAL;
+
 ## Number of bytes per packet to capture from live interfaces.
 const snaplen = 8192 &redef;
 

scripts/base/protocols/http/file-hash.bro

@@ -73,7 +73,7 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
 		delete c$http$md5_handle;
 		
 		NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url),
-		        $sub=c$http$md5, $conn=c, $URL=url]);
+		        $sub=c$http$md5, $conn=c]);
 		}
 	}
 

scripts/base/protocols/http/file-ident.bro

@@ -68,9 +68,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori
 		local message = fmt("%s %s %s", msg, c$http$method, url);
 		NOTICE([$note=Incorrect_File_Type,
 		        $msg=message,
-		        $conn=c,
-		        $method=c$http$method,
-		        $URL=url]);
+		        $conn=c]);
 		}
 	}
 

scripts/base/protocols/socks/main.bro

@@ -67,7 +67,7 @@ event socks_request(c: connection, version: count, request_type: count,
 	# proxied connection.  We treat this as a singular "tunnel".
 	local cid = copy(c$id);
 	cid$orig_p = 0/tcp;
-	Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS, $payload_proxy=T]);
+	Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS]);
 	}
 
 event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=5

scripts/base/protocols/ssl/__load__.bro

@@ -1,3 +1,3 @@
 @load ./consts
 @load ./main
-@load ./mozilla-ca-list
+@load ./mozilla-ca-list

scripts/base/protocols/ssl/main.bro

@@ -72,6 +72,17 @@ export {
 	## utility.
 	const openssl_util = "openssl" &redef;
 
+	## The maximum amount of time a script can delay records from being logged.
+	const max_log_delay = 15secs &redef;
+
+	## Delays an SSL record for a specific token: the record will not be logged
+	## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses.
+	global delay_log: function(info: Info, token: string);
+
+	## Undelays an SSL record for a previously inserted token, allowing the
+	## record to be logged.
+	global undelay_log: function(info: Info, token: string);
+
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
@@ -81,6 +92,13 @@ redef record connection += {
 	ssl: Info &optional;
 };
 
+redef record Info += {
+		# Adding a string "token" to this set will cause the SSL script
+		# to delay logging the record until either the token has been removed or
+		# the record has been delayed for :bro:id:`SSL::max_log_delay`.
+		delay_tokens: set[string] &optional;
+};
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
@@ -115,6 +133,13 @@ redef likely_server_ports += {
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
 
+# A queue that buffers log records.
+global log_delay_queue: table[count] of Info;
+# The top queue index where records are added.
+global log_delay_queue_head = 0;
+# The bottom queue index that points to the next record to be flushed.
+global log_delay_queue_tail = 0;
+
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
@@ -122,12 +147,65 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
+function delay_log(info: Info, token: string)
+	{
+	info$delay_tokens = set();
+	add info$delay_tokens[token];
+
+	log_delay_queue[log_delay_queue_head] = info;
+	++log_delay_queue_head;
+	}
+
+function undelay_log(info: Info, token: string)
+	{
+	if ( token in info$delay_tokens )
+		delete info$delay_tokens[token];
+	}
+
+global log_record: function(info: Info);
+
+event delay_logging(info: Info)
+	{
+	log_record(info);
+	}
+
+function log_record(info: Info)
+	{
+	if ( ! info?$delay_tokens || |info$delay_tokens| == 0 )
+		{
+		Log::write(SSL::LOG, info);
+		}
+	else
+		{
+		for ( unused_index in log_delay_queue )
+			{
+			if ( log_delay_queue_head == log_delay_queue_tail )
+				return;
+			if ( |log_delay_queue[log_delay_queue_tail]$delay_tokens| > 0 )
+				{
+				if ( info$ts + max_log_delay > network_time() )
+					{
+					schedule 1sec { delay_logging(info) };
+					return;
+					}
+				else
+					{
+					Reporter::info(fmt("SSL delay tokens not released in time (%s)",
+					                   info$delay_tokens));
+					}
+				}
+			Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
+			delete log_delay_queue[log_delay_queue_tail];
+			++log_delay_queue_tail;
+			}
+		}
+	}
+
 function finish(c: connection)
 	{
-	Log::write(SSL::LOG, c$ssl);
+	log_record(c$ssl);
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
-	delete c$ssl;
 	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
@@ -228,3 +306,15 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	if ( c?$ssl )
 		finish(c);
 	}
+
+event bro_done()
+	{
+	if ( |log_delay_queue| == 0 )
+		return;
+	for ( unused_index in log_delay_queue )
+		{
+		Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
+		delete log_delay_queue[log_delay_queue_tail];
+		++log_delay_queue_tail;
+		}
+	}