Merge remote-tracking branch 'origin/master' into topic/seth/metrics-merge

CHANGES

@@ -1,4 +1,88 @@
 
+2.1-328 | 2013-02-05 01:34:29 -0500
+
+  * New script to query the ICSI Certificate Notary 
+    (http://notary.icsi.berkeley.edu/) over DNS and add information
+    to the SSL log at runtime. (Matthias Vallentin) 
+
+  * Add delayed logging to SSL base scripts. (Matthias Vallentin)
+
+2.1-319 | 2013-02-04 09:45:34 -0800
+
+  * Update input tests to use exit_only_after_terminate. (Bernhard
+    Amann)
+
+  * New option exit_only_after_terminate to prevent Bro from exiting.
+    If set, the main loop won't terminate before somebody calls
+    terminate(). (Robin Sommer)
+
+2.1-311 | 2013-02-01 08:03:01 -0800
+
+  * Updating submodule(s).
+
+2.1-310 | 2013-01-30 20:09:27 -0800
+
+  * Add an error for record coercions that would orphan a field. (Jon
+    Siwek)
+
+  * Fixing several scripts where a field in an inlined record was
+    never removed after a code refactor. (Jon Siwek)
+
+2.1-307 | 2013-01-25 13:50:57 -0800
+
+  * Fix runaway reference counting bug in record coercion. (Jon Siwek)
+
+  * Fix memory leak in some reporter messaging cases. (Jon Siwek)
+
+2.1-304 | 2013-01-23 19:43:27 -0800
+
+  * Making a test portable. (Robin Sommer)
+
+2.1-302 | 2013-01-23 16:17:29 -0800
+
+  * Refactoring ASCII formatting/parsing from loggers/readers into a
+    separate AsciiFormatter class. (Bernhard Amann)
+
+  * Fix uninitialized locals in event/hook handlers from having a
+    value. Addresses #932. (Jon Siwek)
+
+  * Add a null value check in CompositeHash::ComputeHash. Addresses
+    #930. (Jon Siwek)
+
+  * Change reporter messages to more reliably print to stderr.
+    Addressed #930 (and revisits #836). (Jon Siwek)
+
+  * Changing test=suite's btest call to use "-j" instead of "-j 5".
+    (Robin Sommer)
+    
+  * Require "case" blocks to end with either "break", "return", or a
+    new "fallthrough" statement that passes control on to the
+    subsequent case. This gives us the best mix of safety,
+    readability, and flexibility. Addresses #754. (Jon Siwek)
+
+2.1-279 | 2013-01-18 17:18:22 -0800
+
+  * Revert "Trick for parallelizing input framework unit tests." The
+    old way of doing the tests seems more reliable for now. (Jon
+    Siwek)
+
+  * Fixing variable size issues with http response code in
+    ElasticSearch writer. (Gilbert Clark)
+
+  * Removing unused class member. (Robin Sommer)
+    
+  * Add opaque type-ignoring for the accept_unsupported_types input
+    framework option. (Bernhard Amann)
+
+2.1-271 | 2013-01-08 10:18:57 -0800
+
+  * Change substring index notation to use a colon.  String slice
+    notation is now written as `s[1:2]`. Addresses #422. (Jon Siwek)
+
+2.1-268 | 2013-01-07 09:43:44 -0800
+
+  * Fix memory leak in OpaqueType::DoUnserialize. (Jon Siwek)
+
 2.1-265 | 2012-12-20 17:38:42 -0800
 
   * Add array-style index accessor for strings.  Addresses #422. (Jon

VERSION

@@ -1 +1 @@
-2.1-265
+2.1-328

aux/broccoli

@@ -1 +1 @@
-Subproject commit 073404dd29dc6e90ff0e4eb8bc836f8adbf3931e
+Subproject commit c1ba9b44c4815c61c54c968f462ec5b0865e5990

aux/btest

@@ -1 +1 @@
-Subproject commit d9c747f268ca50275f3bc2e1581464b599d5a3ea
+Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb

doc/scripts/DocSourcesList.cmake

@@ -169,6 +169,7 @@ rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
 rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
 rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
+rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)

scripts/base/frameworks/input/main.bro

@@ -11,6 +11,24 @@ export {
 	## The default reader mode used. Defaults to `MANUAL`.
 	const default_mode = MANUAL &redef;
 
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;
+
 	## Flag that controls if the input framework accepts records
 	## that contain types that are not supported (at the moment
 	## file and function). If true, the input framework will

scripts/base/frameworks/input/readers/ascii.bro

@@ -7,15 +7,15 @@ module InputAscii;
 export {
 	## Separator between fields.
 	## Please note that the separator has to be exactly one character long
-	const separator = "\t" &redef;
+	const separator = Input::separator &redef;
 
 	## Separator between set elements.
 	## Please note that the separator has to be exactly one character long
-	const set_separator = "," &redef;
+	const set_separator = Input::set_separator &redef;
 
 	## String to use for empty fields.
-	const empty_field = "(empty)" &redef;
+	const empty_field = Input::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Input::unset_field &redef;
 }

scripts/base/frameworks/logging/main.bro

@@ -17,6 +17,23 @@ export {
 	## anything else.
 	const default_writer = WRITER_ASCII &redef;
 
+	## Default separator between fields for logwriters.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious. 
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;	
+
 	## Type defining the content of a logging stream.
 	type Stream: record {
 		## A record type defining the log's columns.

scripts/base/frameworks/logging/writers/ascii.bro

@@ -28,17 +28,17 @@ export {
 	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
-	const separator = "\t" &redef;
+	const separator = Log::separator &redef;
 
 	## Separator between set elements.
-	const set_separator = "," &redef;
+	const set_separator = Log::set_separator &redef;
 
 	## String to use for empty fields. This should be different from
         ## *unset_field* to make the output non-ambigious. 
-	const empty_field = "(empty)" &redef;
+	const empty_field = Log::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Log::unset_field &redef;
 }
 
 # Default function to postprocess a rotated ASCII log file. It moves the rotated

scripts/base/frameworks/reporter/main.bro

@@ -1,10 +1,15 @@
 ##! This framework is intended to create an output and filtering path for
 ##! internal messages/warnings/errors.  It should typically be loaded to
-##! avoid Bro spewing internal messages to standard error and instead log
+##! log such messages to a file in a standard way.  For the options to
-##! them to a file in a standard way.  Note that this framework deals with
+##! toggle whether messages are additionally written to STDERR, see
-##! the handling of internally-generated reporter messages, for the
+##! :bro:see:`Reporter::info_to_stderr`,
-##! interface into actually creating reporter messages from the scripting
+##! :bro:see:`Reporter::warnings_to_stderr`, and
-##! layer, use the built-in functions in :doc:`/scripts/base/reporter.bif`.
+##! :bro:see:`Reporter::errors_to_stderr`.
+##!
+##! Note that this framework deals with the handling of internally generated
+##! reporter messages, for the interface in to actually creating interface
+##! into actually creating reporter messages from the scripting layer, use
+##! the built-in functions in :doc:`/scripts/base/reporter.bif`.
 
 module Reporter;
 
@@ -36,26 +41,11 @@ export {
 		## Not all reporter messages will have locations in them though.
 		location: string &log &optional;
 	};
-
-	## Tunable for sending reporter warning messages to STDERR.  The option to
-	## turn it off is presented here in case Bro is being run by some
-	## external harness and shouldn't output anything to the console.
-	const warnings_to_stderr = T &redef;
-
-	## Tunable for sending reporter error messages to STDERR.  The option to
-	## turn it off is presented here in case Bro is being run by some
-	## external harness and shouldn't output anything to the console.
-	const errors_to_stderr = T &redef;
 }
 
-global stderr: file;
-
 event bro_init() &priority=5
 	{
 	Log::create_stream(Reporter::LOG, [$columns=Info]);
-
-	if ( errors_to_stderr || warnings_to_stderr )
-		stderr = open("/dev/stderr");
 	}
 
 event reporter_info(t: time, msg: string, location: string) &priority=-5
@@ -65,26 +55,10 @@ event reporter_info(t: time, msg: string, location: string) &priority=-5
 
 event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
-	if ( warnings_to_stderr )
-		{
-		if ( t > double_to_time(0.0) )
-			print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
-		else
-			print stderr, fmt("WARNING: %s (%s)", msg, location);
-		}
-
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
 event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
-	if ( errors_to_stderr )
-		{
-		if ( t > double_to_time(0.0) )
-			print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
-		else
-			print stderr, fmt("ERROR: %s (%s)", msg, location);
-		}
-
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}

scripts/base/frameworks/signatures/main.bro

@@ -148,7 +148,7 @@ function has_signature_matched(id: string, orig: addr, resp: addr): bool
 event sig_summary(orig: addr, id: string, msg: string)
 	{
 	NOTICE([$note=Signature_Summary, $src=orig,
-	        $filename=id, $msg=fmt("%s: %s", orig, msg),
+	        $msg=fmt("%s: %s", orig, msg),
 	        $n=count_per_orig[orig,id] ]);
 	}
 
@@ -209,7 +209,6 @@ event signature_match(state: signature_state, msg: string, data: string)
 			{
 			NOTICE([$note=Count_Signature, $conn=state$conn,
 				   $msg=msg,
-				   $filename=sig_id,
 				   $n=count_per_resp[dst,sig_id],
 				   $sub=fmt("%d matches of signature %s on host %s",
 						count_per_resp[dst,sig_id],
@@ -240,7 +239,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 	if ( notice )
 		NOTICE([$note=Sensitive_Signature,
 		        $conn=state$conn, $src=src_addr,
-		        $dst=dst_addr, $filename=sig_id, $msg=fmt("%s: %s", src_addr, msg),
+		        $dst=dst_addr, $msg=fmt("%s: %s", src_addr, msg),
 		        $sub=data]);
 	
 	if ( action == SIG_FILE_BUT_NO_SCAN || action == SIG_SUMMARY )
@@ -274,7 +273,7 @@ event signature_match(state: signature_state, msg: string, data: string)
 		     $src_addr=orig, $sig_id=sig_id, $event_msg=msg,
 		     $host_count=hcount, $sub_msg=horz_scan_msg]);
 
-		NOTICE([$note=Multiple_Sig_Responders, $src=orig, $filename=sig_id,
+		NOTICE([$note=Multiple_Sig_Responders, $src=orig,
 			$msg=msg, $n=hcount, $sub=horz_scan_msg]);
 
 		last_hthresh[orig] = hcount;
@@ -295,7 +294,6 @@ event signature_match(state: signature_state, msg: string, data: string)
 			 $sub_msg=vert_scan_msg]);
 
 		NOTICE([$note=Multiple_Signatures, $src=orig, $dst=resp,
-			$filename=sig_id,
 			$msg=fmt("%s different signatures triggered", vcount),
 			$n=vcount, $sub=vert_scan_msg]);
 

scripts/base/init-bare.bro

@@ -2608,6 +2608,15 @@ const gap_report_freq = 1.0 sec &redef;
 ## .. bro:see:: content_gap gap_report partial_connection
 const report_gaps_for_partial = F &redef;
 
+## Flag to prevent Bro from exiting automatically when input is exhausted.
+## Normally Bro terminates when all packets sources have gone dry
+## and  communication isn't enabled. If this flag is set, Bro's main loop will
+## instead keep idleing until :bro:see::`terminate` is explicitly called.
+##
+## This is mainly for testing purposes when termination behaviour needs to be
+## controlled for reproducing results.
+const exit_only_after_terminate = F &redef;
+
 ## The CA certificate file to authorize remote Bros/Broccolis.
 ##
 ## .. bro:see:: ssl_private_key ssl_passphrase
@@ -2857,6 +2866,25 @@ export {
 } # end export
 module GLOBAL;
 
+module Reporter;
+export {
+	## Tunable for sending reporter info messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const info_to_stderr = T &redef;
+
+	## Tunable for sending reporter warning messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
+
+	## Tunable for sending reporter error messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const errors_to_stderr = T &redef;
+}
+module GLOBAL;
+
 ## Number of bytes per packet to capture from live interfaces.
 const snaplen = 8192 &redef;
 

scripts/base/protocols/http/file-hash.bro

@@ -73,7 +73,7 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &
 		delete c$http$md5_handle;
 		
 		NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url),
-		        $sub=c$http$md5, $conn=c, $URL=url]);
+		        $sub=c$http$md5, $conn=c]);
 		}
 	}
 

scripts/base/protocols/http/file-ident.bro

@@ -68,9 +68,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori
 		local message = fmt("%s %s %s", msg, c$http$method, url);
 		NOTICE([$note=Incorrect_File_Type,
 		        $msg=message,
-		        $conn=c,
+		        $conn=c]);
-		        $method=c$http$method,
-		        $URL=url]);
 		}
 	}
 

scripts/base/protocols/socks/main.bro

@@ -67,7 +67,7 @@ event socks_request(c: connection, version: count, request_type: count,
 	# proxied connection.  We treat this as a singular "tunnel".
 	local cid = copy(c$id);
 	cid$orig_p = 0/tcp;
-	Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS, $payload_proxy=T]);
+	Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS]);
 	}
 
 event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=5

scripts/base/protocols/ssl/main.bro

@@ -72,6 +72,17 @@ export {
 	## utility.
 	const openssl_util = "openssl" &redef;
 
+	## The maximum amount of time a script can delay records from being logged.
+	const max_log_delay = 15secs &redef;
+
+	## Delays an SSL record for a specific token: the record will not be logged
+	## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses.
+	global delay_log: function(info: Info, token: string);
+
+	## Undelays an SSL record for a previously inserted token, allowing the
+	## record to be logged.
+	global undelay_log: function(info: Info, token: string);
+
 	## Event that can be handled to access the SSL
 	## record as it is sent on to the logging framework.
 	global log_ssl: event(rec: Info);
@@ -81,6 +92,13 @@ redef record connection += {
 	ssl: Info &optional;
 };
 
+redef record Info += {
+		# Adding a string "token" to this set will cause the SSL script
+		# to delay logging the record until either the token has been removed or
+		# the record has been delayed for :bro:id:`SSL::max_log_delay`.
+		delay_tokens: set[string] &optional;
+};
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]);
@@ -115,6 +133,13 @@ redef likely_server_ports += {
 	989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp
 };
 
+# A queue that buffers log records.
+global log_delay_queue: table[count] of Info;
+# The top queue index where records are added.
+global log_delay_queue_head = 0;
+# The bottom queue index that points to the next record to be flushed.
+global log_delay_queue_tail = 0;
+
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
@@ -122,12 +147,65 @@ function set_session(c: connection)
 		         $client_cert_chain=vector()];
 	}
 
+function delay_log(info: Info, token: string)
+	{
+	info$delay_tokens = set();
+	add info$delay_tokens[token];
+
+	log_delay_queue[log_delay_queue_head] = info;
+	++log_delay_queue_head;
+	}
+
+function undelay_log(info: Info, token: string)
+	{
+	if ( token in info$delay_tokens )
+		delete info$delay_tokens[token];
+	}
+
+global log_record: function(info: Info);
+
+event delay_logging(info: Info)
+	{
+	log_record(info);
+	}
+
+function log_record(info: Info)
+	{
+	if ( ! info?$delay_tokens || |info$delay_tokens| == 0 )
+		{
+		Log::write(SSL::LOG, info);
+		}
+	else
+		{
+		for ( unused_index in log_delay_queue )
+			{
+			if ( log_delay_queue_head == log_delay_queue_tail )
+				return;
+			if ( |log_delay_queue[log_delay_queue_tail]$delay_tokens| > 0 )
+				{
+				if ( info$ts + max_log_delay > network_time() )
+					{
+					schedule 1sec { delay_logging(info) };
+					return;
+					}
+				else
+					{
+					Reporter::info(fmt("SSL delay tokens not released in time (%s)",
+					                   info$delay_tokens));
+					}
+				}
+			Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
+			delete log_delay_queue[log_delay_queue_tail];
+			++log_delay_queue_tail;
+			}
+		}
+	}
+
 function finish(c: connection)
 	{
-	Log::write(SSL::LOG, c$ssl);
+	log_record(c$ssl);
 	if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id )
 		disable_analyzer(c$id, c$ssl$analyzer_id);
-	delete c$ssl;
 	}
 
 event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5
@@ -228,3 +306,15 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	if ( c?$ssl )
 		finish(c);
 	}
+
+event bro_done()
+	{
+	if ( |log_delay_queue| == 0 )
+		return;
+	for ( unused_index in log_delay_queue )
+		{
+		Log::write(SSL::LOG, log_delay_queue[log_delay_queue_tail]);
+		delete log_delay_queue[log_delay_queue_tail];
+		++log_delay_queue_tail;
+		}
+	}

scripts/policy/protocols/http/detect-MHR.bro

@@ -37,7 +37,7 @@ event log_http(rec: HTTP::Info)
 				local url = HTTP::build_url_http(rec);
 				local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url);
 				NOTICE([$note=Malware_Hash_Registry_Match,
-				        $msg=message, $id=rec$id, $URL=url]);
+				        $msg=message, $id=rec$id]);
 				}
 			}
 		}

scripts/policy/protocols/ssl/notary.bro

@@ -0,0 +1,105 @@
+@load base/protocols/ssl
+
+module CertNotary;
+
+export {
+	## A response from the ICSI certificate notary.
+	type Response: record {
+		first_seen: count &log &optional;
+		last_seen:  count &log &optional;
+		times_seen: count &log &optional;
+		valid:      bool  &log &optional;
+	};
+
+	## The notary domain to query.
+	const domain = "notary.icsi.berkeley.edu" &redef;
+}
+
+redef record SSL::Info += {
+	sha1:   string &log &optional;
+	notary: Response &log &optional;
+	};
+
+# The DNS cache of notary responses.
+global notary_cache: table[string] of Response &create_expire = 1 hr;
+
+# The records that wait for a notary response identified by the cert digest.
+# Each digest refers to a list of connection UIDs which are updated when a DNS
+# reply arrives asynchronously.
+global waitlist: table[string] of vector of SSL::Info;
+
+function clear_waitlist(digest: string)
+	{
+	if ( digest in waitlist )
+		{
+		for ( i in waitlist[digest] )
+			SSL::undelay_log(waitlist[digest][i], "notary");
+		delete waitlist[digest];
+		}
+	}
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509,
+                       chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( is_orig || chain_idx != 0 || ! c?$ssl )
+		return;
+
+	local digest = sha1_hash(der_cert);
+	c$ssl$sha1 = digest;
+
+	if ( digest in notary_cache )
+		{
+		c$ssl$notary = notary_cache[digest];
+		return;
+		}
+
+	SSL::delay_log(c$ssl, "notary");
+
+	local waits_already = digest in waitlist;
+	if ( ! waits_already )
+		waitlist[digest] = vector();
+	waitlist[digest][|waitlist[digest]|] = c$ssl;
+	if ( waits_already )
+		return;
+
+	when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) )
+		{
+		notary_cache[digest] = [];
+
+		# Parse notary answer.
+		if ( str == "<???>" ) # NXDOMAIN
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local fields = split(str, / /);
+		if ( |fields| != 5 ) # version 1 has 5 fields.
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local version = split(fields[1], /=/)[2];
+		if ( version != "1" )
+			{
+			clear_waitlist(digest);
+			return;
+			}
+		local r = notary_cache[digest];
+		r$first_seen = to_count(split(fields[2], /=/)[2]);
+		r$last_seen = to_count(split(fields[3], /=/)[2]);
+		r$times_seen = to_count(split(fields[4], /=/)[2]);
+		r$valid = split(fields[5], /=/)[2] == "1";
+
+		# Assign notary answer to all records waiting for this digest.
+		if ( digest in waitlist )
+			{
+			for ( i in waitlist[digest] )
+				{
+				local info = waitlist[digest][i];
+				SSL::undelay_log(info, "notary");
+				info$notary = r;
+				}
+			delete waitlist[digest];
+			}
+		}
+	}

scripts/site/local.bro

@@ -59,6 +59,10 @@ redef Software::vulnerable_versions += {
 # This script enables SSL/TLS certificate validation.
 @load protocols/ssl/validate-certs
 
+# This script checks each SSL certificate hash against the ICSI certificate
+# notary service.
+@load protocols/ssl/notary
+
 # If you have libGeoIP support built in, do some geographic detections and 
 # logging for SSH traffic.
 @load protocols/ssh/geo-data

scripts/test-all-policy.bro

@@ -68,6 +68,7 @@
 @load protocols/ssl/expiring-certs.bro
 @load protocols/ssl/extract-certs-pem.bro
 @load protocols/ssl/known-certs.bro
+#@load protocols/ssl/notary.bro
 @load protocols/ssl/validate-certs.bro
 @load tuning/__load__.bro
 @load tuning/defaults/__load__.bro

src/CMakeLists.txt

@@ -426,6 +426,7 @@ set(bro_SRCS
     strsep.c
     modp_numtoa.c
 
+    threading/AsciiFormatter.cc
     threading/BasicThread.cc
     threading/Manager.cc
     threading/MsgThread.cc

src/CompHash.cc

@@ -273,6 +273,9 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0,
 
 HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const
 	{
+	if ( ! v )
+		reporter->InternalError("null value given to CompositeHash::ComputeHash");
+
 	if ( is_singleton )
 		return ComputeSingletonHash(v, type_check);
 

src/Desc.h

@@ -57,9 +57,13 @@ public:
 	void AddEscapeSequence(const char* s) { escape_sequences.push_back(s); }
 	void AddEscapeSequence(const char* s, size_t n)
 	    { escape_sequences.push_back(string(s, n)); }
+	void AddEscapeSequence(const string & s)
+	    { escape_sequences.push_back(s); }
 	void RemoveEscapeSequence(const char* s) { escape_sequences.remove(s); }
 	void RemoveEscapeSequence(const char* s, size_t n)
 	    { escape_sequences.remove(string(s, n)); }
+	void RemoveEscapeSequence(const string & s)
+	    { escape_sequences.remove(s); }
 
 	void PushIndent();
 	void PopIndent();
@@ -114,6 +118,7 @@ public:
 
 	// Bypasses the escaping enabled via SetEscape().
 	void AddRaw(const char* s, int len)	{ AddBytesRaw(s, len); }
+	void AddRaw(const string &s)		{ AddBytesRaw(s.data(), s.size()); }
 
 	// Returns the description as a string.
 	const char* Description() const		{ return (const char*) base; }

src/Expr.cc

@@ -2801,12 +2801,27 @@ bool AssignExpr::DoUnserialize(UnserialInfo* info)
 	return UNSERIALIZE(&is_init);
 	}
 
-IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2)
+IndexExpr::IndexExpr(Expr* arg_op1, ListExpr* arg_op2, bool is_slice)
 : BinaryExpr(EXPR_INDEX, arg_op1, arg_op2)
 	{
 	if ( IsError() )
 		return;
 
+	if ( is_slice )
+		{
+		if ( ! IsString(op1->Type()->Tag()) )
+			ExprError("slice notation indexing only supported for strings currently");
+		}
+
+	else if ( IsString(op1->Type()->Tag()) )
+		{
+		if ( arg_op2->Exprs().length() != 1 )
+			ExprError("invalid string index expression");
+		}
+
+	if ( IsError() )
+		return;
+
 	int match_type = op1->Type()->MatchesIndex(arg_op2);
 	if ( match_type == DOES_NOT_MATCH_INDEX )
 		SetError("not an index type");
@@ -3906,8 +3921,11 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r)
 			{
 			int t_i = t_r->FieldOffset(sub_r->FieldName(i));
 			if ( t_i < 0 )
-				// Orphane field in rhs, that's ok.
+				{
-				continue;
+				ExprError(fmt("orphaned field \"%s\" in record coercion",
+				              sub_r->FieldName(i)));
+				break;
+				}
 
 			BroType* sub_t_i = sub_r->FieldType(i);
 			BroType* sup_t_i = t_r->FieldType(t_i);
@@ -3952,7 +3970,7 @@ RecordCoerceExpr::~RecordCoerceExpr()
 
 Val* RecordCoerceExpr::Fold(Val* v) const
 	{
-	RecordVal* val = new RecordVal(Type()->Ref()->AsRecordType());
+	RecordVal* val = new RecordVal(Type()->AsRecordType());
 	RecordVal* rv = v->AsRecordVal();
 
 	for ( int i = 0; i < map_size; ++i )

src/Expr.h

@@ -646,7 +646,7 @@ protected:
 
 class IndexExpr : public BinaryExpr {
 public:
-	IndexExpr(Expr* op1, ListExpr* op2);
+	IndexExpr(Expr* op1, ListExpr* op2, bool is_slice = false);
 
 	int CanAdd() const;
 	int CanDel() const;

src/Net.cc

@@ -421,7 +421,8 @@ void net_run()
 	set_processing_status("RUNNING", "net_run");
 
 	while ( io_sources.Size() ||
-		(packet_sorter && ! packet_sorter->Empty()) )
+		(packet_sorter && ! packet_sorter->Empty()) ||
+		(BifConst::exit_only_after_terminate && ! terminating) )
 		{
 		double ts;
 		IOSource* src = io_sources.FindSoonest(&ts);

src/Reporter.cc

@@ -27,6 +27,13 @@ Reporter::Reporter()
 	via_events = false;
 	in_error_handler = 0;
 
+	// Always use stderr at startup/init before scripts have been fully parsed.
+	// Messages may otherwise be missed if an error occurs that prevents events
+	// from ever being dispatched.
+	info_to_stderr = true;
+	warnings_to_stderr = true;
+	errors_to_stderr = true;
+
 	openlog("bro", 0, LOG_LOCAL5);
 	}
 
@@ -35,11 +42,19 @@ Reporter::~Reporter()
 	closelog();
 	}
 
+void Reporter::InitOptions()
+	{
+	info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool();
+	warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool();
+	errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool();
+	}
+
 void Reporter::Info(const char* fmt, ...)
 	{
 	va_list ap;
 	va_start(ap, fmt);
-	DoLog("", reporter_info, stderr, 0, 0, true, true, 0, fmt, ap);
+	FILE* out = info_to_stderr ? stderr : 0;
+	DoLog("", reporter_info, out, 0, 0, true, true, 0, fmt, ap);
 	va_end(ap);
 	}
 
@@ -47,7 +62,8 @@ void Reporter::Warning(const char* fmt, ...)
 	{
 	va_list ap;
 	va_start(ap, fmt);
-	DoLog("warning", reporter_warning, stderr, 0, 0, true, true, 0, fmt, ap);
+	FILE* out = warnings_to_stderr ? stderr : 0;
+	DoLog("warning", reporter_warning, out, 0, 0, true, true, 0, fmt, ap);
 	va_end(ap);
 	}
 
@@ -56,7 +72,8 @@ void Reporter::Error(const char* fmt, ...)
 	++errors;
 	va_list ap;
 	va_start(ap, fmt);
-	DoLog("error", reporter_error, stderr, 0, 0, true, true, 0, fmt, ap);
+	FILE* out = errors_to_stderr ? stderr : 0;
+	DoLog("error", reporter_error, out, 0, 0, true, true, 0, fmt, ap);
 	va_end(ap);
 	}
 
@@ -98,7 +115,9 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...)
 	PushLocation(expr->GetLocationInfo());
 	va_list ap;
 	va_start(ap, fmt);
-	DoLog("expression error", reporter_error, stderr, 0, 0, true, true, d.Description(), fmt, ap);
+	FILE* out = errors_to_stderr ? stderr : 0;
+	DoLog("expression error", reporter_error, out, 0, 0, true, true,
+	      d.Description(), fmt, ap);
 	va_end(ap);
 	PopLocation();
 	throw InterpreterException();
@@ -122,7 +141,9 @@ void Reporter::InternalWarning(const char* fmt, ...)
 	{
 	va_list ap;
 	va_start(ap, fmt);
-	DoLog("internal warning", reporter_warning, stderr, 0, 0, true, true, 0, fmt, ap);
+	FILE* out = warnings_to_stderr ? stderr : 0;
+	DoLog("internal warning", reporter_warning, out, 0, 0, true, true, 0, fmt,
+	      ap);
 	va_end(ap);
 	}
 
@@ -189,7 +210,9 @@ void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name)
 	WeirdFlowHelper(orig, resp, "%s", name);
 	}
 
-void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, const char* postfix, const char* fmt, va_list ap)
+void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out,
+                     Connection* conn, val_list* addl, bool location, bool time,
+                     const char* postfix, const char* fmt, va_list ap)
 	{
 	static char tmp[512];
 
@@ -297,8 +320,16 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Conne
 		else
 			mgr.QueueEvent(event, vl);
 		}
-
 	else
+		{
+		if ( addl )
+			{
+			loop_over_list(*addl, i)
+				Unref((*addl)[i]);
+			}
+		}
+
+	if ( out )
 		{
 		string s = "";
 
@@ -328,12 +359,6 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Conne
 
 		if ( out )
 			fprintf(out, "%s", s.c_str());
-
-		if ( addl )
-			{
-			loop_over_list(*addl, i)
-				Unref((*addl)[i]);
-			}
 		}
 
 	if ( alloced )

src/Reporter.h

@@ -43,6 +43,9 @@ public:
 	Reporter();
 	~Reporter();
 
+	// Initialize reporter-sepcific options	that are defined in script-layer.
+	void InitOptions();
+
 	// Report an informational message, nothing that needs specific
 	// attention.
 	void Info(const char* fmt, ...) FMT_ATTR;
@@ -126,6 +129,9 @@ private:
 	int errors;
 	bool via_events;
 	int in_error_handler;
+	bool info_to_stderr;
+	bool warnings_to_stderr;
+	bool errors_to_stderr;
 
 	std::list<std::pair<const Location*, const Location*> > locations;
 };

src/SerialTypes.h

@@ -171,6 +171,7 @@ SERIAL_STMT(EVENT_BODY_LIST, 16)
 SERIAL_STMT(INIT_STMT, 17)
 SERIAL_STMT(NULL_STMT, 18)
 SERIAL_STMT(WHEN_STMT, 19)
+SERIAL_STMT(FALLTHROUGH_STMT, 20)
 
 #define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE)
 SERIAL_TYPE(BRO_TYPE, 1)

src/Stmt.cc

@@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t)
 		"print", "event", "expr", "if", "when", "switch",
 		"for", "next", "break", "return", "add", "delete",
 		"list", "bodylist",
-		"<init>",
+		"<init>", "fallthrough",
 		"null",
 	};
 
@@ -584,6 +584,32 @@ bool IfStmt::DoUnserialize(UnserialInfo* info)
 	return s2 != 0;
 	}
 
+static BroStmtTag get_last_stmt_tag(const Stmt* stmt)
+	{
+	if ( ! stmt )
+		return STMT_NULL;
+
+	if ( stmt->Tag() != STMT_LIST )
+		return stmt->Tag();
+
+	const StmtList* stmts = stmt->AsStmtList();
+	int len = stmts->Stmts().length();
+
+	if ( len == 0 )
+		return STMT_LIST;
+
+	return get_last_stmt_tag(stmts->Stmts()[len - 1]);
+	}
+
+Case::Case(ListExpr* c, Stmt* arg_s)
+    : cases(simplify_expr_list(c, SIMPLIFY_GENERAL)), s(arg_s)
+	{
+	BroStmtTag t = get_last_stmt_tag(Body());
+
+	if ( t != STMT_BREAK && t != STMT_FALLTHROUGH && t != STMT_RETURN )
+		Error("case block must end in break/fallthrough/return statement");
+	}
+
 Case::~Case()
 	{
 	Unref(cases);
@@ -802,15 +828,12 @@ Val* SwitchStmt::DoExec(Frame* f, Val* v, stmt_flow_type& flow) const
 		flow = FLOW_NEXT;
 		rval = c->Body()->Exec(f, flow);
 
-		if ( flow == FLOW_BREAK )
+		if ( flow == FLOW_BREAK  || flow == FLOW_RETURN )
-			{
-			flow = FLOW_NEXT;
 			break;
 		}
 
-		if ( flow == FLOW_RETURN )
+	if ( flow != FLOW_RETURN )
-			break;
+		flow = FLOW_NEXT;
-		}
 
 	return rval;
 	}
@@ -1467,6 +1490,47 @@ bool BreakStmt::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+Val* FallthroughStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const
+	{
+	RegisterAccess();
+	flow = FLOW_FALLTHROUGH;
+	return 0;
+	}
+
+int FallthroughStmt::IsPure() const
+	{
+	return 1;
+	}
+
+void FallthroughStmt::Describe(ODesc* d) const
+	{
+	Stmt::Describe(d);
+	Stmt::DescribeDone(d);
+	}
+
+TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const
+	{
+	TraversalCode tc = cb->PreStmt(this);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = cb->PostStmt(this);
+	HANDLE_TC_STMT_POST(tc);
+	}
+
+IMPLEMENT_SERIAL(FallthroughStmt, SER_FALLTHROUGH_STMT);
+
+bool FallthroughStmt::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_FALLTHROUGH_STMT, Stmt);
+	return true;
+	}
+
+bool FallthroughStmt::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(Stmt);
+	return true;
+	}
+
 ReturnStmt::ReturnStmt(Expr* arg_e) : ExprStmt(STMT_RETURN, arg_e)
 	{
 	Scope* s = current_scope();
@@ -1789,13 +1853,21 @@ Val* InitStmt::Exec(Frame* f, stmt_flow_type& flow) const
 		ID* aggr = (*inits)[i];
 		BroType* t = aggr->Type();
 
-		Val* v;
+		Val* v = 0;
-		if ( t->Tag() == TYPE_RECORD )
+
+		switch ( t->Tag() ) {
+		case TYPE_RECORD:
 			v = new RecordVal(t->AsRecordType());
-		else if ( aggr->Type()->Tag() == TYPE_VECTOR )
+			break;
+		case TYPE_VECTOR:
 			v = new VectorVal(t->AsVectorType());
-		else
+			break;
+		case TYPE_TABLE:
 			v = new TableVal(t->AsTableType(), aggr->Attrs());
+			break;
+		default:
+			break;
+		}
 
 		f->SetElement(aggr->Offset(), v);
 		}

src/Stmt.h

@@ -46,6 +46,12 @@ public:
 		return (StmtList*) this;
 		}
 
+	const StmtList* AsStmtList() const
+		{
+		CHECK_TAG(tag, STMT_LIST, "Stmt::AsStmtList", stmt_name)
+		return (const StmtList*) this;
+		}
+
 	ForStmt* AsForStmt()
 		{
 		CHECK_TAG(tag, STMT_FOR, "Stmt::AsForStmt", stmt_name)
@@ -189,8 +195,7 @@ protected:
 
 class Case : public BroObj {
 public:
-	Case(ListExpr* c, Stmt* arg_s) :
+	Case(ListExpr* c, Stmt* arg_s);
-	    cases(simplify_expr_list(c,SIMPLIFY_GENERAL)), s(arg_s) { }
 	~Case();
 
 	const ListExpr* Cases() const	{ return cases; }
@@ -365,6 +370,21 @@ protected:
 	DECLARE_SERIAL(BreakStmt);
 };
 
+class FallthroughStmt : public Stmt {
+public:
+	FallthroughStmt() : Stmt(STMT_FALLTHROUGH)	{ }
+
+	Val* Exec(Frame* f, stmt_flow_type& flow) const;
+	int IsPure() const;
+
+	void Describe(ODesc* d) const;
+
+	TraversalCode Traverse(TraversalCallback* cb) const;
+
+protected:
+	DECLARE_SERIAL(FallthroughStmt);
+};
+
 class ReturnStmt : public ExprStmt {
 public:
 	ReturnStmt(Expr* e);

src/StmtEnums.h

@@ -16,6 +16,7 @@ typedef enum {
 	STMT_ADD, STMT_DELETE,
 	STMT_LIST, STMT_EVENT_BODY_LIST,
 	STMT_INIT,
+	STMT_FALLTHROUGH,
 	STMT_NULL
 #define NUM_STMTS (int(STMT_NULL) + 1)
 } BroStmtTag;
@@ -24,7 +25,8 @@ typedef enum {
 	FLOW_NEXT,		// continue on to next statement
 	FLOW_LOOP,		// go to top of loop
 	FLOW_BREAK,		// break out of loop
-	FLOW_RETURN		// return from function
+	FLOW_RETURN,		// return from function
+	FLOW_FALLTHROUGH	// fall through to next switch case
 } stmt_flow_type;
 
 extern const char* stmt_name(BroStmtTag t);

src/Type.cc

@@ -1305,6 +1305,7 @@ bool OpaqueType::DoUnserialize(UnserialInfo* info)
 		return false;
 
 	name = n;
+	delete [] n;
 	return true;
 	}
 

src/Var.cc

@@ -243,10 +243,7 @@ Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init,
 
 	else
 		{
-		if ( t->Tag() == TYPE_RECORD || t->Tag() == TYPE_TABLE ||
-		     t->Tag() == TYPE_VECTOR )
 		current_scope()->AddInit(id);
-
 		return new NullStmt;
 		}
 	}

src/const.bif

@@ -6,6 +6,7 @@ const ignore_keep_alive_rexmit: bool;
 const skip_http_data: bool;
 const use_conn_size_analyzer: bool;
 const report_gaps_for_partial: bool;
+const exit_only_after_terminate: bool;
 
 const NFS3::return_data: bool;
 const NFS3::return_data_max: count;

src/input/Manager.cc

@@ -787,22 +787,23 @@ bool Manager::UnrollRecordType(vector<Field*> *fields, const RecordType *rec,
 
 		if ( ! IsCompatibleType(rec->FieldType(i)) )
 			{
-			// If the field is a file or a function type
+			// If the field is a file, function, or opaque
 			// and it is optional, we accept it nevertheless.
 			// This allows importing logfiles containing this
 			// stuff that we actually cannot read :)
 			if ( allow_file_func )
 				{
 				if ( ( rec->FieldType(i)->Tag() == TYPE_FILE ||
-				       rec->FieldType(i)->Tag() == TYPE_FUNC ) &&
+				       rec->FieldType(i)->Tag() == TYPE_FUNC ||
+				       rec->FieldType(i)->Tag() == TYPE_OPAQUE ) &&
 				       rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) )
 					{
-					reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag()));
+					reporter->Info("Encountered incompatible type \"%s\" in type definition for ReaderFrontend. Ignoring optional field.", type_name(rec->FieldType(i)->Tag()));
 					continue;
 					}
 				}
 
-			reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag()));
+			reporter->Error("Incompatible type \"%s\" in type definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag()));
 			return false;
 			}
 

src/input/ReaderBackend.cc

@@ -281,50 +281,4 @@ bool ReaderBackend::OnHeartbeat(double network_time, double current_time)
 	return DoHeartbeat(network_time, current_time);
 	}
 
-TransportProto ReaderBackend::StringToProto(const string &proto)
-	{
-	if ( proto == "unknown" )
-		return TRANSPORT_UNKNOWN;
-	else if ( proto == "tcp" )
-		return TRANSPORT_TCP;
-	else if ( proto == "udp" )
-		return TRANSPORT_UDP;
-	else if ( proto == "icmp" )
-		return TRANSPORT_ICMP;
-
-	Error(Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
-
-	return TRANSPORT_UNKNOWN;
-	}
-
-
-// More or less verbose copy from IPAddr.cc -- which uses reporter.
-Value::addr_t ReaderBackend::StringToAddr(const string &s)
-	{
-	Value::addr_t val;
-
-	if ( s.find(':') == std::string::npos ) // IPv4.
-		{
-		val.family = IPv4;
-
-		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
-			{
-			Error(Fmt("Bad address: %s", s.c_str()));
-			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
-			}
-		}
-
-	else
-		{
-		val.family = IPv6;
-		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
-			{
-			Error(Fmt("Bad address: %s", s.c_str()));
-			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
-			}
-		}
-
-	return val;
-	}
-
 }

src/input/ReaderBackend.h

@@ -315,21 +315,6 @@ protected:
 	 */
 	void EndCurrentSend();
 
-	/**
-	 *  Convert a string into a TransportProto. This is just a utility
-	 *  function for Readers.
-	 *
-	 * @param proto the transport protocol
-	 */
-	TransportProto StringToProto(const string &proto);
-
-	/**
-	 * Convert a string into a Value::addr_t.  This is just a utility
-	 * function for Readers.
-	 *
-	 * @param addr containing an ipv4 or ipv6 address
-	 */
-	threading::Value::addr_t StringToAddr(const string &addr);
 
 private:
 	// Frontend that instantiated us. This object must not be accessed

src/input/readers/Ascii.cc

@@ -67,11 +67,14 @@ Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend)
 
 	unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(),
 			    BifConst::InputAscii::unset_field->Len());
+
+	ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
 }
 
 Ascii::~Ascii()
 	{
 	DoClose();
+	delete ascii;
 	}
 
 void Ascii::DoClose()
@@ -210,246 +213,6 @@ bool Ascii::GetLine(string& str)
 	return false;
 	}
 
-bool Ascii::CheckNumberError(const string& s, const char * end)
-	{
-	// Do this check first, before executing s.c_str() or similar.
-	// otherwise the value to which *end is pointing at the moment might
-	// be gone ...
-	bool endnotnull =  (*end != '\0');
-
-	if ( s.length() == 0 )
-		{
-		Error("Got empty string for number field");
-		return true;
-		}
-
-	if ( end == s.c_str() ) {
-		Error(Fmt("String '%s' contained no parseable number", s.c_str()));
-		return true;
-	}
-
-	if ( endnotnull )
-		Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
-
-	if ( errno == EINVAL )
-		{
-		Error(Fmt("String '%s' could not be converted to a number", s.c_str()));
-		return true;
-		}
-
-	else if ( errno == ERANGE )
-		{
-		Error(Fmt("Number '%s' out of supported range.", s.c_str()));
-		return true;
-		}
-
-	return false;
-	}
-
-
-Value* Ascii::EntryToVal(string s, FieldMapping field)
-	{
-	if ( s.compare(unset_field) == 0 )  // field is not set...
-		return new Value(field.type, false);
-
-	Value* val = new Value(field.type, true);
-	char* end = 0;
-	errno = 0;
-
-	switch ( field.type ) {
-	case TYPE_ENUM:
-	case TYPE_STRING:
-		s = get_unescaped_string(s);
-		val->val.string_val.length = s.size();
-		val->val.string_val.data = copy_string(s.c_str());
-		break;
-
-	case TYPE_BOOL:
-		if ( s == "T" )
-			val->val.int_val = 1;
-		else if ( s == "F" )
-			val->val.int_val = 0;
-		else
-			{
-			Error(Fmt("Field: %s Invalid value for boolean: %s",
-				  field.name.c_str(), s.c_str()));
-			goto parse_error;
-			}
-		break;
-
-	case TYPE_INT:
-		val->val.int_val = strtoll(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			goto parse_error;
-		break;
-
-	case TYPE_DOUBLE:
-	case TYPE_TIME:
-	case TYPE_INTERVAL:
-		val->val.double_val = strtod(s.c_str(), &end);
-		if ( CheckNumberError(s, end) )
-			goto parse_error;
-		break;
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		val->val.uint_val = strtoull(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			goto parse_error;
-		break;
-
-	case TYPE_PORT:
-		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
-		if ( CheckNumberError(s, end) )
-			goto parse_error;
-
-		val->val.port_val.proto = TRANSPORT_UNKNOWN;
-		break;
-
-	case TYPE_SUBNET:
-		{
-		s = get_unescaped_string(s);
-		size_t pos = s.find("/");
-		if ( pos == s.npos )
-			{
-			Error(Fmt("Invalid value for subnet: %s", s.c_str()));
-			goto parse_error;
-			}
-
-		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
-
-		if ( CheckNumberError(s, end) )
-			goto parse_error;
-
-		string addr = s.substr(0, pos);
-
-		val->val.subnet_val.prefix = StringToAddr(addr);
-		val->val.subnet_val.length = width;
-		break;
-		}
-
-	case TYPE_ADDR:
-		s = get_unescaped_string(s);
-		val->val.addr_val = StringToAddr(s);
-		break;
-
-	case TYPE_TABLE:
-	case TYPE_VECTOR:
-		// First - common initialization
-		// Then - initialization for table.
-		// Then - initialization for vector.
-		// Then - common stuff
-		{
-		// how many entries do we have...
-		unsigned int length = 1;
-		for ( unsigned int i = 0; i < s.size(); i++ )
-			{
-			if ( s[i] == set_separator[0] )
-				length++;
-			}
-
-		unsigned int pos = 0;
-		bool error = false;
-
-		if ( s.compare(empty_field) == 0 )
-			length = 0;
-
-		Value** lvals = new Value* [length];
-
-		if ( field.type == TYPE_TABLE )
-			{
-			val->val.set_val.vals = lvals;
-			val->val.set_val.size = length;
-			}
-
-		else if ( field.type == TYPE_VECTOR )
-			{
-			val->val.vector_val.vals = lvals;
-			val->val.vector_val.size = length;
-			}
-
-		else
-			assert(false);
-
-		if ( length == 0 )
-			break; //empty
-
-		istringstream splitstream(s);
-		while ( splitstream )
-			{
-			string element;
-
-			if ( ! getline(splitstream, element, set_separator[0]) )
-				break;
-
-			if ( pos >= length )
-				{
-				Error(Fmt("Internal error while parsing set. pos %d >= length %d."
-				          " Element: %s", pos, length, element.c_str()));
-				error = true;
-				break;
-				}
-
-			Value* newval = EntryToVal(element, field.subType());
-			if ( newval == 0 )
-				{
-				Error("Error while reading set or vector");
-				error = true;
-				break;
-				}
-
-			lvals[pos] = newval;
-
-			pos++;
-			}
-
-		// Test if the string ends with a set_separator... or if the
-		// complete string is empty. In either of these cases we have
-		// to push an empty val on top of it.
-		if ( ! error && (s.empty() || *s.rbegin() == set_separator[0]) )
-			{
-			lvals[pos] = EntryToVal("", field.subType());
-			if ( lvals[pos] == 0 )
-				{
-				Error("Error while trying to add empty set element");
-				goto parse_error;
-				}
-
-			pos++;
-			}
-
-		if ( error ) {
-			// We had an error while reading a set or a vector.
-			// Hence we have to clean up the values that have
-			// been read so far
-			for ( unsigned int i = 0; i < pos; i++ )
-				delete lvals[i];
-
-			goto parse_error;
-		}
-
-		if ( pos != length )
-			{
-			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
-			goto parse_error;
-			}
-
-		break;
-		}
-
-	default:
-		Error(Fmt("unsupported field format %d for %s", field.type,
-		field.name.c_str()));
-		goto parse_error;
-	}
-
-	return val;
-
-parse_error:
-	delete val;
-	return 0;
-	}
-
 // read the entire file and send appropriate thingies back to InputMgr
 bool Ascii::DoUpdate()
 	{
@@ -560,7 +323,8 @@ bool Ascii::DoUpdate()
 				return false;
 				}
 
-			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
+			Value* val = ascii->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype);
+
 			if ( val == 0 )
 				{
 				Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str()));
@@ -574,7 +338,7 @@ bool Ascii::DoUpdate()
 				assert(val->type == TYPE_PORT );
 				//	Error(Fmt("Got type %d != PORT with secondary position!", val->type));
 
-				val->val.port_val.proto = StringToProto(stringfields[(*fit).secondary_position]);
+				val->val.port_val.proto = ascii->ParseProto(stringfields[(*fit).secondary_position]);
 				}
 
 			fields[fpos] = val;

src/input/readers/Ascii.h

@@ -7,6 +7,7 @@
 #include <vector>
 
 #include "../ReaderBackend.h"
+#include "threading/AsciiFormatter.h"
 
 namespace input { namespace reader {
 
@@ -47,8 +48,6 @@ private:
 
 	bool ReadHeader(bool useCached);
 	bool GetLine(string& str);
-	threading::Value* EntryToVal(string s, FieldMapping type);
-	bool CheckNumberError(const string& s, const char * end);
 
 	ifstream* file;
 	time_t mtime;
@@ -64,6 +63,8 @@ private:
 	string set_separator;
 	string empty_field;
 	string unset_field;
+
+	AsciiFormatter* ascii;
 };
 
 

src/input/readers/Benchmark.cc

@@ -25,11 +25,15 @@ Benchmark::Benchmark(ReaderFrontend *frontend) : ReaderBackend(frontend)
 	stopspreadat = int(BifConst::InputBenchmark::stopspreadat);
 	timedspread = double(BifConst::InputBenchmark::timedspread);
 	heartbeat_interval = double(BifConst::Threading::heartbeat_interval);
+
+	ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo());
 	}
 
 Benchmark::~Benchmark()
 	{
 	DoClose();
+
+	delete ascii;
 	}
 
 void Benchmark::DoClose()
@@ -162,13 +166,13 @@ threading::Value* Benchmark::EntryToVal(TypeTag type, TypeTag subtype)
 
 	case TYPE_SUBNET:
 		{
-		val->val.subnet_val.prefix = StringToAddr("192.168.17.1");
+		val->val.subnet_val.prefix = ascii->ParseAddr("192.168.17.1");
 		val->val.subnet_val.length = 16;
 		}
 		break;
 
 	case TYPE_ADDR:
-		val->val.addr_val = StringToAddr("192.168.17.1");
+		val->val.addr_val = ascii->ParseAddr("192.168.17.1");
 		break;
 
 	case TYPE_TABLE:

src/input/readers/Benchmark.h

@@ -4,6 +4,7 @@
 #define INPUT_READERS_BENCHMARK_H
 
 #include "../ReaderBackend.h"
+#include "threading/AsciiFormatter.h"
 
 namespace input { namespace reader {
 
@@ -38,6 +39,8 @@ private:
 	double heartbeatstarttime;
 	double timedspread;
 	double heartbeat_interval;
+
+	AsciiFormatter* ascii;
 };
 
 

src/logging/WriterBackend.cc

@@ -1,7 +1,6 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #include "util.h"
-#include "bro_inet_ntop.h"
 #include "threading/SerialTypes.h"
 
 #include "Manager.h"
@@ -328,46 +327,3 @@ bool WriterBackend::OnHeartbeat(double network_time, double current_time)
 	SendOut(new FlushWriteBufferMessage(frontend));
 	return DoHeartbeat(network_time, current_time);
 	}
-
-string WriterBackend::Render(const threading::Value::addr_t& addr) const
-	{
-	if ( addr.family == IPv4 )
-		{
-		char s[INET_ADDRSTRLEN];
-
-		if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
-			return "<bad IPv4 address conversion>";
-		else
-			return s;
-		}
-	else
-		{
-		char s[INET6_ADDRSTRLEN];
-
-		if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
-			return "<bad IPv6 address conversion>";
-		else
-			return s;
-		}
-	}
-
-string WriterBackend::Render(const threading::Value::subnet_t& subnet) const
-	{
-	char l[16];
-
-	if ( subnet.prefix.family == IPv4 )
-		modp_uitoa10(subnet.length - 96, l);
-	else
-		modp_uitoa10(subnet.length, l);
-
-	string s = Render(subnet.prefix) + "/" + l;
-
-	return s;
-	}
-
-string WriterBackend::Render(double d) const
-	{
-	char buf[256];
-	modp_dtoa(d, buf, 6);
-	return buf;
-	}

src/logging/WriterBackend.h

@@ -256,30 +256,6 @@ public:
 	 */
 	bool FinishedRotation();
 
-	/** Helper method to render an IP address as a string.
-	  *
-	  * @param addr The address.
-	  *
-	  * @return An ASCII representation of the address.
-	  */
-	string Render(const threading::Value::addr_t& addr) const;
-
-	/** Helper method to render an subnet value as a string.
-	  *
-	  * @param addr The address.
-	  *
-	  * @return An ASCII representation of the address.
-	  */
-	string Render(const threading::Value::subnet_t& subnet) const;
-
-	/** Helper method to render a double in Bro's standard precision.
-	  *
-	  * @param d The double.
-	  *
-	  * @return An ASCII representation of the double.
-	  */
-	string Render(double d) const;
-
 	// Overridden from MsgThread.
 	virtual bool OnHeartbeat(double network_time, double current_time);
 	virtual bool OnFinish(double network_time);

src/logging/writers/Ascii.cc

@@ -24,33 +24,35 @@ Ascii::Ascii(WriterFrontend* frontend) : WriterBackend(frontend)
 	output_to_stdout = BifConst::LogAscii::output_to_stdout;
 	include_meta = BifConst::LogAscii::include_meta;
 
-	separator_len = BifConst::LogAscii::separator->Len();
+	separator.assign(
-	separator = new char[separator_len];
+			(const char*) BifConst::LogAscii::separator->Bytes(),
-	memcpy(separator, BifConst::LogAscii::separator->Bytes(),
+			BifConst::LogAscii::separator->Len()
-	       separator_len);
+			);
 
-	set_separator_len = BifConst::LogAscii::set_separator->Len();
+	set_separator.assign(
-	set_separator = new char[set_separator_len];
+			(const char*) BifConst::LogAscii::set_separator->Bytes(),
-	memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(),
+			BifConst::LogAscii::set_separator->Len()
-	       set_separator_len);
+			);
 
-	empty_field_len = BifConst::LogAscii::empty_field->Len();
+	empty_field.assign(
-	empty_field = new char[empty_field_len];
+			(const char*) BifConst::LogAscii::empty_field->Bytes(),
-	memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(),
+			BifConst::LogAscii::empty_field->Len()
-	       empty_field_len);
+			);
 
-	unset_field_len = BifConst::LogAscii::unset_field->Len();
+	unset_field.assign(
-	unset_field = new char[unset_field_len];
+			(const char*) BifConst::LogAscii::unset_field->Bytes(),
-	memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(),
+			BifConst::LogAscii::unset_field->Len()
-	       unset_field_len);
+			);
 
-	meta_prefix_len = BifConst::LogAscii::meta_prefix->Len();
+	meta_prefix.assign(
-	meta_prefix = new char[meta_prefix_len];
+			(const char*) BifConst::LogAscii::meta_prefix->Bytes(),
-	memcpy(meta_prefix, BifConst::LogAscii::meta_prefix->Bytes(),
+			BifConst::LogAscii::meta_prefix->Len()
-	       meta_prefix_len);
+			);
 
 	desc.EnableEscaping();
-	desc.AddEscapeSequence(separator, separator_len);
+	desc.AddEscapeSequence(separator);
+
+	ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo(set_separator, unset_field, empty_field));
 	}
 
 Ascii::~Ascii()
@@ -61,17 +63,12 @@ Ascii::~Ascii()
 		abort();
 		}
 
-	delete [] separator;
+	delete ascii;
-	delete [] set_separator;
-	delete [] empty_field;
-	delete [] unset_field;
-	delete [] meta_prefix;
 	}
 
 bool Ascii::WriteHeaderField(const string& key, const string& val)
 	{
-	string str = string(meta_prefix, meta_prefix_len) +
+	string str = meta_prefix + key + separator + val + "\n";
-		key + string(separator, separator_len) + val + "\n";
 
 	return safe_write(fd, str.c_str(), str.length());
 	}
@@ -136,8 +133,8 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 			{
 			if ( i > 0 )
 				{
-				names += string(separator, separator_len);
+				names += separator;
-				types += string(separator, separator_len);
+				types += separator;
 				}
 
 			names += string(fields[i]->name);
@@ -154,20 +151,17 @@ bool Ascii::DoInit(const WriterInfo& info, int num_fields, const Field* const *
 			return true;
 			}
 
-		string str = string(meta_prefix, meta_prefix_len)
+		string str = meta_prefix
 			+ "separator " // Always use space as separator here.
-			+ get_escaped_string(string(separator, separator_len), false)
+			+ get_escaped_string(separator, false)
 			+ "\n";
 
 		if ( ! safe_write(fd, str.c_str(), str.length()) )
 			goto write_error;
 
-		if ( ! (WriteHeaderField("set_separator", get_escaped_string(
+		if ( ! (WriteHeaderField("set_separator", get_escaped_string(set_separator, false)) &&
-			    string(set_separator, set_separator_len), false)) &&
+			WriteHeaderField("empty_field", get_escaped_string(empty_field, false)) &&
-			WriteHeaderField("empty_field", get_escaped_string(
+			WriteHeaderField("unset_field", get_escaped_string(unset_field, false)) &&
-			    string(empty_field, empty_field_len), false)) &&
-			WriteHeaderField("unset_field", get_escaped_string(
-			    string(unset_field, unset_field_len), false)) &&
 			WriteHeaderField("path", get_escaped_string(path, false)) &&
 			WriteHeaderField("open", Timestamp(0))) )
 			goto write_error;
@@ -205,151 +199,6 @@ bool Ascii::DoFinish(double network_time)
 	return true;
 	}
 
-
-bool Ascii::DoWriteOne(ODesc* desc, Value* val, const Field* field)
-	{
-	if ( ! val->present )
-		{
-		desc->AddN(unset_field, unset_field_len);
-		return true;
-		}
-
-	switch ( val->type ) {
-
-	case TYPE_BOOL:
-		desc->Add(val->val.int_val ? "T" : "F");
-		break;
-
-	case TYPE_INT:
-		desc->Add(val->val.int_val);
-		break;
-
-	case TYPE_COUNT:
-	case TYPE_COUNTER:
-		desc->Add(val->val.uint_val);
-		break;
-
-	case TYPE_PORT:
-		desc->Add(val->val.port_val.port);
-		break;
-
-	case TYPE_SUBNET:
-		desc->Add(Render(val->val.subnet_val));
-		break;
-
-	case TYPE_ADDR:
-		desc->Add(Render(val->val.addr_val));
-		break;
-
-	case TYPE_DOUBLE:
-		// Rendering via Add() truncates trailing 0s after the
-		// decimal point. The difference with TIME/INTERVAL is mainly
-		// to keep the log format consistent.
-		desc->Add(val->val.double_val);
-		break;
-
-	case TYPE_INTERVAL:
-	case TYPE_TIME:
-		// Rendering via Render() keeps trailing 0s after the decimal
-		// point. The difference with DOUBLEis mainly to keep the log
-		// format consistent.
-		desc->Add(Render(val->val.double_val));
-		break;
-
-	case TYPE_ENUM:
-	case TYPE_STRING:
-	case TYPE_FILE:
-	case TYPE_FUNC:
-		{
-		int size = val->val.string_val.length;
-		const char* data = val->val.string_val.data;
-
-		if ( ! size )
-			{
-			desc->AddN(empty_field, empty_field_len);
-			break;
-			}
-
-		if ( size == unset_field_len && memcmp(data, unset_field, size) == 0 )
-			{
-			// The value we'd write out would match exactly the
-			// place-holder we use for unset optional fields. We
-			// escape the first character so that the output
-			// won't be ambigious.
-			static const char hex_chars[] = "0123456789abcdef";
-			char hex[6] = "\\x00";
-			hex[2] = hex_chars[((*data) & 0xf0) >> 4];
-			hex[3] = hex_chars[(*data) & 0x0f];
-			desc->AddRaw(hex, 4);
-
-			++data;
-			--size;
-			}
-
-		if ( size )
-			desc->AddN(data, size);
-
-		break;
-		}
-
-	case TYPE_TABLE:
-		{
-		if ( ! val->val.set_val.size )
-			{
-			desc->AddN(empty_field, empty_field_len);
-			break;
-			}
-
-		desc->AddEscapeSequence(set_separator, set_separator_len);
-		for ( int j = 0; j < val->val.set_val.size; j++ )
-			{
-			if ( j > 0 )
-				desc->AddRaw(set_separator, set_separator_len);
-
-			if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) )
-				{
-				desc->RemoveEscapeSequence(set_separator, set_separator_len);
-				return false;
-				}
-			}
-		desc->RemoveEscapeSequence(set_separator, set_separator_len);
-
-		break;
-		}
-
-	case TYPE_VECTOR:
-		{
-		if ( ! val->val.vector_val.size )
-			{
-			desc->AddN(empty_field, empty_field_len);
-			break;
-			}
-
-		desc->AddEscapeSequence(set_separator, set_separator_len);
-		for ( int j = 0; j < val->val.vector_val.size; j++ )
-			{
-			if ( j > 0 )
-				desc->AddRaw(set_separator, set_separator_len);
-
-			if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) )
-				{
-				desc->RemoveEscapeSequence(set_separator, set_separator_len);
-				return false;
-				}
-			}
-		desc->RemoveEscapeSequence(set_separator, set_separator_len);
-
-		break;
-		}
-
-	default:
-		Error(Fmt("unsupported field format %d for %s", val->type, field->name));
-		return false;
-	}
-
-	return true;
-	}
-
 bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 			     Value** vals)
 	{
@@ -361,9 +210,9 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	for ( int i = 0; i < num_fields; i++ )
 		{
 		if ( i > 0 )
-			desc.AddRaw(separator, separator_len);
+			desc.AddRaw(separator);
 
-		if ( ! DoWriteOne(&desc, vals[i], fields[i]) )
+		if ( ! ascii->Describe(&desc, vals[i], fields[i]->name) )
 			return false;
 		}
 
@@ -372,7 +221,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	const char* bytes = (const char*)desc.Bytes();
 	int len = desc.Len();
 
-	if ( strncmp(bytes, meta_prefix, meta_prefix_len) == 0 )
+	if ( strncmp(bytes, meta_prefix.data(), meta_prefix.size()) == 0 )
 		{
 		// It would so escape the first character.
 		char buf[16];

src/logging/writers/Ascii.h

@@ -6,6 +6,7 @@
 #define LOGGING_WRITER_ASCII_H
 
 #include "../WriterBackend.h"
+#include "threading/AsciiFormatter.h"
 
 namespace logging { namespace writer {
 
@@ -32,7 +33,6 @@ protected:
 
 private:
 	bool IsSpecial(string path) 	{ return path.find("/dev/") == 0; }
-	bool DoWriteOne(ODesc* desc, threading::Value* val, const threading::Field* field);
 	bool WriteHeaderField(const string& key, const string& value);
 	void CloseFile(double t);
 	string Timestamp(double t); // Uses current time if t is zero.
@@ -47,20 +47,13 @@ private:
 	bool include_meta;
 	bool tsv;
 
-	char* separator;
+	string separator;
-	int separator_len;
+	string set_separator;
+	string empty_field;
+	string unset_field;
+	string meta_prefix;
 
-	char* set_separator;
+	AsciiFormatter* ascii;
-	int set_separator_len;
-
-	char* empty_field;
-	int empty_field_len;
-
-	char* unset_field;
-	int unset_field_len;
-
-	char* meta_prefix;
-	int meta_prefix_len;
 };
 
 }

src/logging/writers/DataSeries.cc

@@ -46,10 +46,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
 		}
 
 	case TYPE_SUBNET:
-		return Render(val->val.subnet_val);
+		return ascii->Render(val->val.subnet_val);
 
 	case TYPE_ADDR:
-		return Render(val->val.addr_val);
+		return ascii->Render(val->val.addr_val);
 
 	// Note: These two cases are relatively special.  We need to convert
 	// these values into their integer equivalents to maximize precision.
@@ -69,10 +69,10 @@ std::string DataSeries::LogValueToString(threading::Value *val)
 			return ostr.str();
 			}
 		else
-			return Render(val->val.double_val);
+			return ascii->Render(val->val.double_val);
 
 	case TYPE_DOUBLE:
-		return Render(val->val.double_val);
+		return ascii->Render(val->val.double_val);
 
 	case TYPE_ENUM:
 	case TYPE_STRING:
@@ -231,11 +231,14 @@ DataSeries::DataSeries(WriterFrontend* frontend) : WriterBackend(frontend)
 	ds_num_threads = BifConst::LogDataSeries::num_threads;
 	ds_use_integer_for_time = BifConst::LogDataSeries::use_integer_for_time;
 	ds_set_separator = ",";
+
+	ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo());
 }
 
 DataSeries::~DataSeries()
-{
+	{
-}
+	delete ascii;
+	}
 
 bool DataSeries::OpenLog(string path)
 	{

src/logging/writers/DataSeries.h

@@ -12,6 +12,7 @@
 #include <DataSeries/GeneralField.hpp>
 
 #include "../WriterBackend.h"
+#include "threading/AsciiFormatter.h"
 
 namespace logging { namespace writer {
 
@@ -116,6 +117,8 @@ private:
 	bool ds_dump_schema;
 	bool ds_use_integer_for_time;
 	string ds_set_separator;
+
+	AsciiFormatter* ascii;
 };
 
 }

src/logging/writers/ElasticSearch.cc

@@ -16,6 +16,7 @@
 #include "BroString.h"
 #include "NetVar.h"
 #include "threading/SerialTypes.h"
+#include "threading/AsciiFormatter.h"
 
 #include <curl/curl.h>
 #include <curl/easy.h>
@@ -51,11 +52,14 @@ ElasticSearch::ElasticSearch(WriterFrontend* frontend) : WriterBackend(frontend)
 	transfer_timeout = static_cast<long>(BifConst::LogElasticSearch::transfer_timeout);
 
 	curl_handle = HTTPSetup();
+
+	ascii = new AsciiFormatter(this, AsciiFormatter::SeparatorInfo());
 }
 
 ElasticSearch::~ElasticSearch()
 	{
 	delete [] cluster_name;
+	delete ascii;
 	}
 
 bool ElasticSearch::DoInit(const WriterInfo& info, int num_fields, const threading::Field* const* fields)
@@ -124,13 +128,13 @@ bool ElasticSearch::AddValueToBuffer(ODesc* b, Value* val)
 
 		case TYPE_SUBNET:
 			b->AddRaw("\"", 1);
-			b->Add(Render(val->val.subnet_val));
+			b->Add(ascii->Render(val->val.subnet_val));
 			b->AddRaw("\"", 1);
 			break;
 
 		case TYPE_ADDR:
 			b->AddRaw("\"", 1);
-			b->Add(Render(val->val.addr_val));
+			b->Add(ascii->Render(val->val.addr_val));
 			b->AddRaw("\"", 1);
 			break;
 
@@ -402,7 +406,7 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 
 		case CURLE_OK:
 			{
-			uint http_code = 0;
+			long http_code = 0;
 			curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code);
 			if ( http_code == 200 )
 				// Hopefully everything goes through here.

src/logging/writers/ElasticSearch.h

@@ -72,6 +72,8 @@ private:
 	bool failing;
 
 	uint64 batch_size;
+
+	AsciiFormatter* ascii;
 };
 
 }

src/main.cc

@@ -824,6 +824,8 @@ int main(int argc, char** argv)
 		exit(1);
 		}
 
+	reporter->InitOptions();
+
 	init_general_global_var();
 
 	if ( user_pcap_filter )
@@ -1053,7 +1055,9 @@ int main(int argc, char** argv)
 
 	io_sources.Register(thread_mgr, true);
 
-	if ( io_sources.Size() > 0 || have_pending_timers )
+	if ( io_sources.Size() > 0 ||
+	     have_pending_timers ||
+	     BifConst::exit_only_after_terminate )
 		{
 		if ( profiling_logger )
 			profiling_logger->Log();

src/parse.y

@@ -8,8 +8,8 @@
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
 %token TOK_BOOL TOK_BREAK TOK_CASE TOK_CONST
 %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_COUNTER TOK_DEFAULT TOK_DELETE
-%token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR
+%token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FALLTHROUGH
-%token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT
+%token TOK_FILE TOK_FOR TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT
 %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE
 %token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT
 %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF
@@ -418,6 +418,14 @@ expr:
 			$$ = new IndexExpr($1, $3);
 			}
 
+	|	expr '[' expr ':' expr ']'
+			{
+			set_location(@1, @6);
+			ListExpr* le = new ListExpr($3);
+			le->Append($5);
+			$$ = new IndexExpr($1, le, true);
+			}
+
 	|	expr '$' TOK_ID
 			{
 			set_location(@1, @3);
@@ -1428,6 +1436,14 @@ stmt:
 			    brofiler.AddStmt($$);
 			}
 
+	|	TOK_FALLTHROUGH ';' opt_no_test
+			{
+			set_location(@1, @2);
+			$$ = new FallthroughStmt;
+			if ( ! $3 )
+				brofiler.AddStmt($$);
+			}
+
 	|	TOK_RETURN ';' opt_no_test
 			{
 			set_location(@1, @2);

src/scan.l

@@ -282,6 +282,7 @@ else	return TOK_ELSE;
 enum	return TOK_ENUM;
 event	return TOK_EVENT;
 export	return TOK_EXPORT;
+fallthrough	return TOK_FALLTHROUGH;
 file	return TOK_FILE;
 for	return TOK_FOR;
 function	return TOK_FUNCTION;

src/threading/AsciiFormatter.cc

@@ -0,0 +1,508 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "config.h"
+
+#include <sstream>
+#include <errno.h>
+
+#include "AsciiFormatter.h"
+#include "bro_inet_ntop.h"
+
+AsciiFormatter::SeparatorInfo::SeparatorInfo()
+	{
+	this->set_separator = "SHOULD_NOT_BE_USED";
+	this->unset_field = "SHOULD_NOT_BE_USED";
+	this->empty_field = "SHOULD_NOT_BE_USED";
+	}
+
+AsciiFormatter::SeparatorInfo::SeparatorInfo(const string & set_separator,
+				 const string & unset_field, const string & empty_field)
+	{
+	this->set_separator = set_separator;
+	this->unset_field = unset_field;
+	this->empty_field = empty_field;
+	}
+
+AsciiFormatter::AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info)
+	{
+	thread = t;
+	this->separators = info;
+	}
+
+AsciiFormatter::~AsciiFormatter()
+	{
+	}
+
+bool AsciiFormatter::Describe(ODesc* desc, threading::Value* val, const string& name) const
+	{
+	if ( ! val->present )
+		{
+		desc->Add(separators.unset_field);
+		return true;
+		}
+
+	switch ( val->type ) {
+
+	case TYPE_BOOL:
+		desc->Add(val->val.int_val ? "T" : "F");
+		break;
+
+	case TYPE_INT:
+		desc->Add(val->val.int_val);
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		desc->Add(val->val.uint_val);
+		break;
+
+	case TYPE_PORT:
+		desc->Add(val->val.port_val.port);
+		break;
+
+	case TYPE_SUBNET:
+		desc->Add(Render(val->val.subnet_val));
+		break;
+
+	case TYPE_ADDR:
+		desc->Add(Render(val->val.addr_val));
+		break;
+
+	case TYPE_DOUBLE:
+		// Rendering via Add() truncates trailing 0s after the
+		// decimal point. The difference with TIME/INTERVAL is mainly
+		// to keep the log format consistent.
+		desc->Add(val->val.double_val);
+		break;
+
+	case TYPE_INTERVAL:
+	case TYPE_TIME:
+		// Rendering via Render() keeps trailing 0s after the decimal
+		// point. The difference with DOUBLE is mainly to keep the
+		// log format consistent.
+		desc->Add(Render(val->val.double_val));
+		break;
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+	case TYPE_FUNC:
+		{
+		int size = val->val.string_val.length;
+		const char* data = val->val.string_val.data;
+
+		if ( ! size )
+			{
+			desc->Add(separators.empty_field);
+			break;
+			}
+
+		if ( size == (int)separators.unset_field.size() && memcmp(data, separators.unset_field.data(), size) == 0 )
+			{
+			// The value we'd write out would match exactly the
+			// place-holder we use for unset optional fields. We
+			// escape the first character so that the output
+			// won't be ambigious.
+			static const char hex_chars[] = "0123456789abcdef";
+			char hex[6] = "\\x00";
+			hex[2] = hex_chars[((*data) & 0xf0) >> 4];
+			hex[3] = hex_chars[(*data) & 0x0f];
+			desc->AddRaw(hex, 4);
+
+			++data;
+			--size;
+			}
+
+		if ( size )
+			desc->AddN(data, size);
+
+		break;
+		}
+
+	case TYPE_TABLE:
+		{
+		if ( ! val->val.set_val.size )
+			{
+			desc->Add(separators.empty_field);
+			break;
+			}
+
+		desc->AddEscapeSequence(separators.set_separator);
+		for ( int j = 0; j < val->val.set_val.size; j++ )
+			{
+			if ( j > 0 )
+				desc->AddRaw(separators.set_separator);
+
+			if ( ! Describe(desc, val->val.set_val.vals[j], name) )
+				{
+				desc->RemoveEscapeSequence(separators.set_separator);
+				return false;
+				}
+			}
+		desc->RemoveEscapeSequence(separators.set_separator);
+
+		break;
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( ! val->val.vector_val.size )
+			{
+			desc->Add(separators.empty_field);
+			break;
+			}
+
+		desc->AddEscapeSequence(separators.set_separator);
+		for ( int j = 0; j < val->val.vector_val.size; j++ )
+			{
+			if ( j > 0 )
+				desc->AddRaw(separators.set_separator);
+
+			if ( ! Describe(desc, val->val.vector_val.vals[j], name) )
+				{
+				desc->RemoveEscapeSequence(separators.set_separator);
+				return false;
+				}
+			}
+		desc->RemoveEscapeSequence(separators.set_separator);
+
+		break;
+		}
+
+	default:
+		thread->Error(thread->Fmt("unsupported field format %d for %s", val->type, name.c_str()));
+		return false;
+	}
+
+	return true;
+	}
+
+
+threading::Value* AsciiFormatter::ParseValue(string s, string name, TypeTag type, TypeTag subtype) const
+	{
+	if ( s.compare(separators.unset_field) == 0 )  // field is not set...
+		return new threading::Value(type, false);
+
+	threading::Value* val = new threading::Value(type, true);
+	char* end = 0;
+	errno = 0;
+
+	switch ( type ) {
+	case TYPE_ENUM:
+	case TYPE_STRING:
+		s = get_unescaped_string(s);
+		val->val.string_val.length = s.size();
+		val->val.string_val.data = copy_string(s.c_str());
+		break;
+
+	case TYPE_BOOL:
+		if ( s == "T" )
+			val->val.int_val = 1;
+		else if ( s == "F" )
+			val->val.int_val = 0;
+		else
+			{
+			thread->Error(thread->Fmt("Field: %s Invalid value for boolean: %s",
+				  name.c_str(), s.c_str()));
+			goto parse_error;
+			}
+		break;
+
+	case TYPE_INT:
+		val->val.int_val = strtoll(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			goto parse_error;
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		val->val.double_val = strtod(s.c_str(), &end);
+		if ( CheckNumberError(s, end) )
+			goto parse_error;
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+		val->val.uint_val = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			goto parse_error;
+		break;
+
+	case TYPE_PORT:
+		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) )
+			goto parse_error;
+
+		val->val.port_val.proto = TRANSPORT_UNKNOWN;
+		break;
+
+	case TYPE_SUBNET:
+		{
+		s = get_unescaped_string(s);
+		size_t pos = s.find("/");
+		if ( pos == s.npos )
+			{
+			thread->Error(thread->Fmt("Invalid value for subnet: %s", s.c_str()));
+			goto parse_error;
+			}
+
+		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
+
+		if ( CheckNumberError(s, end) )
+			goto parse_error;
+
+		string addr = s.substr(0, pos);
+
+		val->val.subnet_val.prefix = ParseAddr(addr);
+		val->val.subnet_val.length = width;
+		break;
+		}
+
+	case TYPE_ADDR:
+		s = get_unescaped_string(s);
+		val->val.addr_val = ParseAddr(s);
+		break;
+
+	case TYPE_TABLE:
+	case TYPE_VECTOR:
+		// First - common initialization
+		// Then - initialization for table.
+		// Then - initialization for vector.
+		// Then - common stuff
+		{
+		// how many entries do we have...
+		unsigned int length = 1;
+		for ( unsigned int i = 0; i < s.size(); i++ )
+			{
+			if ( s[i] == separators.set_separator[0] )
+				length++;
+			}
+
+		unsigned int pos = 0;
+		bool error = false;
+
+		if ( separators.empty_field.size() > 0 && s.compare(separators.empty_field) == 0 )
+			length = 0;
+
+		threading::Value** lvals = new threading::Value* [length];
+
+		if ( type == TYPE_TABLE )
+			{
+			val->val.set_val.vals = lvals;
+			val->val.set_val.size = length;
+			}
+
+		else if ( type == TYPE_VECTOR )
+			{
+			val->val.vector_val.vals = lvals;
+			val->val.vector_val.size = length;
+			}
+
+		else
+			assert(false);
+
+		if ( length == 0 )
+			break; //empty
+
+		istringstream splitstream(s);
+		while ( splitstream )
+			{
+			string element;
+
+			if ( ! getline(splitstream, element, separators.set_separator[0]) )
+				break;
+
+			if ( pos >= length )
+				{
+				thread->Error(thread->Fmt("Internal error while parsing set. pos %d >= length %d."
+				          " Element: %s", pos, length, element.c_str()));
+				error = true;
+				break;
+				}
+
+			threading::Value* newval = ParseValue(element, name, subtype);
+			if ( newval == 0 )
+				{
+				thread->Error("Error while reading set or vector");
+				error = true;
+				break;
+				}
+
+			lvals[pos] = newval;
+
+			pos++;
+			}
+
+		// Test if the string ends with a set_separator... or if the
+		// complete string is empty. In either of these cases we have
+		// to push an empty val on top of it.
+		if ( ! error && (s.empty() || *s.rbegin() == separators.set_separator[0]) )
+			{
+			lvals[pos] = ParseValue("", name, subtype);
+			if ( lvals[pos] == 0 )
+				{
+				thread->Error("Error while trying to add empty set element");
+				goto parse_error;
+				}
+
+			pos++;
+			}
+
+		if ( error ) {
+			// We had an error while reading a set or a vector.
+			// Hence we have to clean up the values that have
+			// been read so far
+			for ( unsigned int i = 0; i < pos; i++ )
+				delete lvals[i];
+
+			goto parse_error;
+		}
+
+		if ( pos != length )
+			{
+			thread->Error(thread->Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
+			goto parse_error;
+			}
+
+		break;
+		}
+
+	default:
+		thread->Error(thread->Fmt("unsupported field format %d for %s", type,
+		name.c_str()));
+		goto parse_error;
+	}
+
+	return val;
+
+parse_error:
+	delete val;
+	return 0;
+	}
+
+bool AsciiFormatter::CheckNumberError(const string& s, const char* end) const
+	{
+	// Do this check first, before executing s.c_str() or similar.
+	// otherwise the value to which *end is pointing at the moment might
+	// be gone ...
+	bool endnotnull =  (*end != '\0');
+
+	if ( s.length() == 0 )
+		{
+		thread->Error("Got empty string for number field");
+		return true;
+		}
+
+	if ( end == s.c_str() ) {
+		thread->Error(thread->Fmt("String '%s' contained no parseable number", s.c_str()));
+		return true;
+	}
+
+	if ( endnotnull )
+		thread->Warning(thread->Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+
+	if ( errno == EINVAL )
+		{
+		thread->Error(thread->Fmt("String '%s' could not be converted to a number", s.c_str()));
+		return true;
+		}
+
+	else if ( errno == ERANGE )
+		{
+		thread->Error(thread->Fmt("Number '%s' out of supported range.", s.c_str()));
+		return true;
+		}
+
+	return false;
+	}
+
+string AsciiFormatter::Render(const threading::Value::addr_t& addr) const
+	{
+	if ( addr.family == IPv4 )
+		{
+		char s[INET_ADDRSTRLEN];
+
+		if ( ! bro_inet_ntop(AF_INET, &addr.in.in4, s, INET_ADDRSTRLEN) )
+			return "<bad IPv4 address conversion>";
+		else
+			return s;
+		}
+	else
+		{
+		char s[INET6_ADDRSTRLEN];
+
+		if ( ! bro_inet_ntop(AF_INET6, &addr.in.in6, s, INET6_ADDRSTRLEN) )
+			return "<bad IPv6 address conversion>";
+		else
+			return s;
+		}
+	}
+
+TransportProto AsciiFormatter::ParseProto(const string &proto) const
+	{
+	if ( proto == "unknown" )
+		return TRANSPORT_UNKNOWN;
+	else if ( proto == "tcp" )
+		return TRANSPORT_TCP;
+	else if ( proto == "udp" )
+		return TRANSPORT_UDP;
+	else if ( proto == "icmp" )
+		return TRANSPORT_ICMP;
+
+	thread->Error(thread->Fmt("Tried to parse invalid/unknown protocol: %s", proto.c_str()));
+
+	return TRANSPORT_UNKNOWN;
+	}
+
+
+// More or less verbose copy from IPAddr.cc -- which uses reporter.
+threading::Value::addr_t AsciiFormatter::ParseAddr(const string &s) const
+	{
+		threading::Value::addr_t val;
+
+	if ( s.find(':') == std::string::npos ) // IPv4.
+		{
+		val.family = IPv4;
+
+		if ( inet_aton(s.c_str(), &(val.in.in4)) <= 0 )
+			{
+			thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
+			memset(&val.in.in4.s_addr, 0, sizeof(val.in.in4.s_addr));
+			}
+		}
+
+	else
+		{
+		val.family = IPv6;
+		if ( inet_pton(AF_INET6, s.c_str(), val.in.in6.s6_addr) <=0 )
+			{
+			thread->Error(thread->Fmt("Bad address: %s", s.c_str()));
+			memset(val.in.in6.s6_addr, 0, sizeof(val.in.in6.s6_addr));
+			}
+		}
+
+	return val;
+	}
+
+string AsciiFormatter::Render(const threading::Value::subnet_t& subnet) const
+	{
+	char l[16];
+
+	if ( subnet.prefix.family == IPv4 )
+		modp_uitoa10(subnet.length - 96, l);
+	else
+		modp_uitoa10(subnet.length, l);
+
+	string s = Render(subnet.prefix) + "/" + l;
+
+	return s;
+	}
+
+string AsciiFormatter::Render(double d) const
+	{
+	char buf[256];
+	modp_dtoa(d, buf, 6);
+	return buf;
+	}
+

src/threading/AsciiFormatter.h

@@ -0,0 +1,142 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#ifndef THREADING_ASCII_FORMATTER_H
+#define THREADING_ASCII_FORMATTER_H
+
+#include "../Desc.h"
+#include "MsgThread.h"
+
+/**
+  * A thread-safe class for converting values into a readable ASCII
+  * representation, and vice versa. This is a utility class that factors out
+  * common rendering/parsing code needed by a number of input/output threads.
+  */
+class AsciiFormatter {
+public:
+	/**
+	 * A struct to pass the necessary configuration values to the
+	 * AsciiFormatter module on initialization.
+	 */
+	struct SeparatorInfo
+		{
+		string set_separator;	// Separator between set elements.
+		string unset_field;	// String marking an unset field.
+		string empty_field;	// String marking an empty (but set) field.
+
+		/**
+		 * Constructor that leaves separators etc unset to dummy
+		 * values. Useful if you use only methods that don't need any
+		 * of them, like StringToAddr, etc.
+		 */
+		SeparatorInfo();
+
+		/**
+		 * Constructor that defines all the configuration options.
+		 * Use if you need either ValToODesc or EntryToVal.
+		 */
+		SeparatorInfo(const string& set_separator, const string& unset_field, const string& empty_field);
+		};
+
+	/**
+	 * Constructor.
+	 *
+	 * @param t The thread that uses this class instance. The class uses
+	 * some of the thread's methods, e.g., for error reporting and
+	 * internal formatting.
+	 *
+	 * @param info SeparatorInfo structure defining the necessary
+	 * separators.
+	 */
+	AsciiFormatter(threading::MsgThread* t, const SeparatorInfo info);
+
+	/**
+	 * Destructor.
+	 */
+	~AsciiFormatter();
+
+	/**
+	 * Convert a threading value into a corresponding ASCII.
+	 * representation.
+	 *
+	 * @param desc The ODesc object to write to.
+	 *
+	 * @param val the Value to render to the ODesc object.
+	 *
+	 * @param The name of a field associated with the value. Used only
+	 * for error reporting.
+	 *
+	 * @return Returns true on success, false on error. Errors are also
+	 * flagged via the reporter.
+	 */
+	bool Describe(ODesc* desc, threading::Value* val, const string& name) const;
+
+	/**
+	 * Convert an IP address into a string.
+	 *
+	 * @param addr The address.
+	 *
+	 * @return An ASCII representation of the address.
+	 */
+	string Render(const threading::Value::addr_t& addr) const;
+
+	/**
+	 * Convert an subnet value into a string.
+	 *
+	 * @param addr The address.
+	 *
+	 * @return An ASCII representation of the subnet.
+	 */
+	string Render(const threading::Value::subnet_t& subnet) const;
+
+	/**
+	 * Convert a double into a string. This renders the double with Bro's
+	 * standard precision.
+	 *
+	 * @param d The double.
+	 *
+	 * @return An ASCII representation of the double.
+	 */
+	string Render(double d) const;
+
+	/**
+	 * Convert the ASCII representation of a field into a value.
+	 *
+	 * @param s The string to parse.
+	 *
+	 * @param The name of a field associated with the value. Used only
+	 * for error reporting.
+	 *
+	 * @return The new value, or null on error. Errors are also flagged
+	 * via the reporter.
+	 */
+	threading::Value* ParseValue(string s, string name, TypeTag type, TypeTag subtype = TYPE_ERROR) const;
+
+	/**
+	 * Convert a string into a TransportProto. The string must be one of
+	 * \c tcp, \c udp, \c icmp, or \c unknown.
+	 *
+	 * @param proto The transport protocol
+	 *
+	 * @return The transport protocol, which will be \c TRANSPORT_UNKNOWN
+	 * on error. Errors are also flagged via the reporter.
+	 */
+	TransportProto ParseProto(const string &proto) const;
+
+	/**
+	 * Convert a string into a Value::addr_t.
+	 *
+	 * @param addr String containing an IPv4 or IPv6 address.
+	 *
+	 * @return The address, which will be all-zero on error. Errors are
+	 * also flagged via the reporter.
+	 */
+	threading::Value::addr_t ParseAddr(const string &addr) const;
+
+private:
+	bool CheckNumberError(const string& s, const char * end) const;
+
+	SeparatorInfo separators;
+	threading::MsgThread* thread;
+};
+
+#endif /* THREADING_ASCII_FORMATTER_H */

src/threading/BasicThread.h

@@ -199,10 +199,6 @@ private:
 	bool terminating;	// Set to to true to signal termination.
 	bool killed;	// Set to true once forcefully killed.
 
-	// Used as a semaphore to tell the pthread thread when it may
-	// terminate.
-	pthread_mutex_t terminate;
-
 	// For implementing Fmt().
 	char* buf;
 	unsigned int buf_len;

src/threading/MsgThread.h

@@ -400,7 +400,7 @@ private:
 };
 
 /**
- * A paremeterized OututMessage that stores a pointer to an argument object.
+ * A parameterized OutputMessage that stores a pointer to an argument object.
  * Normally, the objects will be used from the Process() callback.
  */
 template<typename O>

testing/btest/Baseline/core.reporter-error-in-handler/output

@@ -1,3 +1,3 @@
-error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28: no such index (a[1])
-ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
 1st error printed on script level

testing/btest/Baseline/core.reporter-runtime-error/output

@@ -1,2 +1,2 @@
-error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
-ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9: no such index (a[2])

testing/btest/Baseline/core.reporter/logger-test.log

@@ -1,6 +1,6 @@
-reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_info|init test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_warning|init test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_error|init test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_info|done test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_warning|done test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_error|done test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000

testing/btest/Baseline/core.reporter/output

@@ -1,7 +1,9 @@
-/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
-WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
+/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8: init test-info
-ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
+warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9: init test-warning
-WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10: init test-error
-ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
+/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15: done test-info
+warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16: done test-warning
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17: done test-error

testing/btest/Baseline/language.record-ceorce-orphan/out

@@ -0,0 +1,2 @@
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 19: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; }))
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 21: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; }))

testing/btest/Baseline/language.switch-incomplete/out

@@ -0,0 +1 @@
+error in /home/robin/bro/master/testing/btest/.tmp/language.switch-incomplete/switch-incomplete.bro, lines 7-8: case block must end in break/fallthrough/return statement (case 1:{ print 1})

testing/btest/Baseline/language.uninitialized-local/out

@@ -0,0 +1,2 @@
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.uninitialized-local/uninitialized-local.bro, line 16: value used but not set (my_string)
+Continuing

testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline

@@ -1,8 +1,8 @@
-1355266097.683599 error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range.
+error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range.
-1355266097.683599 error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800	121218446744073709551612' to Val. Ignoring line.
+error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800	121218446744073709551612' to Val. Ignoring line.
-1355266097.683599 warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
+warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
-1355266097.683599 warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
+warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
-1355266097.683599 error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
+error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
-1355266097.683599 error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Ignoring line.
+error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Ignoring line.
-1355266097.683599 received termination signal
+received termination signal
 >>>

testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline

@@ -1,4 +1,4 @@
-1355265853.593476 error: ../input.log/Input::READER_ASCII: String 'l' contained no parseable number
+error: ../input.log/Input::READER_ASCII: String 'l' contained no parseable number
-1355265853.593476 error: ../input.log/Input::READER_ASCII: Could not convert line '	l' to Val. Ignoring line.
+error: ../input.log/Input::READER_ASCII: Could not convert line '	l' to Val. Ignoring line.
-1355265853.593476 received termination signal
+received termination signal
 >>>

testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr

@@ -1,5 +1,5 @@
-1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat
+error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat
-1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: Init failed
+error: does-not-exist.dat/Input::READER_ASCII: Init failed
-1355265996.626106 warning: Stream input is already queued for removal. Ignoring remove.
+warning: Stream input is already queued for removal. Ignoring remove.
-1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: terminating thread
+error: does-not-exist.dat/Input::READER_ASCII: terminating thread
-1355265996.626106 received termination signal
+received termination signal

testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr

@@ -1 +1 @@
-ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9)
+error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9: no such index (test[3])

testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log

@@ -3,8 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
-#open	2012-08-10-20-09-23
+#open	2013-01-18-18-29-30
 #fields	ts	level	message	location
 #types	time	enum	string	string
-0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
+0.000000	Reporter::ERROR	no such index (test[3])	/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
-#close	2012-08-10-20-09-23
+#close	2013-01-18-18-29-30

testing/btest/Makefile

@@ -6,13 +6,13 @@ all: cleanup btest-verbose coverage
 
 # Showing all tests.
 btest-verbose:
-	@$(BTEST) -j 5 -f $(DIAG)
+	@$(BTEST) -j -f $(DIAG)
 
 brief: cleanup btest-brief coverage
 
 # Brief output showing only failed tests.
 btest-brief:
-	@$(BTEST) -j 5 -b -f $(DIAG)
+	@$(BTEST) -j -b -f $(DIAG)
 
 coverage:
 	@../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts

testing/btest/core/leaks/string-indexing.bro

@@ -11,16 +11,16 @@ event new_connection(c: connection)
 	{
 	local s = "0123456789";
 	print s[1];
-	print s[1,2];
+	print s[1:2];
-	print s[1,6];
+	print s[1:6];
-	print s[0,20];
+	print s[0:20];
 	print s[-2];
-	print s[-3,-1];
+	print s[-3:1];
-	print s[-1,-10];
+	print s[-1:10];
-	print s[-1,0];
+	print s[-1:0];
-	print s[-1,5];
+	print s[-1:5];
-	print s[20, 23];
+	print s[20:23];
-	print s[-20, 23];
+	print s[-20:23];
-	print s[0,5][2];
+	print s[0:5][2];
-	print s[0,5][1,3][0];
+	print s[0:5][1:3][0];
 	}

testing/btest/core/leaks/switch-statement.bro

@@ -148,17 +148,19 @@ function switch_empty(v: count): string
 	return "n/a";
 	}
 
-function switch_break(v: count): string
+function switch_fallthrough(v: count): string
 	{
 	local rval = "";
 	switch ( v ) {
 	case 1:
 		rval += "test";
+		fallthrough;
 	case 2:
 		rval += "testing";
-		break;
+		fallthrough;
 	case 3:
 		rval += "tested";
+		break;
 	}
 	return rval + "return";
 	}
@@ -169,12 +171,16 @@ function switch_default(v: count): string
 	switch ( v ) {
 	case 1:
 		rval += "1";
+		fallthrough;
 	case 2:
 		rval += "2";
+		break;
 	case 3:
 		rval += "3";
+		fallthrough;
 	default:
 		rval += "d";
+		break;
 	}
 	return rval + "r";
 	}
@@ -185,13 +191,16 @@ function switch_default_placement(v: count): string
 	switch ( v ) {
 	case 1:
 		rval += "1";
+		fallthrough;
 	default:
 		rval += "d";
+		fallthrough;
 	case 2:
 		rval += "2";
 		break;
 	case 3:
 		rval += "3";
+		break;
 	}
 	return rval + "r";
 	}
@@ -251,11 +260,11 @@ event new_connection(c: connection)
 	test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" );
 	test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" );
 	test_switch( switch_empty(2) , "n/a" );
-	test_switch( switch_break(1) , "testtestingreturn" );
+	test_switch( switch_fallthrough(1) , "testtestingtestedreturn" );
-	test_switch( switch_break(2) , "testingreturn" );
+	test_switch( switch_fallthrough(2) , "testingtestedreturn" );
-	test_switch( switch_break(3) , "testedreturn" );
+	test_switch( switch_fallthrough(3) , "testedreturn" );
-	test_switch( switch_default(1) , "123dr" );
+	test_switch( switch_default(1) , "12r" );
-	test_switch( switch_default(2) , "23dr" );
+	test_switch( switch_default(2) , "2r" );
 	test_switch( switch_default(3) , "3dr" );
 	test_switch( switch_default(4) , "dr" );
 	test_switch( switch_default_placement(1) , "1d2r" );

testing/btest/language/record-ceorce-orphan.bro

@@ -0,0 +1,22 @@
+# @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+
+type myrec: record {
+	a: string;
+	b: count;
+	c: interval &optional;
+};
+
+function myfunc(rec: myrec)
+	{
+	print rec;
+	}
+
+event bro_init()
+	{
+	# Orhpaned fields in a record coercion reflect a programming error, like a typo, so should
+	# be reported at parse-time to prevent unexpected run-time behavior.
+	local rec: myrec = [$a="test", $b=42, $wtf=1sec];
+	print rec;
+	myfunc([$a="test", $b=42, $wtf=1sec]);
+	}

testing/btest/language/string-indexing.bro

@@ -3,15 +3,15 @@
 
 local s = "0123456789";
 print s[1];
-print s[1,2];
+print s[1:2];
-print s[1,6];
+print s[1:6];
-print s[0,20];
+print s[0:20];
 print s[-2];
-print s[-3,-1];
+print s[-3:-1];
-print s[-1,-10];
+print s[-1:-10];
-print s[-1,0];
+print s[-1:0];
-print s[-1,5];
+print s[-1:5];
-print s[20, 23];
+print s[20:23];
-print s[-20, 23];
+print s[-20:23];
-print s[0,5][2];
+print s[0:5][2];
-print s[0,5][1,3][0];
+print s[0:5][1:3][0];

testing/btest/language/switch-incomplete.bro

@@ -0,0 +1,12 @@
+# @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1
+# @TEST-EXEC:      TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+
+event bro_init()
+	{
+	switch ( 1 ) {
+	case 1:
+		print 1;
+		# error: neither break/fallthrough/return here.
+	}
+	}
+

testing/btest/language/switch-statement.bro

@@ -143,17 +143,19 @@ function switch_empty(v: count): string
 	return "n/a";
 	}
 
-function switch_break(v: count): string
+function switch_fallthrough(v: count): string
 	{
 	local rval = "";
 	switch ( v ) {
 	case 1:
 		rval += "test";
+		fallthrough;
 	case 2:
 		rval += "testing";
-		break;
+		fallthrough;
 	case 3:
 		rval += "tested";
+		break;
 	}
 	return rval + "return";
 	}
@@ -164,12 +166,16 @@ function switch_default(v: count): string
 	switch ( v ) {
 	case 1:
 		rval += "1";
+		fallthrough;
 	case 2:
 		rval += "2";
+		break;
 	case 3:
 		rval += "3";
+		fallthrough;
 	default:
 		rval += "d";
+		break;
 	}
 	return rval + "r";
 	}
@@ -180,13 +186,16 @@ function switch_default_placement(v: count): string
 	switch ( v ) {
 	case 1:
 		rval += "1";
+		fallthrough;
 	default:
 		rval += "d";
+		fallthrough;
 	case 2:
 		rval += "2";
 		break;
 	case 3:
 		rval += "3";
+		break;
 	}
 	return rval + "r";
 	}
@@ -246,11 +255,11 @@ event bro_init()
 	test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" );
 	test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" );
 	test_switch( switch_empty(2) , "n/a" );
-	test_switch( switch_break(1) , "testtestingreturn" );
+	test_switch( switch_fallthrough(1) , "testtestingtestedreturn" );
-	test_switch( switch_break(2) , "testingreturn" );
+	test_switch( switch_fallthrough(2) , "testingtestedreturn" );
-	test_switch( switch_break(3) , "testedreturn" );
+	test_switch( switch_fallthrough(3) , "testedreturn" );
-	test_switch( switch_default(1) , "123dr" );
+	test_switch( switch_default(1) , "12r" );
-	test_switch( switch_default(2) , "23dr" );
+	test_switch( switch_default(2) , "2r" );
 	test_switch( switch_default(3) , "3dr" );
 	test_switch( switch_default(4) , "dr" );
 	test_switch( switch_default_placement(1) , "1d2r" );

testing/btest/language/uninitialized-local.bro

@@ -0,0 +1,23 @@
+# @TEST-EXEC: bro -b %INPUT >out 2>&1
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+
+event testit() &priority=10
+	{
+	local my_count: count = 10;
+	}
+
+event testit()
+	{
+	# my_string's value occupies same Frame offset as my_count's from above
+	# handler, but execution of this handler body should still "initialize"
+	# it to a null value instead of referring to a left-over value of my_count.
+	local my_string: string;
+	local my_vector: vector of string;
+	my_vector[0] = my_string;
+	print "Continuing";
+	}
+
+event bro_init()
+	{
+	event testit();
+	}

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -1,7 +1,9 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
+redef exit_only_after_terminate = T;
+
 @TEST-START-FILE input.log
 #separator \x09
 #path	ssh

testing/btest/scripts/base/frameworks/input/bignumber.bro

@@ -1,7 +1,9 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
+redef exit_only_after_terminate = T; 
+
 @TEST-START-FILE input.log
 #separator \x09
 #fields	i	c

testing/btest/scripts/base/frameworks/input/binary.bro

@@ -1,7 +1,9 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
+redef exit_only_after_terminate = T;
+
 redef InputAscii::separator = "|";
 redef InputAscii::set_separator = ",";
 redef InputAscii::empty_field = "(empty)";

testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: cp input1.log input.log
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: cp input2.log input.log
 # @TEST-EXEC: btest-bg-wait -k 5
@@ -20,6 +20,10 @@
 2	TEST	TEST
 @TEST-END-FILE
 
+@load base/frameworks/communication  # let network-time run
+
+redef exit_only_after_terminate = T;
+
 module A;
 
 type Idx: record {

testing/btest/scripts/base/frameworks/input/emptyvals.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -11,6 +11,8 @@ T	1
 -	2
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/event.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -16,6 +16,8 @@
 7	T
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 module A;

testing/btest/scripts/base/frameworks/input/executeraw.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out
 # @TEST-EXEC: btest-diff out
@@ -14,6 +14,8 @@ sdf
 3rw43wRRERLlL#RWERERERE.
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 type Val: record {

testing/btest/scripts/base/frameworks/input/invalidnumbers.bro

@@ -1,8 +1,8 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
-# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff .stderrwithoutfirstline
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
 
 @TEST-START-FILE input.log
 #separator \x09
@@ -14,6 +14,8 @@ Justtext	1
 9223372036854775800	-18446744073709551612
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 module A;

testing/btest/scripts/base/frameworks/input/invalidtext.bro

@@ -1,8 +1,8 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
-# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff .stderrwithoutfirstline
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
 
 @TEST-START-FILE input.log
 #separator \x09
@@ -12,6 +12,8 @@
 	5
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 module A;

testing/btest/scripts/base/frameworks/input/missing-file.bro

@@ -1,7 +1,9 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff bro/.stderr
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 global try: count;
 

testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -10,6 +10,8 @@
 T	-42	
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/onecolumn-record.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -10,6 +10,8 @@
 T	-42	
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/optional.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -16,6 +16,8 @@
 7	T
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/port.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -9,6 +9,8 @@
 1.2.3.6	30	unknown
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/predicate-stream.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 #
@@ -20,6 +20,8 @@
 7	T
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/predicate.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -16,6 +16,8 @@
 7	T
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/predicatemodify.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -11,6 +11,8 @@
 2	T	test2	idx2
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: cp input1.log input.log
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: cp input2.log input.log
 # @TEST-EXEC: sleep 2
@@ -55,6 +55,10 @@
 1	T	test1	idx1
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
+@load base/frameworks/communication  # let network-time run
+
 redef InputAscii::empty_field = "EMPTY";
 
 module A;

testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -14,6 +14,8 @@ need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	95	1342569600	1.228.83.
 need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.228.83.33	-	-	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	spamming;malware domain	public	http://reputation.alienvault.com/reputation.generic
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 
 redef InputAscii::empty_field = "EMPTY";

testing/btest/scripts/base/frameworks/input/raw.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -13,6 +13,8 @@ sdf
 3rw43wRRERLlL#RWERERERE.
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 global try: count;
 

testing/btest/scripts/base/frameworks/input/repeat.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
 
@@ -10,6 +10,8 @@
 1	T
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 global try: count;
 

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -1,5 +1,5 @@
 # @TEST-EXEC: cp input1.log input.log
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: sleep 2
 # @TEST-EXEC: cp input2.log input.log
 # @TEST-EXEC: sleep 2
@@ -56,7 +56,9 @@ F	-48	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz
 @TEST-END-FILE
 
 @load base/protocols/ssh
+@load base/frameworks/communication  # let network-time run
 
+redef exit_only_after_terminate = T;
 redef InputAscii::empty_field = "EMPTY";
 
 module A;

testing/btest/scripts/base/frameworks/input/rereadraw.bro

@@ -1,4 +1,4 @@
-# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
 # @TEST-EXEC: btest-bg-wait -k 5
 # @TEST-EXEC: btest-diff out
 
@@ -13,6 +13,8 @@ sdf
 3rw43wRRERLlL#RWERERERE.
 @TEST-END-FILE
 
+redef exit_only_after_terminate = T;
+
 global outfile: file;
 global try: count;