From 5607e86ad3e8349426d0205fc8867050079d24d4 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 12 Jul 2012 12:55:34 -0400
Subject: [PATCH 01/57] Reporter warnings and error now print to stderr by
 default.

- Changed the geoip warnings to Info.
---
 scripts/base/frameworks/reporter/main.bro | 26 +++++++++++++++++++----
 src/bro.bif                               |  4 ++--
 2 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro
index 3c19005364..8b45819442 100644
--- a/scripts/base/frameworks/reporter/main.bro
+++ b/scripts/base/frameworks/reporter/main.bro
@@ -11,7 +11,7 @@ module Reporter;
 export {
 	## The reporter logging stream identifier.
 	redef enum Log::ID += { LOG };
-
+	
 	## An indicator of reporter message severity.
 	type Level: enum { 
 		## Informational, not needing specific attention.
@@ -36,24 +36,42 @@ export {
 		## Not all reporter messages will have locations in them though.
 		location: string &log &optional;
 	};
+	
+	## Send reporter error messages to STDERR by default.  The option to 
+	## turn it off is presented here in case Bro is being run by some 
+	## external harness and shouldn't output anything to the console.
+	const errors_to_stderr = T &redef;
+	
+	## Send reporter warning messages to STDERR by default.  The option to 
+	## turn it off is presented here in case Bro is being run by some 
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
 }
 
+global stderr: file;
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Reporter::LOG, [$columns=Info]);
+	
+	if ( errors_to_stderr || warnings_to_stderr )
+		stderr = open("/dev/stderr");
 	}
 
-event reporter_info(t: time, msg: string, location: string)
+event reporter_info(t: time, msg: string, location: string) &priority=-5
 	{
 	Log::write(Reporter::LOG, [$ts=t, $level=INFO, $message=msg, $location=location]);
 	}
 	
-event reporter_warning(t: time, msg: string, location: string)
+event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
-event reporter_error(t: time, msg: string, location: string)
+event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( errors_to_stderr )
+		print stderr, fmt("ERROR: %s", msg);
+	
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}
diff --git a/src/bro.bif b/src/bro.bif
index f18d3ba1b5..1d4aeed4d6 100644
--- a/src/bro.bif
+++ b/src/bro.bif
@@ -3764,7 +3764,7 @@ static GeoIP* open_geoip_db(GeoIPDBTypes type)
 		geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE);
 
 	if ( ! geoip )
-		reporter->Warning("Failed to open GeoIP database: %s",
+		reporter->Info("Failed to open GeoIP database: %s",
 		                  GeoIPDBFileName[type]);
 	return geoip;
 	}
@@ -3804,7 +3804,7 @@ function lookup_location%(a: addr%) : geo_location
 			if ( ! geoip )
 				builtin_error("Can't initialize GeoIP City/Country database");
 			else
-				reporter->Warning("Fell back to GeoIP Country database");
+				reporter->Info("Fell back to GeoIP Country database");
 			}
 		else
 			have_city_db = true;

From 9829cf9a296b4f1c6614658b80225cd80f2e24ec Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 3 Aug 2012 10:44:46 -0700
Subject: [PATCH 02/57] Fixing little typo with big impact.

---
 CHANGES                      | 5 +++++
 VERSION                      | 2 +-
 src/logging/writers/Ascii.cc | 2 +-
 3 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 5267fa9f37..644a56d458 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,9 @@
 
+2.0-912 | 2012-08-03 10:44:46 -0700
+
+  * Fixing little typo with big impact. (Robin Sommer)
+
+
 2.1-beta | 2012-07-30 11:59:53 -0700
 
   * Improve log filter compatibility with remote logging. Addresses
diff --git a/VERSION b/VERSION
index 0fb956a360..f1cb181c5c 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta
+2.0-912
diff --git a/src/logging/writers/Ascii.cc b/src/logging/writers/Ascii.cc
index f6df3b9336..11b322f5a3 100644
--- a/src/logging/writers/Ascii.cc
+++ b/src/logging/writers/Ascii.cc
@@ -359,7 +359,7 @@ bool Ascii::DoWrite(int num_fields, const Field* const * fields,
 	if ( ! safe_write(fd, bytes, len) )
 		goto write_error;
 
-        if ( IsBuf() )
+        if ( ! IsBuf() )
 		fsync(fd);
 
 	return true;

From 10b671a6389ab0720a18ec6fb32be6e03ba6fa0b Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Fri, 3 Aug 2012 17:24:04 -0500
Subject: [PATCH 03/57] Add tests for untested BIFs

---
 testing/btest/Baseline/bifs.analyzer_name/out |  1 +
 testing/btest/Baseline/bifs.entropy_test/out  |  2 ++
 testing/btest/Baseline/bifs.global_sizes/out  |  1 +
 testing/btest/Baseline/bifs.identify_data/out |  4 ++++
 testing/btest/Baseline/bifs.strftime/out      |  4 ++++
 testing/btest/bifs/analyzer_name.bro          |  9 +++++++
 testing/btest/bifs/bro_version.bro            |  9 +++++++
 testing/btest/bifs/checkpoint_state.bro       | 10 ++++++++
 testing/btest/bifs/current_analyzer.bro       | 11 +++++++++
 testing/btest/bifs/current_time.bro           |  9 +++++++
 testing/btest/bifs/entropy_test.bro           | 24 +++++++++++++++++++
 testing/btest/bifs/gethostname.bro            |  9 +++++++
 testing/btest/bifs/getpid.bro                 |  9 +++++++
 testing/btest/bifs/global_sizes.bro           | 16 +++++++++++++
 testing/btest/bifs/identify_data.bro          | 16 +++++++++++++
 testing/btest/bifs/resource_usage.bro         |  9 +++++++
 testing/btest/bifs/strftime.bro               | 17 +++++++++++++
 17 files changed, 160 insertions(+)
 create mode 100644 testing/btest/Baseline/bifs.analyzer_name/out
 create mode 100644 testing/btest/Baseline/bifs.entropy_test/out
 create mode 100644 testing/btest/Baseline/bifs.global_sizes/out
 create mode 100644 testing/btest/Baseline/bifs.identify_data/out
 create mode 100644 testing/btest/Baseline/bifs.strftime/out
 create mode 100644 testing/btest/bifs/analyzer_name.bro
 create mode 100644 testing/btest/bifs/bro_version.bro
 create mode 100644 testing/btest/bifs/checkpoint_state.bro
 create mode 100644 testing/btest/bifs/current_analyzer.bro
 create mode 100644 testing/btest/bifs/current_time.bro
 create mode 100644 testing/btest/bifs/entropy_test.bro
 create mode 100644 testing/btest/bifs/gethostname.bro
 create mode 100644 testing/btest/bifs/getpid.bro
 create mode 100644 testing/btest/bifs/global_sizes.bro
 create mode 100644 testing/btest/bifs/identify_data.bro
 create mode 100644 testing/btest/bifs/resource_usage.bro
 create mode 100644 testing/btest/bifs/strftime.bro

diff --git a/testing/btest/Baseline/bifs.analyzer_name/out b/testing/btest/Baseline/bifs.analyzer_name/out
new file mode 100644
index 0000000000..84613e9dd1
--- /dev/null
+++ b/testing/btest/Baseline/bifs.analyzer_name/out
@@ -0,0 +1 @@
+PIA_TCP
diff --git a/testing/btest/Baseline/bifs.entropy_test/out b/testing/btest/Baseline/bifs.entropy_test/out
new file mode 100644
index 0000000000..08a09de4e4
--- /dev/null
+++ b/testing/btest/Baseline/bifs.entropy_test/out
@@ -0,0 +1,2 @@
+[entropy=4.715374, chi_square=591.981818, mean=75.472727, monte_carlo_pi=4.0, serial_correlation=-0.11027]
+[entropy=2.083189, chi_square=3906.018182, mean=69.054545, monte_carlo_pi=4.0, serial_correlation=0.849402]
diff --git a/testing/btest/Baseline/bifs.global_sizes/out b/testing/btest/Baseline/bifs.global_sizes/out
new file mode 100644
index 0000000000..76c40b297a
--- /dev/null
+++ b/testing/btest/Baseline/bifs.global_sizes/out
@@ -0,0 +1 @@
+found bro_init
diff --git a/testing/btest/Baseline/bifs.identify_data/out b/testing/btest/Baseline/bifs.identify_data/out
new file mode 100644
index 0000000000..a2872877f9
--- /dev/null
+++ b/testing/btest/Baseline/bifs.identify_data/out
@@ -0,0 +1,4 @@
+ASCII text, with no line terminators
+text/plain; charset=us-ascii
+PNG image data
+image/png; charset=binary
diff --git a/testing/btest/Baseline/bifs.strftime/out b/testing/btest/Baseline/bifs.strftime/out
new file mode 100644
index 0000000000..b32393b332
--- /dev/null
+++ b/testing/btest/Baseline/bifs.strftime/out
@@ -0,0 +1,4 @@
+1970-01-01 00:00:00
+000000 19700101
+1973-11-29 21:33:09
+213309 19731129
diff --git a/testing/btest/bifs/analyzer_name.bro b/testing/btest/bifs/analyzer_name.bro
new file mode 100644
index 0000000000..034344f5c4
--- /dev/null
+++ b/testing/btest/bifs/analyzer_name.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = 1;
+	print analyzer_name(a);
+	}
diff --git a/testing/btest/bifs/bro_version.bro b/testing/btest/bifs/bro_version.bro
new file mode 100644
index 0000000000..7465cbc0f5
--- /dev/null
+++ b/testing/btest/bifs/bro_version.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = bro_version();
+	if ( |a| == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/checkpoint_state.bro b/testing/btest/bifs/checkpoint_state.bro
new file mode 100644
index 0000000000..2a66bd1729
--- /dev/null
+++ b/testing/btest/bifs/checkpoint_state.bro
@@ -0,0 +1,10 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: test -f .state/state.bst
+
+event bro_init()
+	{
+	local a = checkpoint_state();
+	if ( a != T )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/current_analyzer.bro b/testing/btest/bifs/current_analyzer.bro
new file mode 100644
index 0000000000..45b495c046
--- /dev/null
+++ b/testing/btest/bifs/current_analyzer.bro
@@ -0,0 +1,11 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = current_analyzer();
+	if ( a != 0 )
+		exit(1);
+
+	# TODO: add a test for non-zero return value
+	}
diff --git a/testing/btest/bifs/current_time.bro b/testing/btest/bifs/current_time.bro
new file mode 100644
index 0000000000..5d16df396d
--- /dev/null
+++ b/testing/btest/bifs/current_time.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = current_time();
+	if ( a <= double_to_time(0) )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/entropy_test.bro b/testing/btest/bifs/entropy_test.bro
new file mode 100644
index 0000000000..ca01c79ed7
--- /dev/null
+++ b/testing/btest/bifs/entropy_test.bro
@@ -0,0 +1,24 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = "dh3Hie02uh^s#Sdf9L3frd243h$d78r2G4cM6*Q05d(7rh46f!0|4-f";
+	if ( entropy_test_init(1) != T )
+		exit(1);
+
+	if ( entropy_test_add(1, a) != T )
+		exit(1);
+
+	print entropy_test_finish(1);
+
+	local b = "0011000aaabbbbcccc000011111000000000aaaabbbbcccc0000000";
+	if ( entropy_test_init(2) != T )
+		exit(1);
+
+	if ( entropy_test_add(2, b) != T )
+		exit(1);
+
+	print entropy_test_finish(2);
+	}
diff --git a/testing/btest/bifs/gethostname.bro b/testing/btest/bifs/gethostname.bro
new file mode 100644
index 0000000000..97af719745
--- /dev/null
+++ b/testing/btest/bifs/gethostname.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = gethostname();
+	if ( |a| == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/getpid.bro b/testing/btest/bifs/getpid.bro
new file mode 100644
index 0000000000..98edc19a44
--- /dev/null
+++ b/testing/btest/bifs/getpid.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = getpid();
+	if ( a == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/global_sizes.bro b/testing/btest/bifs/global_sizes.bro
new file mode 100644
index 0000000000..4862db318b
--- /dev/null
+++ b/testing/btest/bifs/global_sizes.bro
@@ -0,0 +1,16 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local a = global_sizes();
+	for ( i in a )
+		{
+		# the table is quite large, so just look for one item we expect
+		if ( i == "bro_init" )
+			print "found bro_init";
+
+		}
+
+	}
diff --git a/testing/btest/bifs/identify_data.bro b/testing/btest/bifs/identify_data.bro
new file mode 100644
index 0000000000..11824b5e85
--- /dev/null
+++ b/testing/btest/bifs/identify_data.bro
@@ -0,0 +1,16 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	# plain text
+	local a = "This is a test";
+	print identify_data(a, F);
+	print identify_data(a, T);
+
+	# PNG image
+	local b = "\x89\x50\x4e\x47\x0d\x0a\x1a\x0a";
+	print identify_data(b, F);
+	print identify_data(b, T);
+	}
diff --git a/testing/btest/bifs/resource_usage.bro b/testing/btest/bifs/resource_usage.bro
new file mode 100644
index 0000000000..35f5b020d6
--- /dev/null
+++ b/testing/btest/bifs/resource_usage.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = resource_usage();
+	if ( a$version != bro_version() )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/strftime.bro b/testing/btest/bifs/strftime.bro
new file mode 100644
index 0000000000..31f9538632
--- /dev/null
+++ b/testing/btest/bifs/strftime.bro
@@ -0,0 +1,17 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	local f1 = "%Y-%m-%d %H:%M:%S";
+	local f2 = "%H%M%S %Y%m%d";
+
+	local a = double_to_time(0);
+	print strftime(f1, a);
+	print strftime(f2, a);
+
+	a = double_to_time(123456789);
+	print strftime(f1, a);
+	print strftime(f2, a);
+	}

From 18550ab009852059ecacc98b8035fc370a5e8fee Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sat, 4 Aug 2012 22:24:44 -0700
Subject: [PATCH 04/57] small bug in test script. Still worked, because the
 internal type checking let this through...

---
 testing/btest/scripts/base/frameworks/input/predicate.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro
index 2cda6f5fb9..fcd986c9a6 100644
--- a/testing/btest/scripts/base/frameworks/input/predicate.bro
+++ b/testing/btest/scripts/base/frameworks/input/predicate.bro
@@ -35,7 +35,7 @@ type Val: record {
 	b: bool;
 };
 
-global servers: table[int] of Val = table();
+global servers: table[int] of bool = table();
 
 event bro_init()
 	{

From a2b5028b58dee3dfd2759235a65a7c829ca40555 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sat, 4 Aug 2012 22:38:26 -0700
Subject: [PATCH 05/57] fix little sneaky bug in input framework with an edge
 case.

An assertion would trigger in the case when a predicate refuses
a new entry and another entry with the same index elements was
already in the table. (I thought that code block was unreachable
... did not think of this case).
---
 src/input/Manager.cc                          |  4 +-
 .../out                                       |  3 +
 .../input/predicaterefusesecondsamerecord.bro | 56 +++++++++++++++++++
 3 files changed, 60 insertions(+), 3 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 64e54f9333..3c29f14928 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1044,9 +1044,7 @@ int Manager::SendEntryTable(Stream* i, const Value* const *vals)
 
 			if ( ! updated )
 				{
-				// throw away. Hence - we quit. And remove the entry from the current dictionary...
-				// (but why should it be in there? assert this).
-				assert ( stream->currDict->RemoveEntry(idxhash) == 0 );
+				// just quit and delete everything we created.
 				delete idxhash;
 				delete h;
 				return stream->num_val_fields + stream->num_idx_fields;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out b/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out
new file mode 100644
index 0000000000..f752ff451a
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.predicaterefusesecondsamerecord/out
@@ -0,0 +1,3 @@
+{
+[1.228.83.33] = [asn=9318 HANARO-AS Hanaro Telecom Inc., severity=medium, confidence=95, detecttime=1342569600.0]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
new file mode 100644
index 0000000000..d572b30090
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro
@@ -0,0 +1,56 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+# Ok, this one tests a fun case.
+# Input file contains two lines mapping to the same index, but with different values,
+# where the predicate accepts the first one and refuses the second one.
+# Desired result -> first entry stays.
+
+@TEST-START-FILE input.log
+#fields	restriction	guid	severity	confidence	detecttime	address	protocol	portlist	asn	prefix	rir	cc	impact	description	alternativeid_restriction	alternativeid
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.0.17.227	-	-	2519 VECTANT VECTANT Ltd.	1.0.16.0/23	apnic	JP	spam infrastructure	spamming	public	http://reputation.alienvault.com/reputation.generic
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	95	1342569600	1.228.83.33	6	25	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	direct ube sources, spam operations & spam services	public	http://www.spamhaus.org/query/bl?ip=1.228.83.33
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.228.83.33	-	-	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	spamming;malware domain	public	http://reputation.alienvault.com/reputation.generic
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	address: addr;
+};
+
+type Val: record {
+	asn: string;
+	severity: string;
+	confidence: count;
+	detecttime: time;
+};
+
+global servers: table[addr] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers,
+				$pred(typ: Input::Event, left: Idx, right: Val) = { if ( right$confidence > 90 ) { return T; } return F; }
+				]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From bda8631f32d366128622fb474567573d54184d8f Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Tue, 7 Aug 2012 14:10:55 -0500
Subject: [PATCH 06/57] Add more BIF tests

---
 testing/btest/Baseline/bifs.capture_state_updates/out |  1 +
 testing/btest/Baseline/bifs.is_local_interface/out    |  4 ++++
 testing/btest/Baseline/bifs.reading_traces/out1       |  1 +
 testing/btest/Baseline/bifs.reading_traces/out2       |  1 +
 testing/btest/bifs/capture_state_updates.bro          |  9 +++++++++
 testing/btest/bifs/get_matcher_stats.bro              |  9 +++++++++
 testing/btest/bifs/is_local_interface.bro             | 11 +++++++++++
 testing/btest/bifs/reading_traces.bro                 | 10 ++++++++++
 8 files changed, 46 insertions(+)
 create mode 100644 testing/btest/Baseline/bifs.capture_state_updates/out
 create mode 100644 testing/btest/Baseline/bifs.is_local_interface/out
 create mode 100644 testing/btest/Baseline/bifs.reading_traces/out1
 create mode 100644 testing/btest/Baseline/bifs.reading_traces/out2
 create mode 100644 testing/btest/bifs/capture_state_updates.bro
 create mode 100644 testing/btest/bifs/get_matcher_stats.bro
 create mode 100644 testing/btest/bifs/is_local_interface.bro
 create mode 100644 testing/btest/bifs/reading_traces.bro

diff --git a/testing/btest/Baseline/bifs.capture_state_updates/out b/testing/btest/Baseline/bifs.capture_state_updates/out
new file mode 100644
index 0000000000..62a6e3c9df
--- /dev/null
+++ b/testing/btest/Baseline/bifs.capture_state_updates/out
@@ -0,0 +1 @@
+T
diff --git a/testing/btest/Baseline/bifs.is_local_interface/out b/testing/btest/Baseline/bifs.is_local_interface/out
new file mode 100644
index 0000000000..328bff6687
--- /dev/null
+++ b/testing/btest/Baseline/bifs.is_local_interface/out
@@ -0,0 +1,4 @@
+T
+F
+F
+T
diff --git a/testing/btest/Baseline/bifs.reading_traces/out1 b/testing/btest/Baseline/bifs.reading_traces/out1
new file mode 100644
index 0000000000..cf84443e49
--- /dev/null
+++ b/testing/btest/Baseline/bifs.reading_traces/out1
@@ -0,0 +1 @@
+F
diff --git a/testing/btest/Baseline/bifs.reading_traces/out2 b/testing/btest/Baseline/bifs.reading_traces/out2
new file mode 100644
index 0000000000..62a6e3c9df
--- /dev/null
+++ b/testing/btest/Baseline/bifs.reading_traces/out2
@@ -0,0 +1 @@
+T
diff --git a/testing/btest/bifs/capture_state_updates.bro b/testing/btest/bifs/capture_state_updates.bro
new file mode 100644
index 0000000000..3abfdffdc1
--- /dev/null
+++ b/testing/btest/bifs/capture_state_updates.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: test -f testfile
+
+event bro_init()
+	{
+	print capture_state_updates("testfile");
+	}
diff --git a/testing/btest/bifs/get_matcher_stats.bro b/testing/btest/bifs/get_matcher_stats.bro
new file mode 100644
index 0000000000..baee49fe1e
--- /dev/null
+++ b/testing/btest/bifs/get_matcher_stats.bro
@@ -0,0 +1,9 @@
+#
+# @TEST-EXEC: bro %INPUT
+
+event bro_init()
+	{
+	local a = get_matcher_stats();
+	if ( a$matchers == 0 )
+		exit(1);
+	}
diff --git a/testing/btest/bifs/is_local_interface.bro b/testing/btest/bifs/is_local_interface.bro
new file mode 100644
index 0000000000..8befdca385
--- /dev/null
+++ b/testing/btest/bifs/is_local_interface.bro
@@ -0,0 +1,11 @@
+#
+# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event bro_init()
+	{
+	print is_local_interface(127.0.0.1);
+	print is_local_interface(1.2.3.4);
+	print is_local_interface([2607::a:b:c:d]);
+	print is_local_interface([::1]);
+	}
diff --git a/testing/btest/bifs/reading_traces.bro b/testing/btest/bifs/reading_traces.bro
new file mode 100644
index 0000000000..fc83c50ccb
--- /dev/null
+++ b/testing/btest/bifs/reading_traces.bro
@@ -0,0 +1,10 @@
+
+# @TEST-EXEC: bro %INPUT >out1
+# @TEST-EXEC: btest-diff out1
+# @TEST-EXEC: bro -r $TRACES/web.trace %INPUT >out2
+# @TEST-EXEC: btest-diff out2
+
+event bro_init()
+	{
+	print reading_traces();
+	}

From 7c6b891b633c0c26298803d19b882b02f4a6f526 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 9 Aug 2012 13:46:58 -0400
Subject: [PATCH 07/57] Small improvements for printing reporter messages to
 STDERR.

---
 scripts/base/frameworks/reporter/main.bro | 27 +++++++++++++++++------
 1 file changed, 20 insertions(+), 7 deletions(-)

diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro
index 8b45819442..0248b82d10 100644
--- a/scripts/base/frameworks/reporter/main.bro
+++ b/scripts/base/frameworks/reporter/main.bro
@@ -37,15 +37,15 @@ export {
 		location: string &log &optional;
 	};
 	
-	## Send reporter error messages to STDERR by default.  The option to 
+	## Tunable for sending reporter warning messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
+
+	## Tunable for sending reporter error messages to STDERR.  The option to 
 	## turn it off is presented here in case Bro is being run by some 
 	## external harness and shouldn't output anything to the console.
 	const errors_to_stderr = T &redef;
-	
-	## Send reporter warning messages to STDERR by default.  The option to 
-	## turn it off is presented here in case Bro is being run by some 
-	## external harness and shouldn't output anything to the console.
-	const warnings_to_stderr = T &redef;
 }
 
 global stderr: file;
@@ -65,13 +65,26 @@ event reporter_info(t: time, msg: string, location: string) &priority=-5
 	
 event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( warnings_to_stderr )
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("WARNING: %s (%s)", msg, location);
+		}
+	
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
 event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
 	if ( errors_to_stderr )
-		print stderr, fmt("ERROR: %s", msg);
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("ERROR: %s (%s)", msg, location);
+		}
 	
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}

From cfe1402281eeb5fc935485f5e8c8082395820c29 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 9 Aug 2012 14:48:46 -0400
Subject: [PATCH 08/57] A couple of tests for printing reporter messages to
 STDERR.

---
 .../.stderr                                         |  0
 .../reporter.log                                    |  8 ++++++++
 .../scripts.base.frameworks.reporter.stderr/.stderr |  1 +
 .../reporter.log                                    |  8 ++++++++
 .../base/frameworks/reporter/disable-stderr.bro     | 13 +++++++++++++
 .../scripts/base/frameworks/reporter/stderr.bro     | 10 ++++++++++
 6 files changed, 40 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
 create mode 100644 testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
 create mode 100644 testing/btest/scripts/base/frameworks/reporter/stderr.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/.stderr
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
new file mode 100644
index 0000000000..5c6e795074
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
@@ -0,0 +1,8 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr
new file mode 100644
index 0000000000..78af1e7a73
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr
@@ -0,0 +1 @@
+ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
new file mode 100644
index 0000000000..4a00bb95ad
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
@@ -0,0 +1,8 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	reporter
+#fields	ts	level	message	location
+#types	time	enum	string	string
+0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
diff --git a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
new file mode 100644
index 0000000000..438e24d80b
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
@@ -0,0 +1,13 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+
+redef Reporter::warnings_to_stderr = F;
+redef Reporter::errors_to_stderr = F;
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}
\ No newline at end of file
diff --git a/testing/btest/scripts/base/frameworks/reporter/stderr.bro b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
new file mode 100644
index 0000000000..7ea748d94f
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
@@ -0,0 +1,10 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}
\ No newline at end of file

From 38912c182c0d6d051b3040fb1a206f102b65966e Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 10 Aug 2012 12:33:45 -0700
Subject: [PATCH 09/57] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 22120825f8..99e7a27431 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
+Subproject commit 99e7a274319619a94a421eb62537c7a5c184f71b
diff --git a/aux/bro-aux b/aux/bro-aux
index 941ee753f7..c691c01e9c 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
+Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
diff --git a/aux/broccoli b/aux/broccoli
index 5ff3e6a8e8..b3692a02ba 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
+Subproject commit b3692a02bae9a47d701d2d547e327dd429a86e76
diff --git a/aux/broctl b/aux/broctl
index 903108f6b4..84428286a1 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 903108f6b43ad228309713da880026d50add41f4
+Subproject commit 84428286a1980e21cafc4e066d95bf58f82a92b8

From d1c78d030045569fa1205e73a52aafe8483e9409 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 10 Aug 2012 13:10:24 -0700
Subject: [PATCH 10/57] Updating baselines.

---
 .../btest/Baseline/core.reporter-error-in-handler/output   | 3 ++-
 testing/btest/Baseline/core.reporter-runtime-error/output  | 2 +-
 testing/btest/Baseline/core.reporter/output                | 7 ++++---
 .../reporter.log                                           | 4 +++-
 .../scripts.base.frameworks.reporter.stderr/reporter.log   | 4 +++-
 .../scripts/base/frameworks/reporter/disable-stderr.bro    | 4 ++--
 testing/btest/scripts/base/frameworks/reporter/stderr.bro  | 4 ++--
 7 files changed, 17 insertions(+), 11 deletions(-)

diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index 83b310ab61..190631f4d1 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,2 +1,3 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+ERROR: no such index (a[1]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
+
 1st error printed on script level
diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output
index 59bcc3ac9b..94f7860cb4 100644
--- a/testing/btest/Baseline/core.reporter-runtime-error/output
+++ b/testing/btest/Baseline/core.reporter-runtime-error/output
@@ -1 +1 @@
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+ERROR: no such index (a[2]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output
index 2735adc931..b4f89bad2f 100644
--- a/testing/btest/Baseline/core.reporter/output
+++ b/testing/btest/Baseline/core.reporter/output
@@ -1,3 +1,4 @@
-/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
-warning in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
-error in /da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+WARNING: init test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
+ERROR: init test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
+WARNING: done test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
+ERROR: done test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
index 5c6e795074..144c094b2f 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.disable-stderr/reporter.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
+#open	2012-08-10-20-09-16
 #fields	ts	level	message	location
 #types	time	enum	string	string
-0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
+0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.disable-stderr/disable-stderr.bro, line 12
+#close	2012-08-10-20-09-16
diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
index 4a00bb95ad..b314bc45c3 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
+++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log
@@ -3,6 +3,8 @@
 #empty_field	(empty)
 #unset_field	-
 #path	reporter
+#open	2012-08-10-20-09-23
 #fields	ts	level	message	location
 #types	time	enum	string	string
-0.000000	Reporter::ERROR	no such index (test[3])	/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
+0.000000	Reporter::ERROR	no such index (test[3])	/da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9
+#close	2012-08-10-20-09-23
diff --git a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
index 438e24d80b..b1afb99b5c 100644
--- a/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
+++ b/testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro
@@ -1,6 +1,6 @@
 # @TEST-EXEC: bro %INPUT
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
-# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
 
 redef Reporter::warnings_to_stderr = F;
 redef Reporter::errors_to_stderr = F;
@@ -10,4 +10,4 @@ global test: table[count] of string = {};
 event bro_init()
 	{
 	print test[3];
-	}
\ No newline at end of file
+	}
diff --git a/testing/btest/scripts/base/frameworks/reporter/stderr.bro b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
index 7ea748d94f..ef01c9fdf9 100644
--- a/testing/btest/scripts/base/frameworks/reporter/stderr.bro
+++ b/testing/btest/scripts/base/frameworks/reporter/stderr.bro
@@ -1,10 +1,10 @@
 # @TEST-EXEC: bro %INPUT
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
-# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff reporter.log
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
 
 global test: table[count] of string = {};
 
 event bro_init()
 	{
 	print test[3];
-	}
\ No newline at end of file
+	}

From eee4fbf7ad2b8855f5d6a488b5a7b83bd75dfe9b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 10 Aug 2012 13:33:57 -0700
Subject: [PATCH 11/57] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 99e7a27431..22120825f8 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 99e7a274319619a94a421eb62537c7a5c184f71b
+Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
diff --git a/aux/bro-aux b/aux/bro-aux
index c691c01e9c..941ee753f7 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit c691c01e9cefae5a79bcd4b0f84ca387c8c587a7
+Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
diff --git a/aux/broccoli b/aux/broccoli
index b3692a02ba..5ff3e6a8e8 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit b3692a02bae9a47d701d2d547e327dd429a86e76
+Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
diff --git a/aux/broctl b/aux/broctl
index 84428286a1..6d0eb6083a 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 84428286a1980e21cafc4e066d95bf58f82a92b8
+Subproject commit 6d0eb6083acdc77e0a912bec0fb23df79b98da63

From 205ad78369701a5e67260b421411a52b28c45440 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Tue, 14 Aug 2012 15:09:38 -0400
Subject: [PATCH 12/57] Fix some problems in logs-to-elasticsearch.bro

---
 scripts/policy/tuning/logs-to-elasticsearch.bro | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/scripts/policy/tuning/logs-to-elasticsearch.bro b/scripts/policy/tuning/logs-to-elasticsearch.bro
index 207a9acc04..2a4b70362a 100644
--- a/scripts/policy/tuning/logs-to-elasticsearch.bro
+++ b/scripts/policy/tuning/logs-to-elasticsearch.bro
@@ -8,13 +8,13 @@ export {
 
 	## Optionally ignore any :bro:type:`Log::ID` from being sent to
 	## ElasticSearch with this script.
-	const excluded_log_ids: set[string] = set("Communication::LOG") &redef;
+	const excluded_log_ids: set[Log::ID] &redef;
 
 	## If you want to explicitly only send certain :bro:type:`Log::ID` 
 	## streams, add them to this set.  If the set remains empty, all will 
 	## be sent.  The :bro:id:`LogElasticSearch::excluded_log_ids` option will remain in 
 	## effect as well.
-	const send_logs: set[string] = set() &redef;
+	const send_logs: set[Log::ID] &redef;
 }
 
 event bro_init() &priority=-5
@@ -24,8 +24,8 @@ event bro_init() &priority=-5
 	
 	for ( stream_id in Log::active_streams )
 		{
-		if ( fmt("%s", stream_id) in excluded_log_ids ||
-		     (|send_logs| > 0 && fmt("%s", stream_id) !in send_logs) )
+		if ( stream_id in excluded_log_ids ||
+		     (|send_logs| > 0 && stream_id !in send_logs) )
 			next;
 
 		local filter: Log::Filter = [$name = "default-es",

From b13196cbf194419836f9b7627aab5cab25c47397 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 16 Aug 2012 09:24:25 -0400
Subject: [PATCH 13/57] Fixed more potential problems with deadlocked ES
 threads and signals from libcurl.

---
 src/logging/writers/ElasticSearch.cc | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc
index e688686b35..cb3248a044 100644
--- a/src/logging/writers/ElasticSearch.cc
+++ b/src/logging/writers/ElasticSearch.cc
@@ -371,7 +371,11 @@ bool ElasticSearch::HTTPSend(CURL *handle)
 	// The best (only?) way to disable that is to just use HTTP 1.0
 	curl_easy_setopt(handle, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_0);
 
-	//curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout);
+	// Some timeout options.  These will need more attention later.
+	curl_easy_setopt(handle, CURLOPT_NOSIGNAL, 1);
+	curl_easy_setopt(handle, CURLOPT_CONNECTTIMEOUT_MS, transfer_timeout);
+	curl_easy_setopt(handle, CURLOPT_TIMEOUT_MS, transfer_timeout*2);
+	curl_easy_setopt(handle, CURLOPT_DNS_CACHE_TIMEOUT, 60*60);
 
 	CURLcode return_code = curl_easy_perform(handle);
 

From 4da209d3b1fe6fa9c5118e055752843b2fb73a45 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Thu, 16 Aug 2012 11:48:56 -0700
Subject: [PATCH 14/57] Installing a handler for running out of memory in
 "new".

Bro will now print an error message in that case rather than abort
with an uncaught exception.
---
 CHANGES     | 6 ++++++
 VERSION     | 2 +-
 src/main.cc | 4 ++++
 src/util.cc | 8 +++++++-
 4 files changed, 18 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 08998ab9f4..f0c73ce8d9 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,10 @@
 
+2.1-beta-21 | 2012-08-16 11:48:56 -0700
+
+  * Installing a handler for running out of memory in "new". Bro will
+    now print an error message in that case rather than abort with an
+    uncaught exception. (Robin Sommer)
+
 2.1-beta-20 | 2012-08-16 11:43:31 -0700
 
   * Fixed potential problems with ElasticSearch output plugin. (Seth
diff --git a/VERSION b/VERSION
index c42c76c8ba..5d7a2a2cce 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta-20
+2.1-beta-21
diff --git a/src/main.cc b/src/main.cc
index 407f67c9af..5999186240 100644
--- a/src/main.cc
+++ b/src/main.cc
@@ -337,6 +337,8 @@ void terminate_bro()
 	delete log_mgr;
 	delete thread_mgr;
 	delete reporter;
+
+	reporter = 0;
 	}
 
 void termination_signal()
@@ -380,6 +382,8 @@ static void bro_new_handler()
 
 int main(int argc, char** argv)
 	{
+	std::set_new_handler(bro_new_handler);
+
 	brofiler.ReadStats();
 
 	bro_argc = argc;
diff --git a/src/util.cc b/src/util.cc
index 2d981e952e..3b6fcac76f 100644
--- a/src/util.cc
+++ b/src/util.cc
@@ -1383,7 +1383,13 @@ void safe_close(int fd)
 
 void out_of_memory(const char* where)
 	{
-	reporter->FatalError("out of memory in %s.\n", where);
+	fprintf(stderr, "out of memory in %s.\n", where);
+
+	if ( reporter )
+		// Guess that might fail here if memory is really tight ...
+		reporter->FatalError("out of memory in %s.\n", where);
+
+	abort();
 	}
 
 void get_memory_usage(unsigned int* total, unsigned int* malloced)

From a6f7fd9c874ffdab31c3c79c9956857617b723d5 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 16 Aug 2012 15:59:26 -0500
Subject: [PATCH 15/57] Fix memory leak of serialized IDs when compiled with
 --enable-debug.

When using --enable-debug, values keep track of the last identifier
to which they were bound by storing a ref'd ID pointer.  This could
lead to some circular dependencies in which an ID is never reclaimed
because the Val is bound to the ID and the ID is bound to the Val, with
both holding references to each other.

There might be more cases where this feature of --enable-debug caused
a leak, but it showed up in particular when running the
core.leaks.remote unit test due to the internal
SendID("peer_description") call during the handshake between remote
processes.  Other tests showed the send_id() BIF leaked more generally.

Tracking the ID last bound to a Val through just the identifier string
instead of a ref'd ID pointer fixes the leak.
---
 src/RemoteSerializer.cc |  5 -----
 src/Val.cc              |  2 +-
 src/Val.h               | 16 +++++++++-------
 3 files changed, 10 insertions(+), 13 deletions(-)

diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc
index cfd20eba39..564ad2be68 100644
--- a/src/RemoteSerializer.cc
+++ b/src/RemoteSerializer.cc
@@ -2897,11 +2897,6 @@ void RemoteSerializer::GotID(ID* id, Val* val)
 					(desc && *desc) ? desc : "not set"),
 			current_peer);
 
-#ifdef USE_PERFTOOLS_DEBUG
-		// May still be cached, but we don't care.
-		heap_checker->IgnoreObject(id);
-#endif
-
 		Unref(id);
 		return;
 		}
diff --git a/src/Val.cc b/src/Val.cc
index 8a8c2b18c0..79fa8a0c69 100644
--- a/src/Val.cc
+++ b/src/Val.cc
@@ -64,7 +64,7 @@ Val::~Val()
 
 	Unref(type);
 #ifdef DEBUG
-	Unref(bound_id);
+	delete [] bound_id;
 #endif
 	}
 
diff --git a/src/Val.h b/src/Val.h
index 2ca18e6131..c3ec5b04fb 100644
--- a/src/Val.h
+++ b/src/Val.h
@@ -347,13 +347,15 @@ public:
 #ifdef DEBUG
 	// For debugging, we keep a reference to the global ID to which a
 	// value has been bound *last*.
-	ID* GetID() const	{ return bound_id; }
+	ID* GetID() const
+		{
+		return bound_id ? global_scope()->Lookup(bound_id) : 0;
+		}
+
 	void SetID(ID* id)
 		{
-		if ( bound_id )
-			::Unref(bound_id);
-		bound_id = id;
-		::Ref(bound_id);
+		delete [] bound_id;
+		bound_id = id ? copy_string(id->Name()) : 0;
 		}
 #endif
 
@@ -401,8 +403,8 @@ protected:
 	RecordVal* attribs;
 
 #ifdef DEBUG
-	// For debugging, we keep the ID to which a Val is bound.
-	ID* bound_id;
+	// For debugging, we keep the name of the ID to which a Val is bound.
+	const char* bound_id;
 #endif
 
 };

From 508ac1c7ba1b9fbddc128a109b51bd6376ba4bd9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 16 Aug 2012 16:33:46 -0500
Subject: [PATCH 16/57] Unit test tweaks/fixes.

- Some baselines for tests in "leaks" group were outdated.

- Changed a few of the cluster/communication tests to terminate
  more explicitly instead of relying on btest-bg-wait to kill
  processes.  This makes the tests finish faster in the success case
  and makes the reason for failing clearer in the that case.
---
 .../manager-1.metrics.log                     |  8 +++--
 .../core.leaks.remote/sender.test.failure.log |  8 +++--
 .../core.leaks.remote/sender.test.log         | 12 ++++---
 .../core.leaks.remote/sender.test.success.log |  6 ++--
 testing/btest/core/leaks/basic-cluster.bro    | 23 +++++++++++++-
 testing/btest/core/leaks/remote.bro           | 31 ++++++++++++++-----
 .../base/frameworks/logging/remote.bro        | 23 +++++++++++---
 .../base/frameworks/metrics/basic-cluster.bro | 23 +++++++++++++-
 .../metrics/cluster-intermediate-update.bro   | 17 +++++++++-
 9 files changed, 122 insertions(+), 29 deletions(-)

diff --git a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log
index 42fcd6a526..cb1bd5af01 100644
--- a/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log
+++ b/testing/btest/Baseline/core.leaks.basic-cluster/manager-1.metrics.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	metrics
+#open	2012-07-20-01-50-41
 #fields	ts	metric_id	filter_name	index.host	index.str	index.network	value
 #types	time	enum	string	addr	string	subnet	count
-1331256494.591966	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
-1331256494.591966	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
-1331256494.591966	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+1342749041.601712	TEST_METRIC	foo-bar	6.5.4.3	-	-	4
+1342749041.601712	TEST_METRIC	foo-bar	7.2.1.5	-	-	2
+1342749041.601712	TEST_METRIC	foo-bar	1.2.3.4	-	-	6
+#close	2012-07-20-01-50-49
diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log
index 5a26f322f4..71e1d18c73 100644
--- a/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log
+++ b/testing/btest/Baseline/core.leaks.remote/sender.test.failure.log
@@ -3,8 +3,10 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.failure
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.log b/testing/btest/Baseline/core.leaks.remote/sender.test.log
index 9d2ba26f48..bc3dac5a1a 100644
--- a/testing/btest/Baseline/core.leaks.remote/sender.test.log
+++ b/testing/btest/Baseline/core.leaks.remote/sender.test.log
@@ -3,10 +3,12 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	US
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	UK
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	failure	MX
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	US
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	failure	MX
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log
index 1b2ed452a0..f0b26454b4 100644
--- a/testing/btest/Baseline/core.leaks.remote/sender.test.success.log
+++ b/testing/btest/Baseline/core.leaks.remote/sender.test.success.log
@@ -3,7 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	test.success
+#open	2012-07-20-01-50-18
 #fields	t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
 #types	time	addr	port	addr	port	string	string
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	unknown
-1331256472.375609	1.2.3.4	1234	2.3.4.5	80	success	BR
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1342749018.970682	1.2.3.4	1234	2.3.4.5	80	success	BR
+#close	2012-07-20-01-50-18
diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index f5b40c1104..d9d2f97b1e 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -9,7 +9,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
-# @TEST-EXEC: btest-bg-wait -k 30
+# @TEST-EXEC: btest-bg-wait 40
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -40,3 +40,24 @@ event bro_init() &priority=5
 		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
 		}
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
+		{
+		terminate_communication();
+		terminate();
+		}
+	}
+
+@endif
diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro
index f888d8f6ee..8c8dc73364 100644
--- a/testing/btest/core/leaks/remote.bro
+++ b/testing/btest/core/leaks/remote.bro
@@ -4,17 +4,19 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -b -m --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
-# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
-# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+# @TEST-EXEC: ( cd sender && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: ( cd receiver && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: cmp receiver/c.test.log sender/c.test.log
+# @TEST-EXEC: cmp receiver/c.test.failure.log sender/c.test.failure.log
+# @TEST-EXEC: cmp receiver/c.test.success.log sender/c.test.success.log
 
 # This is the common part loaded by both sender and receiver.
 module Test;
@@ -43,10 +45,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -68,14 +70,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
diff --git a/testing/btest/scripts/base/frameworks/logging/remote.bro b/testing/btest/scripts/base/frameworks/logging/remote.bro
index 48683148f5..ba577cc92b 100644
--- a/testing/btest/scripts/base/frameworks/logging/remote.bro
+++ b/testing/btest/scripts/base/frameworks/logging/remote.bro
@@ -1,10 +1,10 @@
 # @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender bro -b --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro -b --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
@@ -41,10 +41,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -66,14 +66,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index 09479b7a2f..4aa1afa96f 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -5,7 +5,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -36,3 +36,24 @@ event bro_init() &priority=5
 		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
 		}
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
+		{
+		terminate_communication();
+		terminate();
+		}
+	}
+
+@endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
index 654e42976a..db2c7e9f5d 100644
--- a/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro
@@ -5,7 +5,7 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -37,6 +37,21 @@ event bro_init() &priority=5
 		 $log=T]);
 	}
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	terminate();
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::WORKER )
 
 event do_metrics(i: count)

From 907c92e1ccd692023ea305fa9e1acba5f4819aa9 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 17 Aug 2012 15:22:51 -0500
Subject: [PATCH 17/57] Fix mime type diff canonifier to also skip mime_desc
 columns

In particular, the ftp.log baseline in the new ipv6 test in bro-testing
was failign on various platforms because of this.
---
 testing/scripts/diff-remove-mime-types | 17 ++++++++++++-----
 1 file changed, 12 insertions(+), 5 deletions(-)

diff --git a/testing/scripts/diff-remove-mime-types b/testing/scripts/diff-remove-mime-types
index fb447a9989..b8cc3d1e6d 100755
--- a/testing/scripts/diff-remove-mime-types
+++ b/testing/scripts/diff-remove-mime-types
@@ -3,20 +3,27 @@
 # A diff canonifier that removes all MIME types because libmagic output
 # can differ between installations.
 
-BEGIN { FS="\t"; OFS="\t"; column = -1; }
+BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 }
 
 /^#fields/ {
     for ( i = 2; i < NF; ++i )
+        {
         if ( $i == "mime_type" )
-            column = i-1;
+            type_col = i-1;
+        if ( $i == "mime_desc" )
+            desc_col = i-1;
+        }
 }
 
-column >= 0 {
-    if ( $column != "-" )
+function remove_mime (n) {
+    if ( n >= 0 && $n != "-" )
         # Mark that it's set, but ignore content.
-        $column = "+";
+        $n = "+"
 }
 
+remove_mime(type_col)
+remove_mime(desc_col)
+
 {
     print;
 }

From f201a9f1a7f52329f1c8db35ab46dbfa50f0bda4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Fri, 17 Aug 2012 17:27:02 -0500
Subject: [PATCH 18/57] Fix portability of printing to files returned by
 open("/dev/stderr").

The BroFile ctor now wraps /dev/std{in,out,err} string arguments
into the actual FILE* provided by stdio.h because use of the former
directly isn't POSIX compliant and led to subtle differences that
broke unit tests on certain platforms (e.g. OS X redirection of stderr
behavior started differing from Linux).  The BroFile (un)serialization
methods already did this kind of logic, so adding it in the ctor also
should make things more consistent.

Some of the reporter-related unit tests looked like they were missing
output because of this, and the coverage test for bare-mode errors
needed tweaking to branch on whether or not libcurl was available
(since the error output differs when elasticsearch isn't there).
---
 src/File.cc                                        | 14 ++++++++++++--
 .../Baseline/core.reporter-error-in-handler/output |  4 ++--
 .../Baseline/core.reporter-runtime-error/output    |  3 ++-
 .../btest/Baseline/core.reporter/logger-test.log   | 12 ++++++------
 testing/btest/Baseline/core.reporter/output        | 11 +++++++----
 .../unique_errors_no_elasticsearch                 |  1 +
 testing/btest/coverage/bare-mode-errors.test       |  3 ++-
 7 files changed, 32 insertions(+), 16 deletions(-)
 create mode 100644 testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch

diff --git a/src/File.cc b/src/File.cc
index 20e845c09f..20ab2e1013 100644
--- a/src/File.cc
+++ b/src/File.cc
@@ -138,11 +138,21 @@ BroFile::BroFile(FILE* arg_f, const char* arg_name, const char* arg_access)
 BroFile::BroFile(const char* arg_name, const char* arg_access, BroType* arg_t)
 	{
 	Init();
-
+	f = 0;
 	name = copy_string(arg_name);
 	access = copy_string(arg_access);
 	t = arg_t ? arg_t : base_type(TYPE_STRING);
-	if ( ! Open() )
+
+	if ( streq(name, "/dev/stdin") )
+		f = stdin;
+	else if ( streq(name, "/dev/stdout") )
+		f = stdout;
+	else if ( streq(name, "/dev/stderr") )
+		f = stderr;
+
+	if ( f )
+		is_open = 1;
+	else if ( ! Open() )
 		{
 		reporter->Error("cannot open %s: %s", name, strerror(errno));
 		is_open = 0;
diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output
index 190631f4d1..b20b1b2292 100644
--- a/testing/btest/Baseline/core.reporter-error-in-handler/output
+++ b/testing/btest/Baseline/core.reporter-error-in-handler/output
@@ -1,3 +1,3 @@
-ERROR: no such index (a[1]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
-
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2])
+ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28)
 1st error printed on script level
diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output
index 94f7860cb4..5a03f5feb2 100644
--- a/testing/btest/Baseline/core.reporter-runtime-error/output
+++ b/testing/btest/Baseline/core.reporter-runtime-error/output
@@ -1 +1,2 @@
-ERROR: no such index (a[2]) (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1])
+ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9)
diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log
index 6f7ba1d8c7..5afd904b63 100644
--- a/testing/btest/Baseline/core.reporter/logger-test.log
+++ b/testing/btest/Baseline/core.reporter/logger-test.log
@@ -1,6 +1,6 @@
-reporter_info|init test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
-reporter_warning|init test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
-reporter_error|init test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
-reporter_info|done test-info|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
-reporter_warning|done test-warning|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
-reporter_error|done test-error|/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
+reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000
+reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000
+reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000
+reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000
+reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000
+reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000
diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output
index b4f89bad2f..f2c59259c2 100644
--- a/testing/btest/Baseline/core.reporter/output
+++ b/testing/btest/Baseline/core.reporter/output
@@ -1,4 +1,7 @@
-WARNING: init test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
-ERROR: init test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
-WARNING: done test-warning (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
-ERROR: done test-error (/da/home/robin/bro/master/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
+/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info
+warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning
+error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error
+WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9)
+ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10)
+WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16)
+ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17)
diff --git a/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch b/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch
new file mode 100644
index 0000000000..e95f88e74b
--- /dev/null
+++ b/testing/btest/Baseline/coverage.bare-mode-errors/unique_errors_no_elasticsearch
@@ -0,0 +1 @@
+error: unknown writer type requested
diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test
index 21e7d4f4a9..7084d74e83 100644
--- a/testing/btest/coverage/bare-mode-errors.test
+++ b/testing/btest/coverage/bare-mode-errors.test
@@ -10,4 +10,5 @@
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
-# @TEST-EXEC: btest-diff unique_errors
+# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
+# @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

From 0dbf2f18fa679a1231f957e474a1bb1bb59e5042 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Mon, 20 Aug 2012 13:26:17 -0400
Subject: [PATCH 19/57] Add the Stream record to Log:active_streams to make
 more dynamic logging possible.

---
 scripts/base/frameworks/logging/main.bro | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro
index ccc65ddf67..bed76a1ae5 100644
--- a/scripts/base/frameworks/logging/main.bro
+++ b/scripts/base/frameworks/logging/main.bro
@@ -329,9 +329,9 @@ export {
 	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
 
 	## The streams which are currently active and not disabled.
-	## This set is not meant to be modified by users!  Only use it for
+	## This table is not meant to be modified by users!  Only use it for
 	## examining which streams are active.
-	global active_streams: set[ID] = set();
+	global active_streams: table[ID] of Stream = table();
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
@@ -417,7 +417,7 @@ function create_stream(id: ID, stream: Stream) : bool
 	if ( ! __create_stream(id, stream) )
 		return F;
 
-	add active_streams[id];
+	active_streams[id] = stream;
 
 	return add_default_filter(id);
 	}

From 434d6a84d8bb73cef7704799fcd391375bba5862 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 21 Aug 2012 08:32:42 -0700
Subject: [PATCH 20/57] Linking ES docs into logging document.

---
 CHANGES         | 4 ++++
 VERSION         | 2 +-
 doc/logging.rst | 1 +
 3 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/CHANGES b/CHANGES
index 7b381b5c5d..b6225097db 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,8 @@
 
+2.1-beta-28 | 2012-08-21 08:32:42 -0700
+
+  * Linking ES docs into logging document. (Robin Sommer)
+
 2.1-beta-27 | 2012-08-20 20:06:20 -0700
 
   * Add the Stream record to Log:active_streams to make more dynamic
diff --git a/VERSION b/VERSION
index e82f524ce7..c403b714f8 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta-27
+2.1-beta-28
diff --git a/doc/logging.rst b/doc/logging.rst
index cc6cb1e54d..7fb4205b9a 100644
--- a/doc/logging.rst
+++ b/doc/logging.rst
@@ -383,3 +383,4 @@ Bro supports the following output formats other than ASCII:
    :maxdepth: 1
 
    logging-dataseries
+   logging-elasticsearch

From 06b7379bc3f112faab220d59663844d449add3a8 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 21 Aug 2012 14:54:57 -0500
Subject: [PATCH 21/57] Ignore small mem leak every rotation interval for
 dataseries logs.

Not sure if more can be done to work around it, but reported to
dataseries devs here: https://github.com/dataseries/DataSeries/issues/1

The core/leaks/dataseries-rotate.bro unit test fails without this.
---
 src/logging/writers/DataSeries.cc | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc
index 7d3053e341..bc5a82ec54 100644
--- a/src/logging/writers/DataSeries.cc
+++ b/src/logging/writers/DataSeries.cc
@@ -243,8 +243,25 @@ bool DataSeries::OpenLog(string path)
 	log_file->writeExtentLibrary(log_types);
 
 	for( size_t i = 0; i < schema_list.size(); ++i )
-		extents.insert(std::make_pair(schema_list[i].field_name,
-					      GeneralField::create(log_series, schema_list[i].field_name)));
+		{
+		string fn = schema_list[i].field_name;
+		GeneralField* gf = 0;
+#ifdef USE_PERFTOOLS_DEBUG
+		{
+		// GeneralField isn't cleaning up some results of xml parsing, reported
+		// here: https://github.com/dataseries/DataSeries/issues/1
+		// Ignore for now to make leak tests pass.  There's confidence that
+		// we do clean up the GeneralField* since the ExtentSeries dtor for
+		// member log_series would trigger an assert if dynamically allocated
+		// fields aren't deleted beforehand.
+		HeapLeakChecker::Disabler disabler;
+#endif
+		gf = GeneralField::create(log_series, fn);
+#ifdef USE_PERFTOOLS_DEBUG
+		}
+#endif
+		extents.insert(std::make_pair(fn, gf));
+		}
 
 	if ( ds_extent_size < ROW_MIN )
 		{

From bb4b68946f9530b119a0144191e8e72a27896b9d Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 21 Aug 2012 15:22:54 -0500
Subject: [PATCH 22/57] Tweak to rotate-custom.bro unit test.

This one would fail intermittently in the cases where log files were
opened or closed on a different second of the time of day from each
other since the "out" baseline contains only a single "#open" and
"#close" tag (indicating all logs opened/closed on same second of time
of day).  Piping aggregated log output through the timestamp canonifier
before `uniq` makes it so "#open" and "#close" tags for different
seconds of the time of day are reduced to a single one.
---
 testing/btest/scripts/base/frameworks/logging/rotate-custom.bro | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
index 07fc8cef7c..c0f0ef8643 100644
--- a/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
+++ b/testing/btest/scripts/base/frameworks/logging/rotate-custom.bro
@@ -1,7 +1,7 @@
 #
 # @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out.tmp
 # @TEST-EXEC: cat out.tmp pp.log | sort >out
-# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
+# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | $SCRIPTS/diff-remove-timestamps | uniq >>out
 # @TEST-EXEC: btest-diff out
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
 

From cd67603f49b3e287d5244d702b62373a265ede10 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 21 Aug 2012 21:48:49 -0700
Subject: [PATCH 23/57] add testcase for input of set. Sets can be imported by
 not specifying $val in the add_table call.

This actually was already implemented, I just completely forgot about it.
---
 .../scripts.base.frameworks.input.set/out     |  7 +++
 .../scripts/base/frameworks/input/set.bro     | 46 +++++++++++++++++++
 2 files changed, 53 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.set/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/set.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.set/out b/testing/btest/Baseline/scripts.base.frameworks.input.set/out
new file mode 100644
index 0000000000..998244cf3f
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.set/out
@@ -0,0 +1,7 @@
+{
+192.168.17.7,
+192.168.17.42,
+192.168.17.14,
+192.168.17.1,
+192.168.17.2
+}
diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro
new file mode 100644
index 0000000000..5215523ee3
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/set.bro
@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	ip
+#types	addr
+192.168.17.1
+192.168.17.2
+192.168.17.7
+192.168.17.14
+192.168.17.42
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	ip: addr;
+};
+
+global servers: set[addr] = set();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From ec224ada0679d8dcc1c7925969ba44b459145957 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 21 Aug 2012 22:17:28 -0700
Subject: [PATCH 24/57] single-line documentation addition to main input
 framework script.

---
 scripts/base/frameworks/input/main.bro | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index 55da6ae7ec..758bc94732 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -8,6 +8,7 @@ export {
 	## The default input reader used. Defaults to `READER_ASCII`.
 	const default_reader = READER_ASCII &redef;
 
+	## The default reader mode used. Defaults to `MANUAL`.
 	const default_mode = MANUAL &redef;
 
 	## TableFilter description type used for the `table` method.

From b53be217502d6bf143e61e2f5d09bd7cdd23c525 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 21 Aug 2012 23:00:04 -0700
Subject: [PATCH 25/57] add an option to the input framework that allows the
 user to chose to not die upon encountering files/functions.

I am not entirely sure if I like the approach I took for
this, it is a bit... hacky.
---
 scripts/base/frameworks/input/main.bro        |  7 ++
 src/input.bif                                 |  4 ++
 src/input/Manager.cc                          | 40 ++++++++++--
 src/input/Manager.h                           |  2 +-
 .../out                                       | 14 ++++
 .../frameworks/input/unsupported_types.bro    | 64 +++++++++++++++++++
 6 files changed, 125 insertions(+), 6 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/unsupported_types.bro

diff --git a/scripts/base/frameworks/input/main.bro b/scripts/base/frameworks/input/main.bro
index 55da6ae7ec..e8aa67b23b 100644
--- a/scripts/base/frameworks/input/main.bro
+++ b/scripts/base/frameworks/input/main.bro
@@ -10,6 +10,13 @@ export {
 
 	const default_mode = MANUAL &redef;
 
+	## Flag that controls if the input framework accepts records
+	## that contain types that are not supported (at the moment
+	## file and function). If true, the input framework will
+	## warn in these cases, but continue. If false, it will
+	## abort. Defaults to false (abort)
+	const accept_unsupported_types = F &redef;
+
 	## TableFilter description type used for the `table` method.
 	type TableDescription: record {
 		## Common definitions for tables and events
diff --git a/src/input.bif b/src/input.bif
index f494ef3b2f..199b665fa6 100644
--- a/src/input.bif
+++ b/src/input.bif
@@ -34,6 +34,10 @@ function Input::__force_update%(id: string%) : bool
 	return new Val(res, TYPE_BOOL);
 	%}
 
+# Options for the input framework
+
+const accept_unsupported_types: bool;
+
 # Options for Ascii Reader
 
 module InputAscii;
diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 3c29f14928..4422a9814f 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -388,6 +388,8 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	FuncType* etype = event->FType()->AsFuncType();
 
+	bool allow_file_func = false;
+
 	if ( ! etype->IsEvent() )
 		{
 		reporter->Error("stream event is a function, not an event");
@@ -453,6 +455,8 @@ bool Manager::CreateEventStream(RecordVal* fval)
 			return false;
 			}
 
+		allow_file_func = BifConst::Input::accept_unsupported_types;
+
 		}
 
 	else
@@ -461,7 +465,7 @@ bool Manager::CreateEventStream(RecordVal* fval)
 
 	vector<Field*> fieldsV; // vector, because UnrollRecordType needs it
 
-	bool status = !UnrollRecordType(&fieldsV, fields, "");
+	bool status = !UnrollRecordType(&fieldsV, fields, "", allow_file_func);
 
 	if ( status )
 		{
@@ -609,12 +613,12 @@ bool Manager::CreateTableStream(RecordVal* fval)
 
 	vector<Field*> fieldsV; // vector, because we don't know the length beforehands
 
-	bool status = !UnrollRecordType(&fieldsV, idx, "");
+	bool status = !UnrollRecordType(&fieldsV, idx, "", false);
 
 	int idxfields = fieldsV.size();
 
 	if ( val ) // if we are not a set
-		status = status || !UnrollRecordType(&fieldsV, val, "");
+		status = status || !UnrollRecordType(&fieldsV, val, "", BifConst::Input::accept_unsupported_types);
 
 	int valfields = fieldsV.size() - idxfields;
 
@@ -773,7 +777,7 @@ bool Manager::RemoveStreamContinuation(ReaderFrontend* reader)
 	}
 
 bool Manager::UnrollRecordType(vector<Field*> *fields,
-		const RecordType *rec, const string& nameprepend)
+		const RecordType *rec, const string& nameprepend, bool allow_file_func)
 	{
 
 	for ( int i = 0; i < rec->NumFields(); i++ )
@@ -781,6 +785,23 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 
 		if ( ! IsCompatibleType(rec->FieldType(i)) )
 	       		{
+
+			// if the field is a file or a function type
+			// and it is optional, we accept it nevertheless.
+			// This allows importing logfiles containing this
+			// stuff that we actually cannot read :)
+			if ( allow_file_func ) 
+				{
+				if ( ( rec->FieldType(i)->Tag() == TYPE_FILE ||
+				       rec->FieldType(i)->Tag() == TYPE_FUNC ) &&
+				     rec->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)
+				   ) 
+					{
+					reporter->Info("Encountered incompatible type \"%s\" in table definition for ReaderFrontend. Ignoring field.", type_name(rec->FieldType(i)->Tag()));
+					continue;
+					}
+				}
+
 			reporter->Error("Incompatible type \"%s\" in table definition for ReaderFrontend", type_name(rec->FieldType(i)->Tag()));
 			return false;
 			}
@@ -789,7 +810,7 @@ bool Manager::UnrollRecordType(vector<Field*> *fields,
 			{
 			string prep = nameprepend + rec->FieldName(i) + ".";
 
-			if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep) )
+			if ( !UnrollRecordType(fields, rec->FieldType(i)->AsRecordType(), prep, allow_file_func) )
 				{
 				return false;
 				}
@@ -1675,6 +1696,15 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals,
 		Val* fieldVal = 0;
 		if ( request_type->FieldType(i)->Tag() == TYPE_RECORD )
 			fieldVal = ValueToRecordVal(vals, request_type->FieldType(i)->AsRecordType(), position);
+		else if ( request_type->FieldType(i)->Tag() == TYPE_FILE || 
+			  request_type->FieldType(i)->Tag() == TYPE_FUNC )
+			{
+			// If those two unsupported types are encountered here, they have
+			// been let through by the type checking.
+			// That means that they are optional & the user agreed to ignore
+			// them and has been warned by reporter.
+			// Hence -> assign null to the field, done.
+			}
 		else
 			{
 			fieldVal = ValueToVal(vals[*position], request_type->FieldType(i));
diff --git a/src/input/Manager.h b/src/input/Manager.h
index 1590042183..cc81df38b7 100644
--- a/src/input/Manager.h
+++ b/src/input/Manager.h
@@ -158,7 +158,7 @@ private:
 	// Check if a record is made up of compatible types and return a list
 	// of all fields that are in the record in order. Recursively unrolls
 	// records
-	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend);
+	bool UnrollRecordType(vector<threading::Field*> *fields, const RecordType *rec, const string& nameprepend, bool allow_file_func);
 
 	// Send events
 	void SendEvent(EventHandlerPtr ev, const int numvals, ...);
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out
new file mode 100644
index 0000000000..7ef82cf368
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.unsupported_types/out
@@ -0,0 +1,14 @@
+{
+[-42] = [fi=<uninitialized>, b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
new file mode 100644
index 0000000000..7affa4065d
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro
@@ -0,0 +1,64 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	fi	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	file	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+whatever	T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+redef Input::accept_unsupported_types = T;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	fi: file &optional;
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From b6bd849018aa41c910a9675bf56d08a7e11b4e29 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Wed, 22 Aug 2012 12:12:16 -0400
Subject: [PATCH 26/57] Fixed ack tracking which could overflow quickly in some
 situations.

- Problem presented itself through incorrect results in capture-loss.bro
  under odd traffic circumstances (exact circumstances unknown).

- Changed variables involved in ack tracking to all be uint64 values.
---
 src/Stats.cc           |  8 ++++----
 src/Stats.h            |  8 ++++----
 src/TCP_Reassembler.cc | 16 ++++++++--------
 3 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/src/Stats.cc b/src/Stats.cc
index c3035231e9..8d48c47a25 100644
--- a/src/Stats.cc
+++ b/src/Stats.cc
@@ -12,10 +12,10 @@
 
 int killed_by_inactivity = 0;
 
-uint32 tot_ack_events = 0;
-uint32 tot_ack_bytes = 0;
-uint32 tot_gap_events = 0;
-uint32 tot_gap_bytes = 0;
+uint64 tot_ack_events = 0;
+uint64 tot_ack_bytes = 0;
+uint64 tot_gap_events = 0;
+uint64 tot_gap_bytes = 0;
 
 
 class ProfileTimer : public Timer {
diff --git a/src/Stats.h b/src/Stats.h
index eeebfe2213..a11d66828a 100644
--- a/src/Stats.h
+++ b/src/Stats.h
@@ -116,10 +116,10 @@ extern SampleLogger* sample_logger;
 extern int killed_by_inactivity;
 
 // Content gap statistics.
-extern uint32 tot_ack_events;
-extern uint32 tot_ack_bytes;
-extern uint32 tot_gap_events;
-extern uint32 tot_gap_bytes;
+extern uint64 tot_ack_events;
+extern uint64 tot_ack_bytes;
+extern uint64 tot_gap_events;
+extern uint64 tot_gap_bytes;
 
 
 // A TCPStateStats object tracks the distribution of TCP states for
diff --git a/src/TCP_Reassembler.cc b/src/TCP_Reassembler.cc
index fb67dba7ee..eb2709373c 100644
--- a/src/TCP_Reassembler.cc
+++ b/src/TCP_Reassembler.cc
@@ -20,10 +20,10 @@ const bool DEBUG_tcp_connection_close = false;
 const bool DEBUG_tcp_match_undelivered = false;
 
 static double last_gap_report = 0.0;
-static uint32 last_ack_events = 0;
-static uint32 last_ack_bytes = 0;
-static uint32 last_gap_events = 0;
-static uint32 last_gap_bytes = 0;
+static uint64 last_ack_events = 0;
+static uint64 last_ack_bytes = 0;
+static uint64 last_gap_events = 0;
+static uint64 last_gap_bytes = 0;
 
 TCP_Reassembler::TCP_Reassembler(Analyzer* arg_dst_analyzer,
 				TCP_Analyzer* arg_tcp_analyzer,
@@ -513,10 +513,10 @@ void TCP_Reassembler::AckReceived(int seq)
 		if ( gap_report && gap_report_freq > 0.0 &&
 		     dt >= gap_report_freq )
 			{
-			int devents = tot_ack_events - last_ack_events;
-			int dbytes = tot_ack_bytes - last_ack_bytes;
-			int dgaps = tot_gap_events - last_gap_events;
-			int dgap_bytes = tot_gap_bytes - last_gap_bytes;
+			uint64 devents = tot_ack_events - last_ack_events;
+			uint64 dbytes = tot_ack_bytes - last_ack_bytes;
+			uint64 dgaps = tot_gap_events - last_gap_events;
+			uint64 dgap_bytes = tot_gap_bytes - last_gap_bytes;
 
 			RecordVal* r = new RecordVal(gap_info);
 			r->Assign(0, new Val(devents, TYPE_COUNT));

From e66e9e5d321716ecee47d9ab08155b9fe2ee034a Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 22 Aug 2012 11:12:27 -0500
Subject: [PATCH 27/57] Minor tweak to coverage.bare-mode-errors unit test.

Adding trailing slash to $DIST/scripts makes the `find` work
with a symlinked 'scripts' dir.
---
 testing/btest/coverage/bare-mode-errors.test | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test
index 7084d74e83..635726841b 100644
--- a/testing/btest/coverage/bare-mode-errors.test
+++ b/testing/btest/coverage/bare-mode-errors.test
@@ -8,7 +8,7 @@
 # @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: test -d $DIST/scripts
-# @TEST-EXEC: for script in `find $DIST/scripts -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
+# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0
 # @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors
 # @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi
 # @TEST-EXEC: if [ $(grep -c CURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi

From 201c4aa43aec4371f67851294036556154664808 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Wed, 22 Aug 2012 13:25:22 -0700
Subject: [PATCH 28/57] to be sure - add a small assertion

---
 src/input/Manager.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 4422a9814f..c3176d9c33 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1704,6 +1704,9 @@ RecordVal* Manager::ValueToRecordVal(const Value* const *vals,
 			// That means that they are optional & the user agreed to ignore
 			// them and has been warned by reporter.
 			// Hence -> assign null to the field, done.
+			
+			// better check that it really is optional. you never know.
+			assert(request_type->FieldDecl(i)->FindAttr(ATTR_OPTIONAL));
 			}
 		else
 			{

From 655a73bc13ff6d9cee18e98f90ad42a90b6a5b29 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 22 Aug 2012 16:46:47 -0500
Subject: [PATCH 29/57] Change to metrics/basic-cluster unit test for
 reliability.

If the metrics break interval happened to occur between first and second
worker starting up and getting connected to the cluster, the test
would fail because the second worker didn't get a chance to connect and
send data.  The test now waits for the cluster setup to complete
before workers send metrics data.
---
 testing/btest/core/leaks/basic-cluster.bro    | 43 +++++++++++++------
 .../base/frameworks/metrics/basic-cluster.bro | 39 ++++++++++++-----
 2 files changed, 60 insertions(+), 22 deletions(-)

diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index d9d2f97b1e..7fb176b8db 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -1,21 +1,21 @@
 # Needs perftools support.
 #
 # @TEST-GROUP: leaks
-
+#
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
-
+#
 # @TEST-EXEC: btest-bg-run manager-1 HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro -m %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1   bro -m %INPUT
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2  bro -m -r $TRACES/web.trace --pseudo-realtime %INPUT
-# @TEST-EXEC: btest-bg-wait 40
+# @TEST-EXEC: btest-bg-wait 60
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -32,13 +32,6 @@ event bro_init() &priority=5
 	Metrics::add_filter(TEST_METRIC, 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
-		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
-		}
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -46,9 +39,25 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 global n = 0;
+global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
@@ -60,4 +69,14 @@ event Metrics::log_metrics(rec: Metrics::Info)
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
 @endif
diff --git a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
index 4aa1afa96f..89ae5bf79f 100644
--- a/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
+++ b/testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro
@@ -5,13 +5,13 @@
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")],
-	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
 	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
 	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"],
 };
@@ -28,13 +28,6 @@ event bro_init() &priority=5
 	Metrics::add_filter(TEST_METRIC, 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
-		{
-		Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
-		Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
-		Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
-		}
 	}
 
 event remote_connection_closed(p: event_peer)
@@ -42,9 +35,25 @@ event remote_connection_closed(p: event_peer)
 	terminate();
 	}
 
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
 
 global n = 0;
+global peer_count = 0;
 
 event Metrics::log_metrics(rec: Metrics::Info)
 	{
@@ -56,4 +65,14 @@ event Metrics::log_metrics(rec: Metrics::Info)
 		}
 	}
 
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
 @endif

From 93744c8d9b22888269f466f116559f90f96638d4 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Wed, 22 Aug 2012 16:54:00 -0500
Subject: [PATCH 30/57] Add test serialization to "leak" unit tests that use
 communication.

---
 testing/btest/core/leaks/basic-cluster.bro | 1 +
 testing/btest/core/leaks/remote.bro        | 1 +
 2 files changed, 2 insertions(+)

diff --git a/testing/btest/core/leaks/basic-cluster.bro b/testing/btest/core/leaks/basic-cluster.bro
index 7fb176b8db..319368bc6e 100644
--- a/testing/btest/core/leaks/basic-cluster.bro
+++ b/testing/btest/core/leaks/basic-cluster.bro
@@ -1,5 +1,6 @@
 # Needs perftools support.
 #
+# @TEST-SERIALIZE: comm
 # @TEST-GROUP: leaks
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
diff --git a/testing/btest/core/leaks/remote.bro b/testing/btest/core/leaks/remote.bro
index 8c8dc73364..41bbaec076 100644
--- a/testing/btest/core/leaks/remote.bro
+++ b/testing/btest/core/leaks/remote.bro
@@ -1,5 +1,6 @@
 # Needs perftools support.
 #
+# @TEST-SERIALIZE: comm
 # @TEST-GROUP: leaks
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks

From 95d7055373763787628936431e82ea6562f4d7ba Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 22 Aug 2012 16:17:27 -0700
Subject: [PATCH 31/57] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index 22120825f8..a93ef13735 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 22120825f8ad70e051ef4ca42f2199aa195dff40
+Subproject commit a93ef1373512c661ffcd0d0a61bd19b96667e0d5
diff --git a/aux/bro-aux b/aux/bro-aux
index 941ee753f7..4bc1a6f6a8 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 941ee753f7c71ec08fc29de04f09a8a83aebb69d
+Subproject commit 4bc1a6f6a8816dfacd8288fcf182ba35520e589b
diff --git a/aux/broccoli b/aux/broccoli
index 5ff3e6a8e8..ebfa4de45a 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 5ff3e6a8e8535ed91e1f70d355b815ae8eeacb71
+Subproject commit ebfa4de45a839e58aec200e7e4bad33eaab4f1ed
diff --git a/aux/broctl b/aux/broctl
index 6d0eb6083a..5b3f9e5906 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 6d0eb6083acdc77e0a912bec0fb23df79b98da63
+Subproject commit 5b3f9e5906c90b76c5aa1626e112d4c991cb3fd8

From 25ef0a89e752aec2b1506363ffdeea738d1e3f1b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 22 Aug 2012 18:15:55 -0700
Subject: [PATCH 32/57] Updating NEWS.

---
 NEWS | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/NEWS b/NEWS
index 949b51d832..d7018575d3 100644
--- a/NEWS
+++ b/NEWS
@@ -7,8 +7,8 @@ release. For a complete list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
 their own CHANGES.)
 
-Bro 2.1 Beta
-------------
+Bro 2.1
+-------
 
 New Functionality
 ~~~~~~~~~~~~~~~~~
@@ -161,6 +161,7 @@ the full set.
 - The ASCII writers "header_*" options have been renamed to "meta_*"
   (because there's now also a footer).
 
+
 Bro 2.0
 -------
 

From bef0ce1c98bc2dfc0e2dddef821878b7eb91f4b7 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 11:52:39 -0500
Subject: [PATCH 33/57] Add type checking for signature 'eval' condition
 functions.

Otherwise functions could be called with a mismatching argument list
and cause a crash at run-time.  The incorrect function type is now
reported at parse-time.
---
 src/RuleCondition.cc                          | 17 ++++++++++++++
 .../signatures.bad-eval-condition/.stderr     |  2 ++
 .../signatures.eval-condition/conn.log        | 14 ++++++++++++
 .../output                                    |  0
 testing/btest/btest.cfg                       |  2 +-
 .../btest/signatures/bad-eval-condition.bro   | 22 +++++++++++++++++++
 testing/btest/signatures/eval-condition.bro   | 20 +++++++++++++++++
 .../btest/{core => signatures}/load-sigs.bro  |  0
 8 files changed, 76 insertions(+), 1 deletion(-)
 create mode 100644 testing/btest/Baseline/signatures.bad-eval-condition/.stderr
 create mode 100644 testing/btest/Baseline/signatures.eval-condition/conn.log
 rename testing/btest/Baseline/{core.load-sigs => signatures.load-sigs}/output (100%)
 create mode 100644 testing/btest/signatures/bad-eval-condition.bro
 create mode 100644 testing/btest/signatures/eval-condition.bro
 rename testing/btest/{core => signatures}/load-sigs.bro (100%)

diff --git a/src/RuleCondition.cc b/src/RuleCondition.cc
index 8852747cc4..3e64f9ffca 100644
--- a/src/RuleCondition.cc
+++ b/src/RuleCondition.cc
@@ -126,6 +126,23 @@ RuleConditionEval::RuleConditionEval(const char* func)
 		rules_error("unknown identifier", func);
 		return;
 		}
+
+	if ( id->Type()->Tag() == TYPE_FUNC )
+		{
+		// validate argument quantity and type
+		FuncType* f = id->Type()->AsFuncType();
+
+		if ( f->YieldType()->Tag() != TYPE_BOOL )
+			rules_error("eval function type must yield a 'bool'", func);
+
+		TypeList tl;
+		tl.Append(internal_type("signature_state")->Ref());
+		tl.Append(base_type(TYPE_STRING));
+
+		if ( ! f->CheckArgs(tl.Types()) )
+			rules_error("eval function parameters must be a 'signature_state' "
+			            "and a 'string' type", func);
+		}
 	}
 
 bool RuleConditionEval::DoMatch(Rule* rule, RuleEndpointState* state,
diff --git a/testing/btest/Baseline/signatures.bad-eval-condition/.stderr b/testing/btest/Baseline/signatures.bad-eval-condition/.stderr
new file mode 100644
index 0000000000..c4de35ffe9
--- /dev/null
+++ b/testing/btest/Baseline/signatures.bad-eval-condition/.stderr
@@ -0,0 +1,2 @@
+error: Error in signature (./blah.sig:6): eval function parameters must be a 'signature_state' and a 'string' type (mark_conn)
+
diff --git a/testing/btest/Baseline/signatures.eval-condition/conn.log b/testing/btest/Baseline/signatures.eval-condition/conn.log
new file mode 100644
index 0000000000..a803f74320
--- /dev/null
+++ b/testing/btest/Baseline/signatures.eval-condition/conn.log
@@ -0,0 +1,14 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2012-08-23-16-41-23
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1329843175.736107	arKYeMETxOg	141.142.220.235	37604	199.233.217.249	56666	tcp	ftp-data	0.112432	0	342	SF	-	0	ShAdfFa	4	216	4	562	(empty)
+1329843179.871641	k6kgXLOoSKl	141.142.220.235	59378	199.233.217.249	56667	tcp	ftp-data	0.111218	0	77	SF	-	0	ShAdfFa	4	216	4	297	(empty)
+1329843194.151526	nQcgTWjvg4c	199.233.217.249	61920	141.142.220.235	33582	tcp	ftp-data	0.056211	342	0	SF	-	0	ShADaFf	5	614	3	164	(empty)
+1329843197.783443	j4u32Pc5bif	199.233.217.249	61918	141.142.220.235	37835	tcp	ftp-data	0.056005	77	0	SF	-	0	ShADaFf	5	349	3	164	(empty)
+1329843161.968492	UWkUyAuUGXf	141.142.220.235	50003	199.233.217.249	21	tcp	ftp,blah	38.055625	180	3146	SF	-	0	ShAdDfFa	38	2164	25	4458	(empty)
+#close	2012-08-23-16-41-23
diff --git a/testing/btest/Baseline/core.load-sigs/output b/testing/btest/Baseline/signatures.load-sigs/output
similarity index 100%
rename from testing/btest/Baseline/core.load-sigs/output
rename to testing/btest/Baseline/signatures.load-sigs/output
diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg
index 4c4074ee24..d86b45d8a9 100644
--- a/testing/btest/btest.cfg
+++ b/testing/btest/btest.cfg
@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts istate coverage
+TestDirs    = doc bifs language core scripts istate coverage signatures
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
diff --git a/testing/btest/signatures/bad-eval-condition.bro b/testing/btest/signatures/bad-eval-condition.bro
new file mode 100644
index 0000000000..34997b1124
--- /dev/null
+++ b/testing/btest/signatures/bad-eval-condition.bro
@@ -0,0 +1,22 @@
+# @TEST-EXEC-FAIL: bro -r $TRACES/ftp-ipv4.trace %INPUT
+# @TEST-EXEC: btest-diff .stderr
+
+@load-sigs blah.sig
+
+@TEST-START-FILE blah.sig
+signature blah
+	{
+	ip-proto == tcp
+	src-port == 21
+	payload /.*/
+	eval mark_conn
+	}
+@TEST-END-FILE
+
+# wrong function signature for use with signature 'eval' conditions
+# needs to be reported
+function mark_conn(state: signature_state): bool
+	{
+	add state$conn$service["blah"];
+	return T;
+	}
diff --git a/testing/btest/signatures/eval-condition.bro b/testing/btest/signatures/eval-condition.bro
new file mode 100644
index 0000000000..f3f1171da6
--- /dev/null
+++ b/testing/btest/signatures/eval-condition.bro
@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace %INPUT
+# @TEST-EXEC: btest-diff conn.log
+
+@load-sigs blah.sig
+
+@TEST-START-FILE blah.sig
+signature blah
+	{
+	ip-proto == tcp
+	src-port == 21
+	payload /.*/
+	eval mark_conn
+	}
+@TEST-END-FILE
+
+function mark_conn(state: signature_state, data: string): bool
+	{
+	add state$conn$service["blah"];
+	return T;
+	}
diff --git a/testing/btest/core/load-sigs.bro b/testing/btest/signatures/load-sigs.bro
similarity index 100%
rename from testing/btest/core/load-sigs.bro
rename to testing/btest/signatures/load-sigs.bro

From ff60b0bb4bf9a1d6da38bd273b0ec34eb2f37f60 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 11:59:51 -0500
Subject: [PATCH 34/57] Remove orphaned unit tests.

Looks like they're maybe from 1.5 and not applicable/updateable.
---
 testing/btest/Baseline/analyzers.conn-size-cc/conn.log | 5 -----
 testing/btest/Baseline/analyzers.conn-size/conn.log    | 5 -----
 testing/btest/analyzers/conn-size-cc.bro               | 2 --
 testing/btest/analyzers/conn-size.bro                  | 2 --
 4 files changed, 14 deletions(-)
 delete mode 100644 testing/btest/Baseline/analyzers.conn-size-cc/conn.log
 delete mode 100644 testing/btest/Baseline/analyzers.conn-size/conn.log
 delete mode 100644 testing/btest/analyzers/conn-size-cc.bro
 delete mode 100644 testing/btest/analyzers/conn-size.bro

diff --git a/testing/btest/Baseline/analyzers.conn-size-cc/conn.log b/testing/btest/Baseline/analyzers.conn-size-cc/conn.log
deleted file mode 100644
index 2f703cbcd6..0000000000
--- a/testing/btest/Baseline/analyzers.conn-size-cc/conn.log
+++ /dev/null
@@ -1,5 +0,0 @@
-1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0 cc=1
-1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0
-1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 677 5 713
-1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0
-1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0
diff --git a/testing/btest/Baseline/analyzers.conn-size/conn.log b/testing/btest/Baseline/analyzers.conn-size/conn.log
deleted file mode 100644
index 8129bc37f8..0000000000
--- a/testing/btest/Baseline/analyzers.conn-size/conn.log
+++ /dev/null
@@ -1,5 +0,0 @@
-1128727430.350788 ? 141.42.64.125 125.190.109.199 other 56729 12345 tcp ? ? S0 X 1 60 0 0
-1144876538.705610 5.921003 169.229.147.203 239.255.255.253 other 49370 427 udp 147 ? S0 X 3 231 0 0
-1144876599.397603 0.815763 192.150.186.169 194.64.249.244 http 53063 80 tcp 377 445 SF X 6 697 5 713
-1144876709.032670 9.000191 169.229.147.43 239.255.255.253 other 49370 427 udp 196 ? S0 X 4 308 0 0
-1144876697.068273 0.000650 192.150.186.169 192.150.186.15 icmp-unreach 3 3 icmp 56 ? OTH X 2 112 0 0
diff --git a/testing/btest/analyzers/conn-size-cc.bro b/testing/btest/analyzers/conn-size-cc.bro
deleted file mode 100644
index 0ba7977cf5..0000000000
--- a/testing/btest/analyzers/conn-size-cc.bro
+++ /dev/null
@@ -1,2 +0,0 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
-# @TEST-EXEC: btest-diff conn.log
diff --git a/testing/btest/analyzers/conn-size.bro b/testing/btest/analyzers/conn-size.bro
deleted file mode 100644
index 0ba7977cf5..0000000000
--- a/testing/btest/analyzers/conn-size.bro
+++ /dev/null
@@ -1,2 +0,0 @@
-# @TEST-EXEC: bro -C -r ${TRACES}/conn-size.trace tcp udp icmp report_conn_size_analyzer=T
-# @TEST-EXEC: btest-diff conn.log

From 558ca2867c873073d30522073049a35f5cc52111 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 12:29:42 -0500
Subject: [PATCH 35/57] Doc fixes for signature 'eval' conditions.

---
 doc/signatures.rst | 16 +++-------------
 1 file changed, 3 insertions(+), 13 deletions(-)

diff --git a/doc/signatures.rst b/doc/signatures.rst
index f65215eceb..36099ba40f 100644
--- a/doc/signatures.rst
+++ b/doc/signatures.rst
@@ -229,20 +229,10 @@ matched. The following context conditions are defined:
     confirming the match. If false is returned, no signature match is
     going to be triggered. The function has to be of type ``function
     cond(state: signature_state, data: string): bool``. Here,
-    ``content`` may contain the most recent content chunk available at
+    ``data`` may contain the most recent content chunk available at
     the time the signature was matched. If no such chunk is available,
-    ``content`` will be the empty string. ``signature_state`` is
-    defined as follows:
-
-    .. code:: bro
-
-        type signature_state: record {
-            id: string;          # ID of the signature
-            conn: connection;    # Current connection
-            is_orig: bool;       # True if current endpoint is originator
-            payload_size: count; # Payload size of the first packet
-            };
-
+    ``data`` will be the empty string. See :bro:type:`signature_state`
+    for its definition.
 
 ``payload-size <cmp> <integer>``
     Compares the integer to the size of the payload of a packet. For

From 5f40e153a87b37e7621809a38545504b696202a0 Mon Sep 17 00:00:00 2001
From: Seth Hall <seth@icir.org>
Date: Thu, 23 Aug 2012 13:55:04 -0400
Subject: [PATCH 36/57] Adding an identifier to the SMTP blocklist notices for
 duplicate suppression.

 - Slight addition and revision to inline docs.
---
 scripts/policy/protocols/smtp/blocklists.bro | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/policy/protocols/smtp/blocklists.bro b/scripts/policy/protocols/smtp/blocklists.bro
index a3e75318bb..b1fb0e498d 100644
--- a/scripts/policy/protocols/smtp/blocklists.bro
+++ b/scripts/policy/protocols/smtp/blocklists.bro
@@ -1,3 +1,4 @@
+##! Watch for various SPAM blocklist URLs in SMTP error messages.  
 
 @load base/protocols/smtp
 
@@ -5,9 +6,11 @@ module SMTP;
 
 export {
 	redef enum Notice::Type += { 
-		## Indicates that the server sent a reply mentioning an SMTP block list.
+		## An SMTP server sent a reply mentioning an SMTP block list.
 		Blocklist_Error_Message,
-		## Indicates the client's address is seen in the block list error message.
+		## The originator's address is seen in the block list error message.
+		## This is useful to detect local hosts sending SPAM with a high
+		## positive rate.
 		Blocklist_Blocked_Host,
 	};
 
@@ -52,7 +55,8 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 				message = fmt("%s is on an SMTP block list", c$id$orig_h);
 				}
 			
-			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
+			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg, 
+			        $identifier=cat(c$id$orig_h)]);
 			}
 		}
 	}

From c1c9c9e34af571c5f204b4608b849823922c228f Mon Sep 17 00:00:00 2001
From: Daniel Thayer <dnthayer@ncsa.illinois.edu>
Date: Thu, 23 Aug 2012 13:04:18 -0500
Subject: [PATCH 37/57] Update documentation for builtin types

Add missing description of interval "msec" unit.
Improved description of pattern by clarifying the issue of operand
order and difference between exact and embedded matching.
---
 doc/scripts/builtins.rst | 24 ++++++++++++------------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/doc/scripts/builtins.rst b/doc/scripts/builtins.rst
index 32908f71fd..0501067409 100644
--- a/doc/scripts/builtins.rst
+++ b/doc/scripts/builtins.rst
@@ -55,8 +55,8 @@ The Bro scripting language supports the following built-in types.
 
     A temporal type representing a relative time.  An ``interval``
     constant can be written as a numeric constant followed by a time
-    unit where the time unit is one of ``usec``, ``sec``, ``min``,
-    ``hr``, or ``day`` which respectively represent microseconds,
+    unit where the time unit is one of ``usec``, ``msec``, ``sec``, ``min``,
+    ``hr``, or ``day`` which respectively represent microseconds, milliseconds,
     seconds, minutes, hours, and days.  Whitespace between the numeric
     constant and time unit is optional.  Appending the letter "s" to the
     time unit in order to pluralize it is also optional (to no semantic
@@ -95,14 +95,14 @@ The Bro scripting language supports the following built-in types.
     and embedded.
 
     In exact matching the ``==`` equality relational operator is used
-    with one :bro:type:`string` operand and one :bro:type:`pattern`
-    operand to check whether the full string exactly matches the
-    pattern.  In this case, the ``^`` beginning-of-line and ``$``
-    end-of-line anchors are redundant since pattern is implicitly
-    anchored to the beginning and end of the line to facilitate an exact
-    match.  For example::
+    with one :bro:type:`pattern` operand and one :bro:type:`string`
+    operand (order of operands does not matter) to check whether the full
+    string exactly matches the pattern.  In exact matching, the ``^``
+    beginning-of-line and ``$`` end-of-line anchors are redundant since
+    the pattern is implicitly anchored to the beginning and end of the
+    line to facilitate an exact match.  For example::
 
-        "foo" == /foo|bar/
+        /foo|bar/ == "foo"
 
     yields true, while::
 
@@ -110,9 +110,9 @@ The Bro scripting language supports the following built-in types.
 
     yields false.  The ``!=`` operator would yield the negation of ``==``.
 
-    In embedded matching the ``in`` operator is again used with one
-    :bro:type:`string` operand and one :bro:type:`pattern` operand
-    (which must be on the left-hand side), but tests whether the pattern
+    In embedded matching the ``in`` operator is used with one
+    :bro:type:`pattern` operand (which must be on the left-hand side) and
+    one :bro:type:`string` operand, but tests whether the pattern
     appears anywhere within the given string.  For example::
 
         /foo|bar/ in "foobar"

From 70f1403f1420b738d559c0675bd94703cd5af9aa Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 24 Aug 2012 13:18:51 -0700
Subject: [PATCH 38/57] Updating submodule(s).

 [nomail]
---
 CHANGES    | 2 +-
 VERSION    | 2 +-
 aux/broctl | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 066ee784a8..87da7378b0 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1-beta-54 | 2012-08-23 11:58:50 -0700
+2.1 | 2012-08-24 13:18:51 -0700
 
   * Update documentation for builtin types. (Daniel Thayer)
     
diff --git a/VERSION b/VERSION
index fd6e9996db..879b416e60 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-beta-54
+2.1
diff --git a/aux/broctl b/aux/broctl
index 5b3f9e5906..6b24757768 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 5b3f9e5906c90b76c5aa1626e112d4c991cb3fd8
+Subproject commit 6b24757768cd9aa742fd678d6864235519740ee8

From b5c694518904a5f122bc643c02f0518e11c3dade Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 24 Aug 2012 15:11:49 -0700
Subject: [PATCH 39/57] Updating submodule(s).

 [nomail]
---
 CHANGES     | 2 +-
 aux/bro-aux | 2 +-
 aux/broctl  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 87da7378b0..1c6e9dfafe 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1 | 2012-08-24 13:18:51 -0700
+2.1 | 2012-08-24 15:11:49 -0700
 
   * Update documentation for builtin types. (Daniel Thayer)
     
diff --git a/aux/bro-aux b/aux/bro-aux
index 4bc1a6f6a8..6748ec3a96 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 4bc1a6f6a8816dfacd8288fcf182ba35520e589b
+Subproject commit 6748ec3a96d582a977cd9114ef19c76fe75c57ff
diff --git a/aux/broctl b/aux/broctl
index 6b24757768..2fb9ff62bf 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 6b24757768cd9aa742fd678d6864235519740ee8
+Subproject commit 2fb9ff62bf08f78071753016863640022fbfe338

From 124c985d7af91a98eb8a7aff8f66b0300849e854 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 14:49:37 -0700
Subject: [PATCH 40/57] Bug found bei Keith & Seth: input framework was not
 handling counts and ints out of 32-bit-range correctly.

Note - another bugfix will be coming later (problem reading sets containing zero-length-strings & un-escaping-bug in sets)
---
 src/input/readers/Ascii.cc                    |  6 +--
 .../out                                       |  3 ++
 .../base/frameworks/input/bignumber.bro       | 44 +++++++++++++++++++
 3 files changed, 50 insertions(+), 3 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/bignumber.bro

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index fd936b07b6..28b1ed29c9 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -238,7 +238,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		break;
 
 	case TYPE_INT:
-		val->val.int_val = atoi(s.c_str());
+		val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10);
 		break;
 
 	case TYPE_DOUBLE:
@@ -249,7 +249,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 
 	case TYPE_COUNT:
 	case TYPE_COUNTER:
-		val->val.uint_val = atoi(s.c_str());
+		val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10);
 		break;
 
 	case TYPE_PORT:
@@ -344,7 +344,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 
 		if ( pos != length )
 			{
-			Error("Internal error while parsing set: did not find all elements");
+			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
 			return 0;
 			}
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
new file mode 100644
index 0000000000..ab095ca36c
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
@@ -0,0 +1,3 @@
+{
+[9223372036854775800] = [c=18446744073709551612]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro
new file mode 100644
index 0000000000..519992be05
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro
@@ -0,0 +1,44 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+9223372036854775800	18446744073709551612
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From 977c1d7c5adbf1b3bb2be55a99c4bd018e78a524 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 17:52:07 -0700
Subject: [PATCH 41/57] make set_separators different from , work for input
 framework.

1-line-patch + test.
---
 .../out                                       | 10 ++++
 .../base/frameworks/input/setseparator.bro    | 46 +++++++++++++++++++
 2 files changed, 56 insertions(+)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/setseparator.bro

diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out
new file mode 100644
index 0000000000..d0e0f53310
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.setseparator/out
@@ -0,0 +1,10 @@
+{
+[1] = [s={
+b,
+e,
+d,
+c,
+f,
+a
+}, ss=[1, 2, 3, 4, 5, 6]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro
new file mode 100644
index 0000000000..44b9d08d54
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro
@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	a|b|c|d|e|f	1|2|3|4|5|6
+@TEST-END-FILE
+
+redef InputAscii::set_separator = "|";
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	ss:vector of count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From 6bf733ce513a39804ba73b1e281adba5322f2de6 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 17:53:34 -0700
Subject: [PATCH 42/57] sorry. the patch for the set_separator.

---
 src/input/readers/Ascii.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 28b1ed29c9..e0be235700 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -288,7 +288,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		// how many entries do we have...
 		unsigned int length = 1;
 		for ( unsigned int i = 0; i < s.size(); i++ )
-			if ( s[i] == ',' ) length++;
+			if ( s[i] == set_separator[0] ) length++;
 
 		unsigned int pos = 0;
 

From a9e6d9ae8154eecb415f86ca9f786f21886fff94 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 19:17:21 -0700
Subject: [PATCH 43/57] Fix two little bugs:

Escaped ,'s in sets and vectors were unescaped before tokenization

Handling of zero-length-strings as last element in a set was broken (sets ending with a ,).

Hashing of lines just containing zero-length-strings was broken (now a \0 is appended to each
string before it is hashed - giving us a hash of something for a line just consisting of \0s.
This also allows to differentiate between vectors with varying numbers of zero-length-strings).
---
 src/input/Manager.cc                          |  6 ++-
 src/input/readers/Ascii.cc                    | 18 ++++++-
 .../out                                       | 20 ++++++++
 .../base/frameworks/input/setspecialcases.bro | 49 +++++++++++++++++++
 4 files changed, 90 insertions(+), 3 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/setspecialcases.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 3c29f14928..07ce5b20fc 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1718,7 +1718,7 @@ int Manager::GetValueLength(const Value* val) {
 	case TYPE_STRING:
 	case TYPE_ENUM:
 		{
-		length += val->val.string_val.length;
+		length += val->val.string_val.length+1;
 		break;
 		}
 
@@ -1818,7 +1818,9 @@ int Manager::CopyValue(char *data, const int startpos, const Value* val)
 	case TYPE_ENUM:
 		{
 		memcpy(data+startpos, val->val.string_val.data, val->val.string_val.length);
-		return val->val.string_val.length;
+		// and add a \0 to the end. To be able to hash zero-length strings and differentiate from !present
+		memset(data+startpos+val->val.string_val.length, 0, 1); 		
+		return val->val.string_val.length+1;
 		}
 
 	case TYPE_ADDR:
diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index e0be235700..4bf82c6a13 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -220,6 +220,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 	switch ( field.type ) {
 	case TYPE_ENUM:
 	case TYPE_STRING:
+		s = get_unescaped_string(s);
 		val->val.string_val.length = s.size();
 		val->val.string_val.data = copy_string(s.c_str());
 		break;
@@ -259,6 +260,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 
 	case TYPE_SUBNET:
 		{
+		s = get_unescaped_string(s);
 		size_t pos = s.find("/");
 		if ( pos == s.npos )
 			{
@@ -275,6 +277,7 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		}
 
 	case TYPE_ADDR:
+		s = get_unescaped_string(s);
 		val->val.addr_val = StringToAddr(s);
 		break;
 
@@ -342,6 +345,20 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			pos++;
 			}
 
+		// test if the string ends with a set_separator... if it does we have to push an zero-lenght
+		// val on top of it.
+		if ( *s.rbegin() == set_separator[0] )
+			{
+			lvals[pos] = EntryToVal("", field.subType());
+			if ( lvals[pos] == 0 )
+				{
+				Error("Error while trying to add empty set element");
+				return 0;
+				}
+
+			pos++;
+			}
+
 		if ( pos != length )
 			{
 			Error(Fmt("Internal error while parsing set: did not find all elements: %s", s.c_str()));
@@ -438,7 +455,6 @@ bool Ascii::DoUpdate()
 			if ( ! getline(splitstream, s, separator[0]) )
 				break;
 
-			s = get_unescaped_string(s);
 
 			stringfields[pos] = s;
 			pos++;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
new file mode 100644
index 0000000000..28d1e025bf
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
@@ -0,0 +1,20 @@
+{
+[2] = [s={
+,
+testing
+}, s=[testing, , testing]],
+[4] = [s={
+,
+testing
+}, s=[testing, ]],
+[1] = [s={
+testing,testing,testing,
+}, s=[testing,testing,testing,]],
+[5] = [s={
+
+}, s=[, , , ]],
+[3] = [s={
+,
+testing
+}, s=[, testing]]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
new file mode 100644
index 0000000000..29819a795f
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
@@ -0,0 +1,49 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	testing\x2ctesting\x2ctesting\x2c	testing\x2ctesting\x2ctesting\x2c
+2	testing,,testing	testing,,testing
+3	,testing	,testing
+4	testing,	testing,
+5	,,,	,,,
+@TEST-END-FILE
+
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	s: vector of string;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

From fbe464ffa348c59b980584ad321e206d9a794ac2 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 20:26:08 -0700
Subject: [PATCH 44/57] another small bug found while searching for something
 else...

...one of the change events got the wrong parameters. This actually is a bit embarassing...
---
 src/input/Manager.cc                          |   2 +-
 .../scripts.base.frameworks.input.reread/out  | 240 ++++++++++++++++--
 2 files changed, 223 insertions(+), 19 deletions(-)

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 07ce5b20fc..44d7140485 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1210,7 +1210,7 @@ void Manager::EndCurrentSend(ReaderFrontend* reader)
 			Ref(predidx);
 			Ref(val);
 			Ref(ev);
-			SendEvent(stream->event, 3, ev, predidx, val);
+			SendEvent(stream->event, 4, stream->description->Ref(), ev, predidx, val);
 			}
 
 		if ( predidx )  // if we have a stream or an event...
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
index 8b55ced2ac..acc9bfe846 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
@@ -1174,10 +1174,45 @@ BB
 }, vc=[10, 20, 30], ve=[]]
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-43]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1190,13 +1225,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-46]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1209,13 +1278,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-44]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1228,13 +1331,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-47]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1247,13 +1384,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
+Input::EVENT_REMOVED
+Left
 [i=-45]
-Left
+Right
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1266,13 +1437,47 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ============EVENT============
 Description
-Input::EVENT_REMOVED
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
 Type
-[i=-42]
+Input::EVENT_REMOVED
 Left
+[i=-42]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1285,7 +1490,6 @@ BB
 }, se={
 
 }, vc=[10, 20, 30], ve=[]]
-Right
 ==========SERVERS============
 {
 [-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={

From 7e46936728f08b1214a6610e194793eb145a1f37 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 20:49:21 -0700
Subject: [PATCH 45/57] Ok, this one is not really necessary for 2.1 and more
 of a nice-to-have

Before this patch, empty values were not hashed at all. Which had the unfortunate side-effect
that e.g. the lines

TEST	-
and
-	TEST

have the same hash values. On re-reads that means that the change will
be ignored.

This is probably pretty academic, but this patch changes it and adds a testcase.

Output of the reread test changes due to re-ordering of the output (probably
due to the fact that the internal hash values are changed and thus transferred
in a different order)
---
 src/input/Manager.cc                          |  17 +-
 .../out                                       | 155 +++++++++++
 .../scripts.base.frameworks.input.reread/out  | 248 +++++++++---------
 .../frameworks/input/empty-values-hashing.bro |  89 +++++++
 4 files changed, 382 insertions(+), 127 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

diff --git a/src/input/Manager.cc b/src/input/Manager.cc
index 44d7140485..e230c0e489 100644
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@@ -1911,11 +1911,16 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
 		const Value* val = vals[i];
 		if ( val->present )
 			length += GetValueLength(val);
-		}
 
-	if ( length == 0 )
+		// and in any case add 1 for the end-of-field-identifier
+		length++;
+		}
+	
+
+	assert ( length >= num_elements );
+
+	if ( length == num_elements )
 		{
-		reporter->Error("Input reader sent line where all elements are null values. Ignoring line");
 		return NULL;
 		}
 
@@ -1929,6 +1934,12 @@ HashKey* Manager::HashValues(const int num_elements, const Value* const *vals)
 		const Value* val = vals[i];
 		if ( val->present )
 			position += CopyValue(data, position, val);
+
+		memset(data+position, 1, 1); // add end-of-field-marker. does not really matter which value it is,
+					// it just has to be... something
+
+		position++;
+
 		}
 
 	HashKey *key = new HashKey(data, length);
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out
new file mode 100644
index 0000000000..474ef45cc2
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.empty-values-hashing/out
@@ -0,0 +1,155 @@
+============PREDICATE============
+Input::EVENT_NEW
+[i=1]
+[s=<uninitialized>, ss=TEST]
+============PREDICATE============
+Input::EVENT_NEW
+[i=2]
+[s=<uninitialized>, ss=<uninitialized>]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=1]
+Right
+[s=<uninitialized>, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_NEW
+Left
+[i=2]
+Right
+[s=<uninitialized>, ss=<uninitialized>]
+==========SERVERS============
+{
+[2] = [s=<uninitialized>, ss=<uninitialized>],
+[1] = [s=<uninitialized>, ss=TEST]
+}
+============PREDICATE============
+Input::EVENT_CHANGED
+[i=1]
+[s=TEST, ss=<uninitialized>]
+============PREDICATE============
+Input::EVENT_CHANGED
+[i=2]
+[s=TEST, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_CHANGED
+Left
+[i=1]
+Right
+[s=<uninitialized>, ss=TEST]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_CHANGED
+Left
+[i=2]
+Right
+[s=<uninitialized>, ss=<uninitialized>]
+==========SERVERS============
+{
+[2] = [s=TEST, ss=TEST],
+[1] = [s=TEST, ss=<uninitialized>]
+}
+done
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
index acc9bfe846..538a6dec18 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.reread/out
@@ -1084,7 +1084,7 @@ BB
 }
 ============PREDICATE============
 Input::EVENT_REMOVED
-[i=-43]
+[i=-44]
 [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1096,6 +1096,21 @@ AA,
 BB
 }, se={
 
+}, vc=[10, 20, 30], ve=[]]
+============PREDICATE============
+Input::EVENT_REMOVED
+[i=-42]
+[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
@@ -1111,21 +1126,6 @@ AA,
 BB
 }, se={
 
-}, vc=[10, 20, 30], ve=[]]
-============PREDICATE============
-Input::EVENT_REMOVED
-[i=-44]
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
@@ -1159,7 +1159,113 @@ BB
 }, vc=[10, 20, 30], ve=[]]
 ============PREDICATE============
 Input::EVENT_REMOVED
+[i=-43]
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
+[i=-44]
+Right
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+============EVENT============
+Description
+[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
+[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+2,
+4,
+1,
+3
+}, ss={
+CC,
+AA,
+BB
+}, se={
+
+}, vc=[10, 20, 30], ve=[]]
+}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
+{ 
+print A::outfile, ============EVENT============;
+print A::outfile, Description;
+print A::outfile, A::description;
+print A::outfile, Type;
+print A::outfile, A::tpe;
+print A::outfile, Left;
+print A::outfile, A::left;
+print A::outfile, Right;
+print A::outfile, A::right;
+}, pred=anonymous-function
+{ 
+print A::outfile, ============PREDICATE============;
+print A::outfile, A::typ;
+print A::outfile, A::left;
+print A::outfile, A::right;
+return (T);
+}, config={
+
+}]
+Type
+Input::EVENT_REMOVED
+Left
 [i=-42]
+Right
 [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
@@ -1207,59 +1313,6 @@ print A::outfile, A::right;
 return (T);
 }, config={
 
-}]
-Type
-Input::EVENT_REMOVED
-Left
-[i=-43]
-Right
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-============EVENT============
-Description
-[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
-[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-{ 
-print A::outfile, ============EVENT============;
-print A::outfile, Description;
-print A::outfile, A::description;
-print A::outfile, Type;
-print A::outfile, A::tpe;
-print A::outfile, Left;
-print A::outfile, A::left;
-print A::outfile, Right;
-print A::outfile, A::right;
-}, pred=anonymous-function
-{ 
-print A::outfile, ============PREDICATE============;
-print A::outfile, A::typ;
-print A::outfile, A::left;
-print A::outfile, A::right;
-return (T);
-}, config={
-
 }]
 Type
 Input::EVENT_REMOVED
@@ -1313,59 +1366,6 @@ print A::outfile, A::right;
 return (T);
 }, config={
 
-}]
-Type
-Input::EVENT_REMOVED
-Left
-[i=-44]
-Right
-[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-============EVENT============
-Description
-[source=../input.log, reader=Input::READER_ASCII, mode=Input::REREAD, name=ssh, destination={
-[-48] = [b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
-2,
-4,
-1,
-3
-}, ss={
-CC,
-AA,
-BB
-}, se={
-
-}, vc=[10, 20, 30], ve=[]]
-}, idx=<no value description>, val=<no value description>, want_record=T, ev=line
-{ 
-print A::outfile, ============EVENT============;
-print A::outfile, Description;
-print A::outfile, A::description;
-print A::outfile, Type;
-print A::outfile, A::tpe;
-print A::outfile, Left;
-print A::outfile, A::left;
-print A::outfile, Right;
-print A::outfile, A::right;
-}, pred=anonymous-function
-{ 
-print A::outfile, ============PREDICATE============;
-print A::outfile, A::typ;
-print A::outfile, A::left;
-print A::outfile, A::right;
-return (T);
-}, config={
-
 }]
 Type
 Input::EVENT_REMOVED
@@ -1476,9 +1476,9 @@ return (T);
 Type
 Input::EVENT_REMOVED
 Left
-[i=-42]
+[i=-43]
 Right
-[b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
+[b=F, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, sc={
 2,
 4,
 1,
diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
new file mode 100644
index 0000000000..b66febba82
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro
@@ -0,0 +1,89 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	-	TEST
+2	-	-
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	TEST	-
+2	TEST	TEST
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: string;
+	ss: string;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	}
+
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 2 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}

From f133e8808a0f8b199f47141f497cb33ed6a6955f Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Sun, 26 Aug 2012 22:00:37 -0700
Subject: [PATCH 46/57] ok, this one might really be a bit too big for 2.1

Give all kinds of errors when encountering invalid numbers (like out-of-range-warnings, etc).
---
 src/input/readers/Ascii.cc                    | 57 ++++++++++++++++---
 src/input/readers/Ascii.h                     |  1 +
 .../out                                       |  3 +-
 .../.stderrwithoutfirstline                   |  8 +++
 .../out                                       |  5 ++
 .../base/frameworks/input/bignumber.bro       |  1 +
 .../base/frameworks/input/invalidnumbers.bro  | 55 ++++++++++++++++++
 7 files changed, 122 insertions(+), 8 deletions(-)
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
 create mode 100644 testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
 create mode 100644 testing/btest/scripts/base/frameworks/input/invalidnumbers.bro

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 4bf82c6a13..1923532103 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -11,6 +11,7 @@
 #include <sys/types.h>
 #include <sys/stat.h>
 #include <unistd.h>
+#include <errno.h>
 
 using namespace input::reader;
 using threading::Value;
@@ -209,6 +210,34 @@ bool Ascii::GetLine(string& str)
 	return false;
 	}
 
+bool Ascii::CheckNumberError(const string & s, const char * end)
+	{
+
+	if ( s.length() == 0 )
+		{
+		Error("Got empty string for number field");
+		return true;
+		}
+
+	if ( end == s.c_str() ) {
+		Error(Fmt("String '%s' contained no parseable number", s.c_str()));
+		return true;
+	}
+
+	if ( *end != '\0' )
+		Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+
+	if ( errno == EINVAL )
+		{
+		Error(Fmt("String '%s' could not be converted to a number", s.c_str()));
+		return true;
+		}
+	else if ( errno == ERANGE ) 
+		Error(Fmt("Number '%s' out of supported range. Number was truncated", s.c_str()));
+
+	return false;
+	}
+
 
 Value* Ascii::EntryToVal(string s, FieldMapping field)
 	{
@@ -216,6 +245,8 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		return new Value(field.type, false);
 
 	Value* val = new Value(field.type, true);
+	char* end;
+	errno = 0;
 
 	switch ( field.type ) {
 	case TYPE_ENUM:
@@ -239,22 +270,31 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 		break;
 
 	case TYPE_INT:
-		val->val.int_val = strtoll(s.c_str(), (char**) NULL, 10);
+		val->val.int_val = strtoll(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 		break;
 
 	case TYPE_DOUBLE:
 	case TYPE_TIME:
 	case TYPE_INTERVAL:
-		val->val.double_val = atof(s.c_str());
+		val->val.double_val = strtod(s.c_str(), &end);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 		break;
 
 	case TYPE_COUNT:
 	case TYPE_COUNTER:
-		val->val.uint_val = strtoull(s.c_str(),(char**) NULL, 10);
+		val->val.uint_val = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 		break;
-
+		
 	case TYPE_PORT:
-		val->val.port_val.port = atoi(s.c_str());
+		val->val.port_val.port = strtoull(s.c_str(), &end, 10);
+		if ( CheckNumberError(s, end) ) 
+			return 0;
+
 		val->val.port_val.proto = TRANSPORT_UNKNOWN;
 		break;
 
@@ -268,8 +308,11 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			return 0;
 			}
 
-		int width = atoi(s.substr(pos+1).c_str());
+		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
 		string addr = s.substr(0, pos);
+		
+		if ( CheckNumberError(s, end) ) 
+			return 0;
 
 		val->val.subnet_val.prefix = StringToAddr(addr);
 		val->val.subnet_val.length = width;
@@ -490,7 +533,7 @@ bool Ascii::DoUpdate()
 			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
 			if ( val == 0 )
 				{
-				Error("Could not convert String value to Val");
+				Error(Fmt("Could not convert line '%s' to Val. Aborting file read.", line.c_str()));
 				return false;
 				}
 
diff --git a/src/input/readers/Ascii.h b/src/input/readers/Ascii.h
index e1506cbe82..2228e491b0 100644
--- a/src/input/readers/Ascii.h
+++ b/src/input/readers/Ascii.h
@@ -48,6 +48,7 @@ private:
 	bool ReadHeader(bool useCached);
 	bool GetLine(string& str);
 	threading::Value* EntryToVal(string s, FieldMapping type);
+	bool CheckNumberError(const string & s, const char * end);
 
 	ifstream* file;
 	time_t mtime;
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
index ab095ca36c..8b95ed8b19 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.bignumber/out
@@ -1,3 +1,4 @@
 {
-[9223372036854775800] = [c=18446744073709551612]
+[9223372036854775800] = [c=18446744073709551612],
+[-9223372036854775800] = [c=18446744073709551612]
 }
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
new file mode 100644
index 0000000000..bd32495a6f
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
@@ -0,0 +1,8 @@
+error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. Number was truncated
+error: ../input.log/Input::READER_ASCII: Number '121218446744073709551612' out of supported range. Number was truncated
+error: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
+error: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
+error: ../input2.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
+error: ../input2.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Aborting file read.
+received termination signal
+>>>
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
new file mode 100644
index 0000000000..9be82c13a9
--- /dev/null
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
@@ -0,0 +1,5 @@
+{
+[9223372036854775807] = [c=18446744073709551615],
+[9223372036854775800] = [c=4],
+[9223372036854775801] = [c=1]
+}
diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro
index 519992be05..250f84bbb2 100644
--- a/testing/btest/scripts/base/frameworks/input/bignumber.bro
+++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro
@@ -10,6 +10,7 @@
 #fields	i	c
 #types	int	count
 9223372036854775800	18446744073709551612
+-9223372036854775800	18446744073709551612
 @TEST-END-FILE
 
 @load frameworks/communication/listen
diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
new file mode 100644
index 0000000000..7914b53d94
--- /dev/null
+++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
@@ -0,0 +1,55 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+12129223372036854775800	121218446744073709551612
+9223372036854775801TEXTHERE	1Justtext
+9223372036854775800	-18446744073709551612
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+#separator \x09
+#fields	i	c
+#types	int	count
+Justtext	1
+@TEST-END-FILE
+
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	Input::add_table([$source="../input2.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers]);
+	}

From a4ca5b0d829fa61a706913848620d85f2b125dd6 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 27 Aug 2012 09:49:57 -0700
Subject: [PATCH 47/57] fix handline of sets only containing a zero-length
 string.

Thank you Robin...
---
 src/input/readers/Ascii.cc                                 | 7 ++++---
 .../scripts.base.frameworks.input.setspecialcases/out      | 3 +++
 .../scripts/base/frameworks/input/setspecialcases.bro      | 1 +
 3 files changed, 8 insertions(+), 3 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 4bf82c6a13..f1664a555a 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -345,9 +345,10 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			pos++;
 			}
 
-		// test if the string ends with a set_separator... if it does we have to push an zero-lenght
-		// val on top of it.
-		if ( *s.rbegin() == set_separator[0] )
+		// test if the string ends with a set_separator... or if the complete string is
+		// empty.
+		// In either of these cases we have to push an empty val on top of it.
+		if ( s.empty() ||  *s.rbegin() == set_separator[0] )
 			{
 			lvals[pos] = EntryToVal("", field.subType());
 			if ( lvals[pos] == 0 )
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
index 28d1e025bf..62229f7f37 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.setspecialcases/out
@@ -7,6 +7,9 @@ testing
 ,
 testing
 }, s=[testing, ]],
+[6] = [s={
+
+}, s=[]],
 [1] = [s={
 testing,testing,testing,
 }, s=[testing,testing,testing,]],
diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
index 29819a795f..239bdfe7e7 100644
--- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
+++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro
@@ -13,6 +13,7 @@
 3	,testing	,testing
 4	testing,	testing,
 5	,,,	,,,
+6		
 @TEST-END-FILE
 
 

From 5c486dae7e82ce308a6553a5dc53afb2fcae9ed8 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 27 Aug 2012 10:54:33 -0700
Subject: [PATCH 48/57] Ok, this one was a little bit sneaky.

If I understand things correctly, calling other string functions
on an stl string may alter the contents of the buffer to which
earlier .c_str()-calls pointed. Kind of makes sense when thinking
about it.

Basically moving around a few lines should fix this.

(And thank you again Robin)
---
 src/input/readers/Ascii.cc | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 1923532103..276391ef84 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -213,6 +213,9 @@ bool Ascii::GetLine(string& str)
 bool Ascii::CheckNumberError(const string & s, const char * end)
 	{
 
+	bool endnotnull =  (*end != '\0'); // do this check first, before executing s.c_str() or similar.
+	// otherwise the value to which *end is pointing at the moment might be gone...
+
 	if ( s.length() == 0 )
 		{
 		Error("Got empty string for number field");
@@ -224,7 +227,7 @@ bool Ascii::CheckNumberError(const string & s, const char * end)
 		return true;
 	}
 
-	if ( *end != '\0' )
+	if ( endnotnull )
 		Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
 
 	if ( errno == EINVAL )
@@ -309,11 +312,12 @@ Value* Ascii::EntryToVal(string s, FieldMapping field)
 			}
 
 		uint8_t width = (uint8_t) strtol(s.substr(pos+1).c_str(), &end, 10);
-		string addr = s.substr(0, pos);
-		
+
 		if ( CheckNumberError(s, end) ) 
 			return 0;
 
+		string addr = s.substr(0, pos);
+
 		val->val.subnet_val.prefix = StringToAddr(addr);
 		val->val.subnet_val.length = width;
 		break;

From 56fa56ffa946581d7b4806b494821fe79f9974dc Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Mon, 27 Aug 2012 11:38:20 -0700
Subject: [PATCH 49/57] ...and another small change to error handling -> now
 errors in single lines do not kill processing, but simply ignore the line,
 log it, and continue.

---
 src/input/readers/Ascii.cc                    | 28 ++++++++++++++++---
 .../.stderrwithoutfirstline                   | 12 ++++----
 .../out                                       |  1 -
 .../base/frameworks/input/invalidnumbers.bro  | 11 ++------
 4 files changed, 32 insertions(+), 20 deletions(-)

diff --git a/src/input/readers/Ascii.cc b/src/input/readers/Ascii.cc
index 276391ef84..9c25953864 100644
--- a/src/input/readers/Ascii.cc
+++ b/src/input/readers/Ascii.cc
@@ -228,7 +228,7 @@ bool Ascii::CheckNumberError(const string & s, const char * end)
 	}
 
 	if ( endnotnull )
-		Error(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
+		Warning(Fmt("Number '%s' contained non-numeric trailing characters. Ignored trailing characters '%s'", s.c_str(), end));
 
 	if ( errno == EINVAL )
 		{
@@ -236,7 +236,10 @@ bool Ascii::CheckNumberError(const string & s, const char * end)
 		return true;
 		}
 	else if ( errno == ERANGE ) 
-		Error(Fmt("Number '%s' out of supported range. Number was truncated", s.c_str()));
+		{
+		Error(Fmt("Number '%s' out of supported range.", s.c_str()));
+		return true;
+		}
 
 	return false;
 	}
@@ -492,6 +495,7 @@ bool Ascii::DoUpdate()
 	while ( GetLine(line ) )
 		{
 		// split on tabs
+		bool error = false;
 		istringstream splitstream(line);
 
 		map<int, string> stringfields;
@@ -537,8 +541,9 @@ bool Ascii::DoUpdate()
 			Value* val = EntryToVal(stringfields[(*fit).position], *fit);
 			if ( val == 0 )
 				{
-				Error(Fmt("Could not convert line '%s' to Val. Aborting file read.", line.c_str()));
-				return false;
+				Error(Fmt("Could not convert line '%s' to Val. Ignoring line.", line.c_str()));
+				error = true;
+				break;
 				}
 
 			if ( (*fit).secondary_position != -1 )
@@ -555,6 +560,21 @@ bool Ascii::DoUpdate()
 			fpos++;
 			}
 
+
+		if ( error ) 
+			{
+			// encountered non-fatal error. ignoring line.
+			// first - delete all successfully read fields and the array structure.
+			
+			for ( int i = 0; i < fpos; i++ )
+				delete fields[fpos];
+
+			delete[] fields;
+			continue;
+			}
+
+
+
 		//printf("fpos: %d, second.num_fields: %d\n", fpos, (*it).second.num_fields);
 		assert ( fpos == NumFields() );
 
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
index bd32495a6f..3ef51e40f2 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline
@@ -1,8 +1,8 @@
-error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. Number was truncated
-error: ../input.log/Input::READER_ASCII: Number '121218446744073709551612' out of supported range. Number was truncated
-error: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
-error: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
-error: ../input2.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
-error: ../input2.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Aborting file read.
+error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range.
+error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800	121218446744073709551612' to Val. Ignoring line.
+warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE'
+warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext'
+error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number
+error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext	1' to Val. Ignoring line.
 received termination signal
 >>>
diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
index 9be82c13a9..56b2736006 100644
--- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
+++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/out
@@ -1,5 +1,4 @@
 {
-[9223372036854775807] = [c=18446744073709551615],
 [9223372036854775800] = [c=4],
 [9223372036854775801] = [c=1]
 }
diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
index 7914b53d94..3c755f1d08 100644
--- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
+++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro
@@ -13,17 +13,10 @@
 #types	int	count
 12129223372036854775800	121218446744073709551612
 9223372036854775801TEXTHERE	1Justtext
+Justtext	1
 9223372036854775800	-18446744073709551612
 @TEST-END-FILE
 
-@TEST-START-FILE input2.log
-#separator \x09
-#fields	i	c
-#types	int	count
-Justtext	1
-@TEST-END-FILE
-
-
 @load frameworks/communication/listen
 
 global outfile: file;
@@ -51,5 +44,5 @@ event bro_init()
 event Input::update_finished(name: string, source:string)
 	{
 	print outfile, servers;
-	Input::add_table([$source="../input2.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers]);
+	terminate();
 	}

From 26f5aee7f6376d65031517efa78a1a6e7cbf1b46 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 28 Aug 2012 00:44:39 -0700
Subject: [PATCH 50/57] on 32-bit machines only unsigned long longs are 64-bits
 long. Not just unsigned longs...

Note that this means that up to now all outputs (including logs) of counts > 32 bits were broken on 32-bit systems.
---
 src/modp_numtoa.c | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/modp_numtoa.c b/src/modp_numtoa.c
index 6deb8a70ed..6fa49b460f 100644
--- a/src/modp_numtoa.c
+++ b/src/modp_numtoa.c
@@ -56,7 +56,7 @@ void modp_uitoa10(uint32_t value, char* str)
 void modp_litoa10(int64_t value, char* str)
 {
     char* wstr=str;
-    unsigned long uvalue = (value < 0) ? -value : value;
+    unsigned long long uvalue = (value < 0) ? -value : value;
 
     // Conversion. Number is reversed.
     do *wstr++ = (char)(48 + (uvalue % 10)); while(uvalue /= 10);

From 03f5795095642f89e11265ed36fda17f97a97ea9 Mon Sep 17 00:00:00 2001
From: Bernhard Amann <bernhard@icsi.berkeley.edu>
Date: Tue, 28 Aug 2012 07:33:05 -0700
Subject: [PATCH 51/57] parse 64-bit consts correctly.

---
 src/scan.l | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/scan.l b/src/scan.l
index 645ce659cd..3148ba58ad 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -439,7 +439,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 {D}		{
  		// TODO: check if we can use strtoull instead of atol,
 		// and similarly for {HEX}.
-		RET_CONST(new Val(static_cast<unsigned int>(atol(yytext)),
+		RET_CONST(new Val(static_cast<bro_uint_t>(strtoll(yytext, (char**) NULL, 10)),
 			  TYPE_COUNT))
 		}
 {FLOAT}		RET_CONST(new Val(atof(yytext), TYPE_DOUBLE))
@@ -483,7 +483,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 
 ({D}"."){3}{D}		RET_CONST(new AddrVal(yytext))
 
-"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtol(yytext, 0, 16)), TYPE_COUNT))
+"0x"{HEX}+	RET_CONST(new Val(static_cast<bro_uint_t>(strtoull(yytext, 0, 16)), TYPE_COUNT))
 
 {H}("."{H})+		RET_CONST(dns_mgr->LookupHost(yytext))
 

From b815b7ca5c133960102409d32bb492080112dde0 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 28 Aug 2012 10:57:21 -0500
Subject: [PATCH 52/57] Fix uninitialized value for 'is_partial' in TCP
 analyzer.

This led to non-deterministic behavior in cases where the first packet
analyzed wasn't from the originator side (see the conditionals in
TCP_Analyzer::CheckFlagCombos()).

The 'short' test in private test suite showed this behavior most often.
---
 src/TCP.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/TCP.cc b/src/TCP.cc
index 57e4449bf8..555adf1b57 100644
--- a/src/TCP.cc
+++ b/src/TCP.cc
@@ -46,6 +46,7 @@ TCP_Analyzer::TCP_Analyzer(Connection* conn)
 	finished = 0;
 	reassembling = 0;
 	first_packet_seen = 0;
+	is_partial = 0;
 
 	orig = new TCP_Endpoint(this, 1);
 	resp = new TCP_Endpoint(this, 0);

From 393ded1efe378a3f2109ccf49623e5050c12e048 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 28 Aug 2012 09:19:33 -0700
Subject: [PATCH 53/57] Set VERSION to 2.1-rc3 so that we don't get confused.

---
 CHANGES | 2 +-
 VERSION | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 02d7d74046..7df00f352c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1 | 2012-08-24 15:11:49 -0700
+2.1-rc3 | 2012-08-24 15:11:49 -0700
 
   * Input framework fixes, including:  (Bernhard Amann)
 
diff --git a/VERSION b/VERSION
index 879b416e60..1537f322a9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1
+2.1-rc3

From cc49193f93ba8c60b65b61047a0874982ad93db3 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@ncsa.illinois.edu>
Date: Tue, 28 Aug 2012 13:11:12 -0500
Subject: [PATCH 54/57] Remove automatic use of gperftools on non-Linux
 systems.

--enable-perftools must now explicity be supplied to ./configure on
non-Linux systems to link against the tcmalloc library that a
gperftools installation provides.  Linux systems still automatically
link it if it's found.  The rationale is that gperftools was developed
and most throroughly tested on Linux so it's safer there.

There especially seems to be potential problems with gperftools on OS X
(e.g. see http://code.google.com/p/gperftools/issues/detail?id=413),
and Bro currently doesn't work with gpertools there using clang or gcc.
---
 CMakeLists.txt | 29 ++++++++++++++++++-----------
 configure      |  7 +++++++
 2 files changed, 25 insertions(+), 11 deletions(-)

diff --git a/CMakeLists.txt b/CMakeLists.txt
index f667c0cfe0..2c8a726a1a 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -88,24 +88,30 @@ if (LIBGEOIP_FOUND)
     list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
 endif ()
 
-set(USE_PERFTOOLS false)
+set(HAVE_PERFTOOLS false)
 set(USE_PERFTOOLS_DEBUG false)
+set(USE_PERFTOOLS_TCMALLOC false)
 
 if (NOT DISABLE_PERFTOOLS)
    find_package(GooglePerftools)
 endif ()
 
 if (GOOGLEPERFTOOLS_FOUND)
-    include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
-    set(USE_PERFTOOLS true)
+    set(HAVE_PERFTOOLS true)
+    # Non-Linux systems may not be well-supported by gperftools, so
+    # require explicit request from user to enable it in that case.
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS)
+        set(USE_PERFTOOLS_TCMALLOC true)
 
-    if (ENABLE_PERFTOOLS_DEBUG)
-        # Enable heap debugging with perftools.
-        set(USE_PERFTOOLS_DEBUG true)
-        list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
-    else ()
-        # Link in tcmalloc for better performance.
-        list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
+        if (ENABLE_PERFTOOLS_DEBUG)
+            # Enable heap debugging with perftools.
+            set(USE_PERFTOOLS_DEBUG true)
+            include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+            list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
+        else ()
+            # Link in tcmalloc for better performance.
+            list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
+        endif ()
     endif ()
 endif ()
 
@@ -224,7 +230,8 @@ message(
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
-    "\nGoogle perftools:  ${USE_PERFTOOLS}"
+    "\ngperftools found:  ${HAVE_PERFTOOLS}"
+    "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
     "\ncURL:              ${USE_CURL}"
     "\n"
diff --git a/configure b/configure
index b4ca606103..8e4aaa8425 100755
--- a/configure
+++ b/configure
@@ -29,6 +29,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
   Optional Features:
     --enable-debug         compile in debugging mode
     --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
+    --enable-perftools     force use of Google perftools on non-Linux systems
+                           (automatically on when perftools is present on Linux)
     --enable-perftools-debug use Google's perftools for debugging
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
@@ -98,6 +100,7 @@ append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/broctl
 append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
+append_cache_entry ENABLE_PERFTOOLS     BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
@@ -146,7 +149,11 @@ while [ $# -ne 0 ]; do
         --enable-mobile-ipv6)
             append_cache_entry ENABLE_MOBILE_IPV6         BOOL   true
             ;;
+        --enable-perftools)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
+            ;;
         --enable-perftools-debug)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
         --disable-broccoli)

From e4b7ffa8ac0718ace6d37371c8283efc50502c4f Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 28 Aug 2012 16:44:30 -0700
Subject: [PATCH 55/57] Updating CHANGES and VERSION.

---
 VERSION | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/VERSION b/VERSION
index 2097bb1d94..9459d4ba2a 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.1-rc3-5
+1.1

From b915db86d5c7b30c7d50d8b5ddfbbbdadd32107d Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Tue, 28 Aug 2012 16:46:37 -0700
Subject: [PATCH 56/57] Updating CHANGES and VERSION.

---
 CHANGES | 2 +-
 VERSION | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGES b/CHANGES
index 232e2faa19..516c36974e 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-1.1 | 2012-08-28 16:29:30 -0700
+2.1 | 2012-08-28 16:46:42 -0700
 
   * Remove automatic use of gperftools on non-Linux systems.
     --enable-perftools must now explicity be supplied to ./configure
diff --git a/VERSION b/VERSION
index 9459d4ba2a..879b416e60 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-1.1
+2.1

From 22cf75dae553dc2aa2a103bf7721cd466b764d64 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Wed, 29 Aug 2012 08:09:44 -0700
Subject: [PATCH 57/57] Two fixes.

    - Typo in recent scanner fix.

    - Make bif.identify_magic robust against FreeBSD's libmagic config.
---
 CHANGES                                       | 3 +++
 src/scan.l                                    | 4 +---
 testing/btest/Baseline/bifs.identify_data/out | 2 +-
 testing/btest/bifs/identify_data.bro          | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/CHANGES b/CHANGES
index 516c36974e..f8e4444f1d 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,6 +1,9 @@
 
 2.1 | 2012-08-28 16:46:42 -0700
 
+  * Make bif.identify_magic robust against FreeBSD's libmagic config.
+    (Robin Sommer)
+
   * Remove automatic use of gperftools on non-Linux systems.
     --enable-perftools must now explicity be supplied to ./configure
     on non-Linux systems to link against the tcmalloc library.
diff --git a/src/scan.l b/src/scan.l
index 3148ba58ad..1b3d09f879 100644
--- a/src/scan.l
+++ b/src/scan.l
@@ -437,9 +437,7 @@ F	RET_CONST(new Val(false, TYPE_BOOL))
 	}
 
 {D}		{
- 		// TODO: check if we can use strtoull instead of atol,
-		// and similarly for {HEX}.
-		RET_CONST(new Val(static_cast<bro_uint_t>(strtoll(yytext, (char**) NULL, 10)),
+		RET_CONST(new Val(static_cast<bro_uint_t>(strtoul(yytext, (char**) NULL, 10)),
 			  TYPE_COUNT))
 		}
 {FLOAT}		RET_CONST(new Val(atof(yytext), TYPE_DOUBLE))
diff --git a/testing/btest/Baseline/bifs.identify_data/out b/testing/btest/Baseline/bifs.identify_data/out
index a2872877f9..1cadefbf6e 100644
--- a/testing/btest/Baseline/bifs.identify_data/out
+++ b/testing/btest/Baseline/bifs.identify_data/out
@@ -1,4 +1,4 @@
 ASCII text, with no line terminators
 text/plain; charset=us-ascii
-PNG image data
+PNG image
 image/png; charset=binary
diff --git a/testing/btest/bifs/identify_data.bro b/testing/btest/bifs/identify_data.bro
index 11824b5e85..39f289d40b 100644
--- a/testing/btest/bifs/identify_data.bro
+++ b/testing/btest/bifs/identify_data.bro
@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro %INPUT >out
+# @TEST-EXEC: bro %INPUT | sed 's/PNG image data/PNG image/g' >out
 # @TEST-EXEC: btest-diff out
 
 event bro_init()