Merge remote-tracking branch 'origin/master' into topic/seth/metrics-updates

Conflicts:
	testing/btest/Baseline/scripts.base.frameworks.metrics.basic-cluster/manager-1.metrics.log
	testing/btest/Baseline/scripts.base.frameworks.metrics.basic/metrics.log
	testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log
	testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log
	testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro

testing/btest/scripts/base/frameworks/cluster/start-it-up.bro

@@ -1,11 +1,13 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
+# @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff proxy-1/.stdout
 # @TEST-EXEC: btest-diff proxy-2/.stdout
@@ -22,7 +24,42 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
+global fully_connected: event();
+
+global peer_count = 0;
+
+global fully_connected_nodes = 0;
+
+event fully_connected()
+	{
+	fully_connected_nodes = fully_connected_nodes + 1;
+	if ( Cluster::node == "manager-1" )
+		{
+		if ( peer_count == 4 && fully_connected_nodes == 4 )
+			terminate_communication();
+		}
+	}
+
+redef Cluster::worker2manager_events += /fully_connected/;
+redef Cluster::proxy2manager_events += /fully_connected/;
+
 event remote_connection_handshake_done(p: event_peer)
 	{
 	print "Connected to a peer";
+	peer_count = peer_count + 1;
+	if ( Cluster::node == "manager-1" )
+		{
+		if ( peer_count == 4 && fully_connected_nodes == 4 )
+			terminate_communication();
+		}
+	else
+		{
+		if ( peer_count == 2 )
+			event fully_connected();
+		}
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
 	}

testing/btest/scripts/base/frameworks/communication/communication_log_baseline.bro

@@ -1,11 +1,11 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run receiver bro -b ../receiver.bro
 # @TEST-EXEC: btest-bg-run sender   bro -b ../sender.bro
 # @TEST-EXEC: btest-bg-wait -k 10
 #
 # Don't diff the receiver log just because port is always going to change
-# @TEST-EXEC: egrep -v 'pid|socket buffer size' sender/communication.log >send.log
+# @TEST-EXEC: egrep -v 'CPU|bytes|pid|socket buffer size' sender/communication.log >send.log
 # @TEST-EXEC: btest-diff send.log
 
 @TEST-START-FILE sender.bro
@@ -19,6 +19,10 @@ redef Communication::nodes += {
 event remote_connection_handshake_done(p: event_peer)
 	{
 	terminate_communication();
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
 	terminate();
 	}
 
@@ -30,9 +34,8 @@ event remote_connection_handshake_done(p: event_peer)
 
 @load frameworks/communication/listen
 
-event remote_connection_handshake_done(p: event_peer)
+event remote_connection_closed(p: event_peer)
 	{
-	terminate_communication();
 	terminate();
 	}
 

testing/btest/scripts/base/frameworks/control/configuration_update.bro

@@ -1,7 +1,9 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65531/tcp 
+# @TEST-EXEC: sleep 5
 # @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
+# @TEST-EXEC: sleep 5
 # @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown
 # @TEST-EXEC: btest-bg-wait 10
 # @TEST-EXEC: btest-diff controllee/.stdout

testing/btest/scripts/base/frameworks/control/id_value.bro

@@ -1,4 +1,4 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port=65532/tcp 
 # @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var
@@ -22,4 +22,5 @@ redef test_var = "This is the value from the controllee";
 event Control::id_value_response(id: string, val: string)
 	{
 	print fmt("Got an id_value_response(%s, %s) event", id, val);
+	terminate();
 	}

testing/btest/scripts/base/frameworks/control/shutdown.bro

@@ -1,4 +1,4 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port=65530/tcp 
 # @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown

testing/btest/scripts/base/frameworks/input/basic.bro

@@ -0,0 +1,62 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/bignumber.bro

@@ -0,0 +1,45 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+9223372036854775800	18446744073709551612
+-9223372036854775800	18446744073709551612
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/binary.bro

@@ -0,0 +1,56 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+redef InputAscii::separator = "|";
+redef InputAscii::set_separator = ",";
+redef InputAscii::empty_field = "(empty)";
+redef InputAscii::unset_field = "-";
+
+@TEST-START-FILE input.log
+#separator |
+#set_separator|,
+#empty_field|(empty)
+#unset_field|-
+#path|ssh
+#open|2012-07-20-01-49-19
+#fields|data|data2
+#types|string|string
+abc\x0a\xffdef|DATA2
+abc\x7c\xffdef|DATA2
+abc\xff\x7cdef|DATA2
+#end|2012-07-20-01-49-19
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+type Val: record {
+		data: string;
+		data2: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, a: string, b: string)
+	{
+	print outfile, a;
+	print outfile, b;
+	try = try + 1;
+	if ( try == 3 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro

@@ -0,0 +1,89 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	-	TEST
+2	-	-
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#fields	i	s	ss
+#types	int	sting	string
+1	TEST	-
+2	TEST	TEST
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: string;
+	ss: string;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	}
+
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 2 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}

testing/btest/scripts/base/frameworks/input/emptyvals.bro

@@ -0,0 +1,48 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i
+##types	bool	int
+T	1
+-	2
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+    outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/event.bro

@@ -0,0 +1,54 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	i: int;
+	b: bool;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, i;
+	print outfile, b;
+	try = try + 1;
+	if ( try == 7 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/executeraw.bro

@@ -0,0 +1,42 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	close(outfile);
+	terminate();
+	}
+
+event bro_init()
+	{
+	outfile = open("../out.tmp");
+	Input::add_event([$source="wc -l ../input.log |", $reader=Input::READER_RAW, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/invalidnumbers.bro

@@ -0,0 +1,48 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	c
+#types	int	count
+12129223372036854775800	121218446744073709551612
+9223372036854775801TEXTHERE	1Justtext
+Justtext	1
+9223372036854775800	-18446744073709551612
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	c: count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/missing-file.bro

@@ -0,0 +1,30 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff bro/.stderr
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	i: int;
+	b: bool;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, i: int, b: bool)
+	{
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="does-not-exist.dat", $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro

@@ -0,0 +1,47 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	
+#types	bool	int
+T	-42	
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}
+

testing/btest/scripts/base/frameworks/input/onecolumn-record.bro

@@ -0,0 +1,47 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	b	i	
+#types	bool	int
+T	-42	
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	Input::add_table([$name="input", $source="../input.log", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}
+

testing/btest/scripts/base/frameworks/input/optional.bro

@@ -0,0 +1,56 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	notb: bool &optional;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+    outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
+				$pred(typ: Input::Event, left: Idx, right: Val) = { right$notb = !right$b; return T; }
+				]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/port.bro

@@ -0,0 +1,53 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#fields	i	p	t
+1.2.3.4	80	tcp
+1.2.3.5	52	udp
+1.2.3.6	30	unknown
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: addr;
+};
+
+type Val: record {
+	p: port &type_column="t";
+};
+
+global servers: table[addr] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers]);
+	if ( 1.2.3.4 in servers )
+		print outfile, servers[1.2.3.4];
+	if ( 1.2.3.5 in servers )
+		print outfile, servers[1.2.3.5];
+	if ( 1.2.3.6 in servers )
+		print outfile, servers[1.2.3.6];
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers[1.2.3.4];
+	print outfile, servers[1.2.3.5];
+	print outfile, servers[1.2.3.6];
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/predicate-stream.bro

@@ -0,0 +1,79 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+#
+# only difference from predicate.bro is, that this one uses a stream source.
+# the reason is, that the code-paths are quite different, because then the
+# ascii reader uses the put and not the sendevent interface
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of Val = table();
+global ct: int;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool)
+	{
+	ct = ct + 1;
+	if ( ct < 3 )
+		return;
+
+	if ( 1 in servers )
+		print outfile, "VALID";
+	if ( 2 in servers )
+		print outfile, "VALID";
+	if ( !(3 in servers) )
+		print outfile, "VALID";
+	if ( !(4 in servers) )
+		print outfile, "VALID";
+	if ( !(5 in servers) )
+		print outfile, "VALID";
+	if ( !(6 in servers) )
+		print outfile, "VALID";
+	if ( 7 in servers )
+		print outfile, "VALID";
+	close(outfile);
+	terminate();
+	}
+
+event bro_init()
+	{
+    outfile = open("../out");
+	ct = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F, $ev=line,
+				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
+				]);
+	Input::remove("input");
+	}
+

testing/btest/scripts/base/frameworks/input/predicate.bro

@@ -0,0 +1,68 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global servers: table[int] of bool = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $want_record=F,
+				$pred(typ: Input::Event, left: Idx, right: bool) = { return right; }
+				]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	if ( 1 in servers )
+		print outfile, "VALID";
+	if ( 2 in servers )
+		print outfile, "VALID";
+	if ( !(3 in servers) )
+		print outfile, "VALID";
+	if ( !(4 in servers) )
+		print outfile, "VALID";
+	if ( !(5 in servers) )
+		print outfile, "VALID";
+	if ( !(6 in servers) )
+		print outfile, "VALID";
+	if ( 7 in servers )
+		print outfile, "VALID";
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/predicatemodify.bro

@@ -0,0 +1,59 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	T	test1	idx1
+2	T	test2	idx2
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+	ss: string;
+};
+
+type Val: record {
+	b: bool;
+	s: string;
+};
+
+global servers: table[int, string] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, 
+				$pred(typ: Input::Event, left: Idx, right: Val) = { 
+				if ( left$i == 1 )
+					right$s = "testmodified";
+				if ( left$i == 2 )
+					left$ss = "idxmodified";
+				return T; 
+				}
+				]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro

@@ -0,0 +1,109 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input4.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input5.log input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+#
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	T	test1	idx1
+2	T	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	F	test1	idx1
+2	T	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	F	test1	idx1
+2	F	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input4.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+2	F	test2	idx2
+@TEST-END-FILE
+
+@TEST-START-FILE input5.log
+#separator \x09
+#path	ssh
+#fields	i	b	s	ss
+#types	int	bool	string	string
+1	T	test1	idx1
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+	ss: string;
+};
+
+type Val: record {
+	b: bool;
+	s: string;
+};
+
+global servers: table[int, string] of Val = table();
+global outfile: file;
+global try: count;
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers, $mode=Input::REREAD,
+				$pred(typ: Input::Event, left: Idx, right: Val) = { 
+				if ( left$i == 1 )
+					right$s = "testmodified";
+				if ( left$i == 2 )
+					left$ss = "idxmodified";
+				return T; 
+				}
+				]);
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	try = try + 1;
+	print outfile, fmt("Update_finished for %s, try %d", name, try);
+	print outfile, servers;
+	
+	if ( try == 5 )
+		{
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}

testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro

@@ -0,0 +1,56 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+# Ok, this one tests a fun case.
+# Input file contains two lines mapping to the same index, but with different values,
+# where the predicate accepts the first one and refuses the second one.
+# Desired result -> first entry stays.
+
+@TEST-START-FILE input.log
+#fields	restriction	guid	severity	confidence	detecttime	address	protocol	portlist	asn	prefix	rir	cc	impact	description	alternativeid_restriction	alternativeid
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.0.17.227	-	-	2519 VECTANT VECTANT Ltd.	1.0.16.0/23	apnic	JP	spam infrastructure	spamming	public	http://reputation.alienvault.com/reputation.generic
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	95	1342569600	1.228.83.33	6	25	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	direct ube sources, spam operations & spam services	public	http://www.spamhaus.org/query/bl?ip=1.228.83.33
+need-to-know	8c864306-d21a-37b1-8705-746a786719bf	medium	65	1342656000	1.228.83.33	-	-	9318 HANARO-AS Hanaro Telecom Inc.	1.224.0.0/13	apnic	KR	spam infrastructure	spamming;malware domain	public	http://reputation.alienvault.com/reputation.generic
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	address: addr;
+};
+
+type Val: record {
+	asn: string;
+	severity: string;
+	confidence: count;
+	detecttime: time;
+};
+
+global servers: table[addr] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=servers,
+				$pred(typ: Input::Event, left: Idx, right: Val) = { if ( right$confidence > 90 ) { return T; } return F; }
+				]);
+	Input::remove("input");
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/raw.bro

@@ -0,0 +1,49 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	try = try + 1;
+	if ( try == 8 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/repeat.bro

@@ -0,0 +1,59 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global destination: table[int] of Val = table();
+
+const one_to_32: vector of count = {1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32};
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	for ( i in one_to_32 )
+		{
+		Input::add_table([$source="../input.log", $name=fmt("input%d", i), $idx=Idx, $val=Val, $destination=destination, $want_record=F]);
+		Input::remove(fmt("input%d", i));
+		}
+	}
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, name;
+	print outfile, source;
+	print outfile, destination;
+	try = try + 1;
+	if ( try == 32 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}

testing/btest/scripts/base/frameworks/input/reread.bro

@@ -0,0 +1,139 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input2.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input4.log input.log
+# @TEST-EXEC: sleep 2
+# @TEST-EXEC: cp input5.log input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input4.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-44	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-45	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-46	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-47	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+F	-48	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input5.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+F	-48	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print outfile, "============EVENT============";
+	print outfile, "Description";
+	print outfile, description;
+	print outfile, "Type";
+	print outfile, tpe;
+	print outfile, "Left";
+	print outfile, left;
+	print outfile, "Right";
+	print outfile, right;
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print outfile, "============PREDICATE============";
+	print outfile, typ;
+	print outfile, left;
+	print outfile, right;
+	return T;
+	}
+	]);
+	}
+
+
+event Input::update_finished(name: string, source: string)
+	{
+	print outfile, "==========SERVERS============";
+	print outfile, servers;
+	
+	try = try + 1;
+	if ( try == 5 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}

testing/btest/scripts/base/frameworks/input/rereadraw.bro

@@ -0,0 +1,50 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+	try = try + 1;
+	if ( try == 16 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::REREAD, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	Input::force_update("input");
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/set.bro

@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	ip
+#types	addr
+192.168.17.1
+192.168.17.2
+192.168.17.7
+192.168.17.14
+192.168.17.42
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	ip: addr;
+};
+
+global servers: set[addr] = set();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/setseparator.bro

@@ -0,0 +1,46 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	a|b|c|d|e|f	1|2|3|4|5|6
+@TEST-END-FILE
+
+redef InputAscii::set_separator = "|";
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	ss:vector of count;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/setspecialcases.bro

@@ -0,0 +1,50 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#fields	i	s	ss
+1	testing\x2ctesting\x2ctesting\x2c	testing\x2ctesting\x2ctesting\x2c
+2	testing,,testing	testing,,testing
+3	,testing	,testing
+4	testing,	testing,
+5	,,,	,,,
+6		
+@TEST-END-FILE
+
+
+@load frameworks/communication/listen
+
+global outfile: file;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	s: set[string];
+	s: vector of string;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/stream.bro

@@ -0,0 +1,88 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+F	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global outfile: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print outfile, "============EVENT============";
+	print outfile, tpe;
+	print outfile, left;
+	print outfile, right;
+	print outfile, "============SERVERS============";
+	print outfile, servers;
+
+	try = try + 1;
+	
+	if ( try == 3 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::STREAM, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line]);
+	}

testing/btest/scripts/base/frameworks/input/streamraw.bro

@@ -0,0 +1,62 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT 
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input2.log >> input.log
+# @TEST-EXEC: sleep 3
+# @TEST-EXEC: cat input3.log >> input.log
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input1.log
+sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF
+@TEST-END-FILE
+
+@TEST-START-FILE input2.log
+DSF"DFKJ"SDFKLh304yrsdkfj@#(*U$34jfDJup3UF
+q3r3057fdf
+@TEST-END-FILE
+
+@TEST-START-FILE input3.log
+sdfs\d
+
+dfsdf
+sdf
+3rw43wRRERLlL#RWERERERE.
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+module A;
+
+type Val: record {
+	s: string;
+};
+
+global try: count;
+global outfile: file;
+
+event line(description: Input::EventDescription, tpe: Input::Event, s: string)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, s;
+
+	try = try + 1;
+	if ( try == 8 )
+		{
+		print outfile, "done";
+		close(outfile);
+		Input::remove("input");
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	outfile = open("../out");
+	try = 0;
+	Input::add_event([$source="../input.log", $reader=Input::READER_RAW, $mode=Input::STREAM, $name="input", $fields=Val, $ev=line, $want_record=F]);
+	}

testing/btest/scripts/base/frameworks/input/subrecord-event.bro

@@ -0,0 +1,75 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	sub.b	i	sub.e	sub.c	sub.p	sub.sn	sub.two.a	sub.two.d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type SubVal2: record {
+	a: addr;
+	d: double;
+};
+
+type SubVal: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	two: SubVal2;
+};
+
+type Val: record {
+	sub: SubVal;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+
+
+event line(description: Input::EventDescription, tpe: Input::Event, value: Val)
+	{
+	print outfile, value;
+	try = try + 1;
+	if ( try == 1 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;	
+	outfile = open("../out");
+	Input::add_event([$source="../input.log", $name="ssh", $fields=Val, $ev=line, $want_record=T]);
+	Input::remove("ssh");
+	}

testing/btest/scripts/base/frameworks/input/subrecord.bro

@@ -0,0 +1,70 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	sub.b	i	sub.e	sub.c	sub.p	sub.sn	sub.two.a	sub.two.d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type SubVal2: record {
+	a: addr;
+	d: double;
+};
+
+type SubVal: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	two: SubVal2;
+};
+
+type Val: record {
+	sub: SubVal;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/input/tableevent.bro

@@ -0,0 +1,59 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	i	b	
+#types	int	bool
+1	T
+2	T
+3	F
+4	F
+5	F
+6	F
+7	T
+@TEST-END-FILE
+
+@load frameworks/communication/listen
+
+global outfile: file;
+global try: count;
+
+redef InputAscii::empty_field = "EMPTY";
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+};
+
+global destination: table[int] of Val = table();
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool)
+	{
+	print outfile, description;
+	print outfile, tpe;
+	print outfile, left;
+	print outfile, right;
+	try = try + 1;
+	if ( try == 7 )
+		{
+		close(outfile);
+		terminate();
+		}
+	}
+
+event bro_init()
+	{
+	try = 0;
+	outfile = open("../out");
+	Input::add_table([$source="../input.log", $name="input", $idx=Idx, $val=Val, $destination=destination, $want_record=F,$ev=line]);
+	Input::remove("input");
+	}

testing/btest/scripts/base/frameworks/input/twotables.bro

@@ -0,0 +1,134 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: cp input1.log input.log
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: sleep 5
+# @TEST-EXEC: cp input3.log input.log
+# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-diff event.out
+# @TEST-EXEC: btest-diff pred1.out
+# @TEST-EXEC: btest-diff pred2.out
+# @TEST-EXEC: btest-diff fin.out
+
+@TEST-START-FILE input1.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input2.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+T	-43	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+@TEST-START-FILE input3.log
+#separator \x09
+#path	ssh
+#fields	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+F	-44	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+redef InputAscii::empty_field = "EMPTY";
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+global event_out: file;
+global pred1_out: file;
+global pred2_out: file;
+global fin_out: file;
+
+global try: count;
+
+event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val)
+	{
+	print event_out, "============EVENT============";
+#	print event_out, "Description";
+#	print event_out, description;
+#	print event_out, "Type";
+#	print event_out, tpe;
+#	print event_out, "Left";
+#	print event_out, left;
+#	print event_out, "Right";
+#	print event_out, right;
+	}
+
+event bro_init()
+	{
+	event_out = open ("../event.out");
+	pred1_out = open ("../pred1.out");
+	pred2_out = open ("../pred2.out");
+	fin_out = open ("../fin.out");
+	try = 0;
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $mode=Input::REREAD, $name="ssh", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print pred1_out, "============PREDICATE============";
+	print pred1_out, typ;
+	print pred1_out, left;
+	print pred1_out, right;
+	return T;
+	}
+	]);
+	Input::add_table([$source="../input2.log", $mode=Input::REREAD, $name="ssh2", $idx=Idx, $val=Val, $destination=servers, $ev=line,
+	$pred(typ: Input::Event, left: Idx, right: Val) = { 
+	print pred2_out, "============PREDICATE 2============";
+	print pred2_out, typ;
+	print pred2_out, left;
+	print pred2_out, right;
+	return T;
+	}
+	]);
+	}
+
+
+event Input::update_finished(name: string, source: string)
+	{
+	print fin_out, "==========SERVERS============";
+	#print fin_out, servers;
+	
+	try = try + 1;
+	if ( try == 3 )
+		{
+		print fin_out, "done";
+		print fin_out, servers;
+		close(event_out);
+		close(pred1_out);
+		close(pred2_out);
+		close(fin_out);
+		Input::remove("input");
+		Input::remove("input2");
+		terminate();
+		}
+	}

testing/btest/scripts/base/frameworks/input/unsupported_types.bro

@@ -0,0 +1,64 @@
+# (uses listen.bro just to ensure input sources are more reliably fully-read).
+# @TEST-SERIALIZE: comm
+#
+# @TEST-EXEC: btest-bg-run bro bro -b %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff out
+
+@TEST-START-FILE input.log
+#separator \x09
+#path	ssh
+#fields	fi	b	i	e	c	p	sn	a	d	t	iv	s	sc	ss	se	vc	ve	f
+#types	file	bool	int	enum	count	port	subnet	addr	double	time	interval	string	table	table	table	vector	vector	func
+whatever	T	-42	SSH::LOG	21	123	10.0.0.0/24	1.2.3.4	3.14	1315801931.273616	100.000000	hurz	2,4,1,3	CC,AA,BB	EMPTY	10,20,30	EMPTY	SSH::foo\x0a{ \x0aif (0 < SSH::i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a}
+@TEST-END-FILE
+
+@load base/protocols/ssh
+@load frameworks/communication/listen
+
+global outfile: file;
+
+redef InputAscii::empty_field = "EMPTY";
+redef Input::accept_unsupported_types = T;
+
+module A;
+
+type Idx: record {
+	i: int;
+};
+
+type Val: record {
+	fi: file &optional;
+	b: bool;
+	e: Log::ID;
+	c: count;
+	p: port;
+	sn: subnet;
+	a: addr;
+	d: double;
+	t: time;
+	iv: interval;
+	s: string;
+	sc: set[count];
+	ss: set[string];
+	se: set[string];
+	vc: vector of int;
+	ve: vector of int;
+};
+
+global servers: table[int] of Val = table();
+
+event bro_init()
+	{
+	outfile = open("../out");
+	# first read in the old stuff into the table...
+	Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]);
+	Input::remove("ssh");
+	}
+
+event Input::update_finished(name: string, source:string)
+	{
+	print outfile, servers;
+	close(outfile);
+	terminate();
+	}

testing/btest/scripts/base/frameworks/logging/ascii-empty.bro

@@ -1,12 +1,13 @@
 #
 # @TEST-EXEC: bro -b %INPUT
-# @TEST-EXEC: btest-diff ssh.log
+# @TEST-EXEC: cat ssh.log | grep -v PREFIX.*20..- >ssh-filtered.log
+# @TEST-EXEC: btest-diff ssh-filtered.log
 
 redef LogAscii::output_to_stdout = F;
 redef LogAscii::separator = "|";
 redef LogAscii::empty_field = "EMPTY";
 redef LogAscii::unset_field = "NOT-SET";
-redef LogAscii::header_prefix = "PREFIX<>";
+redef LogAscii::meta_prefix = "PREFIX<>";
 
 module SSH;
 

testing/btest/scripts/base/frameworks/logging/ascii-escape.bro

@@ -1,5 +1,6 @@
 #
 # @TEST-EXEC: bro -b %INPUT 
+# @TEST-EXEC: cat ssh.log | egrep -v '#open|#close' >ssh.log.tmp && mv ssh.log.tmp ssh.log
 # @TEST-EXEC: btest-diff ssh.log
 
 redef LogAscii::separator = "||";

testing/btest/scripts/base/frameworks/logging/ascii-line-like-comment.bro

@@ -0,0 +1,23 @@
+#
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff test.log
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		data: string &log;
+		c: count &log &default=42;
+	};
+}
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Info]);
+	Log::write(Test::LOG, [$data="Test1"]);
+	Log::write(Test::LOG, [$data="#Kaputt"]);
+	Log::write(Test::LOG, [$data="Test2"]);
+}
+

testing/btest/scripts/base/frameworks/logging/ascii-options.bro

@@ -4,7 +4,7 @@
 
 redef LogAscii::output_to_stdout = F;
 redef LogAscii::separator = "|";
-redef LogAscii::include_header = F;
+redef LogAscii::include_meta = F;
 
 module SSH;
 

testing/btest/scripts/base/frameworks/logging/dataseries/options.bro

@@ -0,0 +1,44 @@
+#
+# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
+# @TEST-GROUP: dataseries
+#
+# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: test -e ssh.ds.xml
+# @TEST-EXEC: btest-diff ssh.ds.xml
+
+module SSH;
+
+redef LogDataSeries::dump_schema = T;
+
+# Haven't yet found a way to check for the effect of these.
+redef LogDataSeries::compression = "bz2";
+redef LogDataSeries::extent_size = 1000;
+redef LogDataSeries::num_threads = 5;
+
+# LogDataSeries::use_integer_for_time is tested separately.
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+

testing/btest/scripts/base/frameworks/logging/dataseries/rotate.bro

@@ -0,0 +1,34 @@
+#
+# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
+# @TEST-GROUP: dataseries
+#
+# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 Log::default_writer=Log::WRITER_DATASERIES | grep "test" >out
+# @TEST-EXEC: for i in test.*.ds; do printf '> %s\n' $i; ds2txt --skip-index $i; done >>out
+# @TEST-EXEC: btest-diff out
+
+module Test;
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { LOG };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+	} &log;
+}
+
+redef Log::default_rotation_interval = 1hr;
+redef Log::default_rotation_postprocessor_cmd = "echo";
+
+event bro_init()
+{
+	Log::create_stream(Test::LOG, [$columns=Log]);
+}
+
+event new_connection(c: connection)
+	{
+	Log::write(Test::LOG, [$t=network_time(), $id=c$id]);
+	}

testing/btest/scripts/base/frameworks/logging/dataseries/test-logging.bro

@@ -0,0 +1,35 @@
+#
+# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
+# @TEST-GROUP: dataseries
+#
+# @TEST-EXEC: bro -b %INPUT Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: ds2txt --skip-index ssh.ds >ssh.ds.txt
+# @TEST-EXEC: btest-diff ssh.ds.txt
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+

testing/btest/scripts/base/frameworks/logging/dataseries/time-as-int.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
+# @TEST-GROUP: dataseries
+#
+# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt
+# @TEST-EXEC: btest-diff conn.ds.txt
+
+redef LogDataSeries::use_integer_for_time = T;

testing/btest/scripts/base/frameworks/logging/dataseries/wikipedia.bro

@@ -0,0 +1,9 @@
+#
+# @TEST-REQUIRES: has-writer DataSeries && which ds2txt
+# @TEST-GROUP: dataseries
+#
+# @TEST-EXEC: bro -r $TRACES/wikipedia.trace Log::default_writer=Log::WRITER_DATASERIES
+# @TEST-EXEC: ds2txt --skip-index conn.ds >conn.ds.txt
+# @TEST-EXEC: ds2txt --skip-index http.ds >http.ds.txt
+# @TEST-EXEC: btest-diff conn.ds.txt
+# @TEST-EXEC: btest-diff http.ds.txt

testing/btest/scripts/base/frameworks/logging/none-debug.bro

@@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: bro -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+redef Log::default_writer = Log::WRITER_NONE;
+redef LogNone::debug = T;
+redef Log::default_rotation_interval= 1hr;
+redef log_rotate_base_time = "00:05";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+        local config: table[string] of string;
+        config["foo"]="bar";
+        config["foo2"]="bar2";
+
+	local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::create_stream(SSH::LOG, [$columns=Log]);
+
+	Log::remove_default_filter(SSH::LOG);
+	Log::add_filter(SSH::LOG, [$name="f1", $exclude=set("t", "id.orig_h"), $config=config]);
+	Log::write(SSH::LOG, [$t=network_time(), $id=cid, $status="success"]);
+}
+

testing/btest/scripts/base/frameworks/logging/remote-types.bro

@@ -1,10 +1,12 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
-# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run sender bro -B threading,logging --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run receiver bro -B threading,logging --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 10
 # @TEST-EXEC: btest-diff receiver/test.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
+# @TEST-EXEC: cat receiver/test.log | egrep -v '#open|#close' >r.log
+# @TEST-EXEC: cat sender/test.log   | egrep -v '#open|#close' >s.log
+# @TEST-EXEC: cmp r.log s.log
 
 # Remote version testing all types.
 

testing/btest/scripts/base/frameworks/logging/remote.bro

@@ -1,16 +1,18 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
-# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender bro -b --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro -b --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 15
 # @TEST-EXEC: btest-diff sender/test.log
 # @TEST-EXEC: btest-diff sender/test.failure.log
 # @TEST-EXEC: btest-diff sender/test.success.log
-# @TEST-EXEC: cmp receiver/test.log sender/test.log
-# @TEST-EXEC: cmp receiver/test.failure.log sender/test.failure.log
-# @TEST-EXEC: cmp receiver/test.success.log sender/test.success.log
+# @TEST-EXEC: ( cd sender && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: ( cd receiver && for i in *.log; do cat $i | $SCRIPTS/diff-remove-timestamps >c.$i; done )
+# @TEST-EXEC: cmp receiver/c.test.log sender/c.test.log
+# @TEST-EXEC: cmp receiver/c.test.failure.log sender/c.test.failure.log
+# @TEST-EXEC: cmp receiver/c.test.success.log sender/c.test.success.log
 
 # This is the common part loaded by both sender and receiver.
 module Test;
@@ -39,10 +41,10 @@ event bro_init()
 
 @TEST-START-FILE sender.bro
 
-module Test;
-
 @load frameworks/communication/listen
 
+module Test;
+
 function fail(rec: Log): bool
 	{
 	return rec$status != "success";
@@ -64,14 +66,27 @@ event remote_connection_handshake_done(p: event_peer)
 	Log::write(Test::LOG, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	disconnect(p);
 	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE
 
 @TEST-START-FILE receiver.bro
 
 #####
 
+@load base/frameworks/communication
+
 redef Communication::nodes += {
     ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T]
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
 @TEST-END-FILE

testing/btest/scripts/base/frameworks/logging/rotate-custom.bro

@@ -1,8 +1,9 @@
 #
-# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT | egrep "test|test2" | sort >out
-# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
+# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT | egrep "test|test2" | sort >out.tmp
+# @TEST-EXEC: cat out.tmp pp.log | sort >out
+# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | $SCRIPTS/diff-remove-timestamps | uniq >>out
 # @TEST-EXEC: btest-diff out
-# @TEST-EXEC: btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
 
 module Test;
 
@@ -19,7 +20,7 @@ export {
 }
 
 redef Log::default_rotation_interval = 1hr;
-redef Log::default_rotation_postprocessor_cmd = "echo 1st";
+redef Log::default_rotation_postprocessor_cmd = "echo 1st >>pp.log";
 
 function custom_rotate(info: Log::RotationInfo) : bool
 {

testing/btest/scripts/base/frameworks/logging/rotate.bro

@@ -1,6 +1,6 @@
 #
-# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT 2>&1 | grep "test" >out
-# @TEST-EXEC: for i in test.*.log; do printf '> %s\n' $i; cat $i; done >>out
+# @TEST-EXEC: bro -b -r ${TRACES}/rotation.trace %INPUT 2>&1 | grep "test" >out
+# @TEST-EXEC: for i in `ls test.*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out
 # @TEST-EXEC: btest-diff out
 
 module Test;

testing/btest/scripts/base/frameworks/logging/writer-path-conflict.bro

@@ -0,0 +1,24 @@
+# @TEST-EXEC: bro -C -r $TRACES/wikipedia.trace %INPUT
+# @TEST-EXEC: btest-diff reporter.log
+# @TEST-EXEC: btest-diff http.log
+# @TEST-EXEC: btest-diff http-2.log
+# @TEST-EXEC: btest-diff http-3.log
+# @TEST-EXEC: btest-diff http-2-2.log
+
+@load base/protocols/http
+
+event bro_init()
+	{
+	# Both the default filter for the http stream and this new one will
+	# attempt to have the same writer write to path "http", which will
+	# be reported as a warning and the path auto-corrected to "http-2"
+	local filter: Log::Filter = [$name="host-only", $include=set("host")];
+	# Same deal here, but should be auto-corrected to "http-3".
+	local filter2: Log::Filter = [$name="uri-only", $include=set("uri")];
+	# Conflict between auto-correct paths needs to be corrected, too, this
+	# time it will be "http-2-2".
+	local filter3: Log::Filter = [$path="http-2", $name="status-only", $include=set("status_code")];
+	Log::add_filter(HTTP::LOG, filter);
+	Log::add_filter(HTTP::LOG, filter2);
+	Log::add_filter(HTTP::LOG, filter3);
+	}

testing/btest/scripts/base/frameworks/metrics/basic-cluster.bro

@@ -1,11 +1,11 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 30
 # @TEST-EXEC: btest-diff manager-1/metrics.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -24,11 +24,51 @@ event bro_init() &priority=5
 	Metrics::add_filter("test.metric", 
 		[$name="foo-bar",
 		 $break_interval=3secs]);
-		
-	if ( Cluster::local_node_type() == Cluster::WORKER )
+	}
+
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready_for_data: event();
+
+redef Cluster::manager2worker_events += /ready_for_data/;
+
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready_for_data()
+	{
+	Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3);
+	Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2);
+	Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1);
+	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global n = 0;
+global peer_count = 0;
+
+event Metrics::log_metrics(rec: Metrics::Info)
+	{
+	n = n + 1;
+	if ( n == 3 )
 		{
-		Metrics::add_data("test.metric", [$host=1.2.3.4], 3);
-		Metrics::add_data("test.metric", [$host=6.5.4.3], 2);
-		Metrics::add_data("test.metric", [$host=7.2.1.5], 1);
+		terminate_communication();
+		terminate();
 		}
 	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	print p;
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		{
+		event ready_for_data();
+		}
+	}
+
+@endif

testing/btest/scripts/base/frameworks/metrics/cluster-intermediate-update.bro

@@ -1,11 +1,11 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT 
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -33,6 +33,21 @@ event bro_init() &priority=5
 		 $log=T]);
 	}
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	terminate();
+	}
+
+@endif
+
 @if ( Cluster::local_node_type() == Cluster::WORKER )
 
 event do_metrics(i: count)

testing/btest/scripts/base/frameworks/notice/cluster.bro

@@ -1,10 +1,10 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -21,13 +21,44 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready: event();
+
+redef Cluster::manager2worker_events += /ready/;
+
 event delayed_notice()
 	{
 	if ( Cluster::node == "worker-1" )
 		NOTICE([$note=Test_Notice, $msg="test notice!"]);
 	}
 
-event bro_init()
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready()
 	{
 	schedule 1secs { delayed_notice() };
 	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	peer_count = peer_count + 1;
+	if ( peer_count == 2 )
+		event ready();
+	}
+
+event Notice::log_notice(rec: Notice::Info)
+	{
+	terminate_communication();
+	}
+
+@endif

testing/btest/scripts/base/frameworks/notice/default-policy-order.test

@@ -1,10 +1,10 @@
 # This test checks that the default notice policy ordering does not
 # change from run to run.
 # @TEST-EXEC: bro -e ''
-# @TEST-EXEC: mv notice_policy.log notice_policy.log.1
+# @TEST-EXEC: cat notice_policy.log | $SCRIPTS/diff-remove-timestamps > notice_policy.log.1
 # @TEST-EXEC: bro -e ''
-# @TEST-EXEC: mv notice_policy.log notice_policy.log.2
+# @TEST-EXEC: cat notice_policy.log | $SCRIPTS/diff-remove-timestamps > notice_policy.log.2
 # @TEST-EXEC: bro -e ''
-# @TEST-EXEC: mv notice_policy.log notice_policy.log.3
+# @TEST-EXEC: cat notice_policy.log | $SCRIPTS/diff-remove-timestamps > notice_policy.log.3
 # @TEST-EXEC: diff notice_policy.log.1 notice_policy.log.2
 # @TEST-EXEC: diff notice_policy.log.1 notice_policy.log.3

testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro

@@ -1,11 +1,11 @@
-# @TEST-GROUP: comm
+# @TEST-SERIALIZE: comm
 #
 # @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: sleep 1
+# @TEST-EXEC: sleep 2
 # @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
 # @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
-# @TEST-EXEC: btest-bg-wait -k 10
+# @TEST-EXEC: btest-bg-wait 20
 # @TEST-EXEC: btest-diff manager-1/notice.log
 
 @TEST-START-FILE cluster-layout.bro
@@ -23,6 +23,15 @@ redef enum Notice::Type += {
 	Test_Notice,
 };
 
+event remote_connection_closed(p: event_peer)
+	{
+	terminate();
+	}
+
+global ready: event();
+
+redef Cluster::manager2worker_events += /ready/;
+
 event delayed_notice()
 	{
 	NOTICE([$note=Test_Notice,
@@ -30,10 +39,33 @@ event delayed_notice()
 	        $identifier="this identifier is static"]);
 	}
 
-event bro_init() &priority=5
-	{
+@if ( Cluster::local_node_type() == Cluster::WORKER )
+
+event ready()
+    {
 	if ( Cluster::node == "worker-1" )
 		schedule 4secs { delayed_notice() };
 	if ( Cluster::node == "worker-2" )
 		schedule 1secs { delayed_notice() };
+    }
+
+event Notice::suppressed(n: Notice::Info)
+	{
+	if ( Cluster::node == "worker-1" )
+		terminate_communication();
 	}
+
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+global peer_count = 0;
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	peer_count = peer_count + 1;
+	if ( peer_count == 3 )
+		event ready();
+	}
+
+@endif

testing/btest/scripts/base/frameworks/reporter/disable-stderr.bro

@@ -0,0 +1,13 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
+
+redef Reporter::warnings_to_stderr = F;
+redef Reporter::errors_to_stderr = F;
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}

testing/btest/scripts/base/frameworks/reporter/stderr.bro

@@ -0,0 +1,10 @@
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff reporter.log
+
+global test: table[count] of string = {};
+
+event bro_init()
+	{
+	print test[3];
+	}