Improve termination reliability/speed for brokerstore btests

testing/btest/Baseline/broker.store.brokerstore-attr-clone/clonetwo.out

@@ -1,10 +1,4 @@
 Peer added
 [[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
 [hi]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-attr-clone/master.out

@@ -2,19 +2,4 @@ Peer added
 Peer added 
 [[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
 [hi]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]
-[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
-[hi]
-[[key=a, val=[a=1, b=c, c={
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-attr-persistence-clone/output1

@@ -1,9 +1,3 @@
 [[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
 [Believe me - I am a set, I am a set!, I am really a set!]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-attr-persistence-clone/output2

@@ -1,9 +1,3 @@
 [[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
 [Believe me - I am a set, I am a set!, I am really a set!]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-attr-persistence-clone/output3

@@ -1,9 +1,3 @@
 [[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
 [Believe me - I am a set, I am a set!, I am really a set!]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-attr-persistence/output1

@@ -1,9 +1,3 @@
 [[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
 [Believe me - I am a set, I am a set!, I am really a set!]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-attr-persistence/output2

@@ -1,9 +1,3 @@
 [[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
 [Believe me - I am a set, I am a set!, I am really a set!]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-backend-simple-incompatible/worker-1-stderr

@@ -1,3 +0,0 @@
-error: ProcessStoreEvent Insert: could not convert value "b" for key "a" in store "___sync_store_TestModule::s" while receiving remote data. This probably means the tables have different types on different nodes.
-error: ProcessStoreEvent Insert: could not convert key "a" for store "___sync_store_TestModule::t" while receiving remote data. This probably means the tables have different types on different nodes.
-received termination signal

testing/btest/Baseline/broker.store.brokerstore-backend-simple-incompatible/worker-1.err.log

@@ -0,0 +1,2 @@
+ProcessStoreEvent Insert: could not convert key "a" for store "___sync_store_TestModule::t" while receiving remote data. This probably means the tables have different types on different nodes.
+ProcessStoreEvent Insert: could not convert value "b" for key "a" in store "___sync_store_TestModule::s" while receiving remote data. This probably means the tables have different types on different nodes.

testing/btest/Baseline/broker.store.brokerstore-backend-simple-incompatible/worker-2.err.log

@@ -0,0 +1,2 @@
+ProcessStoreEvent Insert: could not convert key "a" for store "___sync_store_TestModule::t" while receiving remote data. This probably means the tables have different types on different nodes.
+ProcessStoreEvent Insert: could not convert value "b" for key "a" in store "___sync_store_TestModule::s" while receiving remote data. This probably means the tables have different types on different nodes.

testing/btest/Baseline/broker.store.brokerstore-backend-simple-reverse/clone.out

@@ -1,9 +1,3 @@
 [[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
 [hi]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-backend-simple-reverse/master.out

@@ -1,9 +1,3 @@
 [[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
 [hi]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-backend-simple/clone.out

@@ -1,9 +1,3 @@
 [[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
 [hi]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/Baseline/broker.store.brokerstore-backend-simple/master.out

@@ -1,9 +1,3 @@
 [[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
 [hi]
-[[key=a, val=[a=1, b=c, c={
+[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]
-elem1,
-elem2
-}]], [key=b, val=[a=2, b=d, c={
-elem1,
-elem2
-}]]]

testing/btest/broker/store/brokerstore-attr-clone.zeek

@@ -2,19 +2,17 @@
 
 # @TEST-PORT: BROKER_PORT
 
-# @TEST-EXEC: btest-bg-run master "zeek -B broker -b %DIR/sort-stuff.zeek ../master.zeek >../master.out"
+# @TEST-EXEC: btest-bg-run master "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
-# @TEST-EXEC: btest-bg-run cloneone "zeek -B broker -b %DIR/sort-stuff.zeek ../cloneone.zeek >../cloneone.out"
+# @TEST-EXEC: btest-bg-run cloneone "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../cloneone.zeek >../cloneone.out"
-# @TEST-EXEC: btest-bg-run clonetwo "zeek -B broker -b %DIR/sort-stuff.zeek ../clonetwo.zeek >../clonetwo.out"
+# @TEST-EXEC: btest-bg-run clonetwo "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../clonetwo.zeek >../clonetwo.out"
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 20
 #
 # @TEST-EXEC: btest-diff master.out
 # @TEST-EXEC: btest-diff clonetwo.out
 
-@TEST-START-FILE master.zeek
+@TEST-START-FILE common.zeek
 redef exit_only_after_terminate = T;
 
-module TestModule;
-
 global tablestore: opaque of Broker::Store;
 global setstore: opaque of Broker::Store;
 global recordstore: opaque of Broker::Store;
@@ -22,21 +20,13 @@ global recordstore: opaque of Broker::Store;
 type testrec: record {
 	a: count;
 	b: string;
-	c: set[string];
+	c: vector of string;
 };
 
 global t: table[string] of count &broker_store="table";
 global s: set[string] &broker_store="set";
 global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
 
-event zeek_init()
-	{
-	Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
-	tablestore = Broker::create_master("table");
-	setstore = Broker::create_master("set");
-	recordstore = Broker::create_master("rec");
-	}
-
 event dump_tables()
 	{
 	print sort_table(t);
@@ -44,39 +34,62 @@ event dump_tables()
 	print sort_table(r);
 	}
 
+event do_terminate()
+	{ terminate(); }
+@TEST-END-FILE
+
+@TEST-START-FILE master.zeek
+
+event zeek_init()
+	{
+	Broker::subscribe("master");
+	tablestore = Broker::create_master("table");
+	setstore = Broker::create_master("set");
+	recordstore = Broker::create_master("rec");
+	Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
+	}
+
+global peers = 0;
+
+event check_all_set()
+	{
+	if ( "whatever" in t && "hi" in s && "b" in r )
+		{
+		event dump_tables();
+		Broker::publish("cloneone", do_terminate);
+		Broker::publish("clonetwo", check_all_set);
+		}
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
+global send_stuff_over: event();
+
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
+	++peers;
 	print "Peer added ";
-  schedule 5secs { dump_tables() };
+
+	if ( peers == 2 )
+		{
+		Broker::publish("cloneone", send_stuff_over);
+		schedule 0.1sec { check_all_set() };
+		}
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
+	--peers;
+
+	if ( peers == 0 )
 		terminate();
 	}
 @TEST-END-FILE
 
 @TEST-START-FILE cloneone.zeek
-redef exit_only_after_terminate = T;
-
-module TestModule;
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &broker_store="table";
-global s: set[string] &broker_store="set";
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
-
 event zeek_init()
 	{
+	Broker::subscribe("cloneone");
 	Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
 	}
 
@@ -90,63 +103,42 @@ event send_stuff_over()
 	t["a"] = 3;
 	t["b"] = 3;
 	t["c"] = 4;
-	t["whatever"] = 5;
 	delete t["c"];
-	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
+	t["whatever"] = 5;
-	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
-	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
+	r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
 	print sort_table(t);
 	print sort_set(s);
 	print sort_table(r);
 	}
 
-event killmeplease()
-	{
-	terminate();
-	}
-
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Peer added", endpoint;
 	tablestore = Broker::create_clone("table");
 	setstore = Broker::create_clone("set");
 	recordstore = Broker::create_clone("rec");
-	schedule 2secs { send_stuff_over() };
-	schedule 5secs { killmeplease() };
 	}
 @TEST-END-FILE
 
 @TEST-START-FILE clonetwo.zeek
-redef exit_only_after_terminate = T;
-
-module TestModule;
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &broker_store="table";
-global s: set[string] &broker_store="set";
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
-
 event zeek_init()
 	{
+	Broker::subscribe("clonetwo");
 	Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
 	}
 
-event dump_tables()
+event check_all_set()
 	{
-	print sort_table(t);
+	if ( "whatever" in t && "hi" in s && "b" in r )
-	print sort_set(s);
+		{
-	print sort_table(r);
+		event dump_tables();
 		terminate();
 		}
+	else
+		schedule 0.1sec { check_all_set() };
+	}
 
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
@@ -154,6 +146,5 @@ event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	tablestore = Broker::create_clone("table");
 	setstore = Broker::create_clone("set");
 	recordstore = Broker::create_clone("rec");
-	schedule 5secs { dump_tables() };
 	}
 @TEST-END-FILE

testing/btest/broker/store/brokerstore-attr-expire.zeek

@@ -8,25 +8,16 @@
 
 # @TEST-PORT: BROKER_PORT
 
-# @TEST-EXEC: btest-bg-run master "zeek -B broker -b ../master.zeek >../master.out"
+# @TEST-EXEC: btest-bg-run master "zeek -B broker -b ../common.zeek ../master.zeek >../master.out"
-# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b ../clone.zeek >../clone.out"
+# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b ../common.zeek ../clone.zeek >../clone.out"
 # @TEST-EXEC: btest-bg-wait 20
 #
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone.out
 
-@TEST-START-FILE master.zeek
+@TEST-START-FILE common.zeek
 redef exit_only_after_terminate = T;
 redef table_expire_interval = 0.5sec;
 
-module TestModule;
-
-global start_time: time;
-
-function time_past(): interval
-	{
-	return network_time() - start_time;
-	}
-
 global tablestore: opaque of Broker::Store;
 global setstore: opaque of Broker::Store;
 global recordstore: opaque of Broker::Store;
@@ -34,7 +25,7 @@ global recordstore: opaque of Broker::Store;
 type testrec: record {
 	a: count;
 	b: string;
-	c: set[string];
+	c: vector of string;
 };
 
 function change_t(tbl: any, tpe: TableChange, idx: string, idxb: count)
@@ -55,23 +46,12 @@ function change_r(tbl: any, tpe: TableChange, idx: string, idxb: testrec)
 		print fmt("Expiring r: %s", idx);
 	}
 
-function print_keys()
-	{
-	print "Printing keys";
-	when ( local s = Broker::keys(tablestore) )
-		{
-		print "keys", s;
-		}
-	timeout 2sec
-		{
-		print fmt("<timeout for print keys>");
-		}
-	}
-
 global t: table[string] of count &broker_store="table" &create_expire=4sec &on_change=change_t;
 global s: table[string] of count &broker_store="set" &write_expire=3sec &on_change=change_s;
 global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec" &write_expire=5sec &on_change=change_r;
+@TEST-END-FILE
 
+@TEST-START-FILE master.zeek
 event zeek_init()
 	{
 	Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
@@ -85,13 +65,12 @@ event update_stuff()
 	t["a"] = 3;
 	t["expire_later_in_t_not_with_a"] = 4;
 	s["expire_later"] = 2;
-	r["reca"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
+	r["reca"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
 	}
 
 event insert_stuff()
 	{
 	print "Inserting stuff";
-	start_time = network_time();
 	t["a"] = 5;
 	delete t["a"];
 	s["expire_first"] = 0;
@@ -99,8 +78,8 @@ event insert_stuff()
 	t["a"] = 2;
 	t["b"] = 3;
 	t["whatever"] = 5;
-	r["reca"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
+	r["reca"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
-	r["recb"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
+	r["recb"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
 	print t;
 	print s;
 	print r;
@@ -110,59 +89,17 @@ event insert_stuff()
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Peer added ", endpoint;
-  schedule 3secs { insert_stuff() };
+	event insert_stuff();
-	}
-
-event terminate_me()
-	{
-	print "Terminating";
-	terminate();
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
-	print_keys();
+	print "Terminating";
-	schedule 3secs { terminate_me() };
+	terminate();
 	}
 @TEST-END-FILE
 
 @TEST-START-FILE clone.zeek
-redef exit_only_after_terminate = T;
-redef table_expire_interval = 0.5sec;
-
-module TestModule;
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-function change_t(tbl: any, tpe: TableChange, idx: string, idxb: count)
-	{
-	if ( tpe == TABLE_ELEMENT_EXPIRED )
-		print fmt("Expiring t: %s", idx);
-	}
-
-function change_s(tbl: any, tpe: TableChange, idx: string, idbx: count)
-	{
-	if ( tpe == TABLE_ELEMENT_EXPIRED )
-		print fmt("Expiring s: %s", idx);
-	}
-
-function change_r(tbl: any, tpe: TableChange, idx: string, idxb: testrec)
-	{
-		if ( tpe == TABLE_ELEMENT_EXPIRED )
-			print fmt("Expiring r: %s", idx);
-	}
-
-global t: table[string] of count &broker_store="table" &create_expire=4sec &on_change=change_t;
-global s: table[string] of count &broker_store="set" &write_expire=3sec &on_change=change_s;
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec" &write_expire=5sec &on_change=change_r;
 
 event zeek_init()
 	{
@@ -178,12 +115,28 @@ event dump_tables()
 	terminate();
 	}
 
+event check_all_unset()
+	{
+	if ( |t| == 0 && |s| == 0 && |r| == 0 )
+		event dump_tables();
+	else
+		schedule 0.1sec { check_all_unset() };
+	}
+
+event check_all_set()
+	{
+	if ( "whatever" in t && "expire_later" in s && "recb" in r )
+		schedule 0.1sec { check_all_unset() };
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Peer added";
 	tablestore = Broker::create_clone("table");
 	setstore = Broker::create_clone("set");
 	recordstore = Broker::create_clone("rec");
-	schedule 15secs { dump_tables() };
+	schedule 0.1sec { check_all_set() };
 	}
 @TEST-END-FILE

testing/btest/broker/store/brokerstore-attr-persistence-clone.zeek

@@ -1,9 +1,9 @@
 # @TEST-PORT: BROKER_PORT
 
-# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek one.zeek > output1
+# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek common.zeek one.zeek > output1
-# @TEST-EXEC: btest-bg-run master "cp ../*.sqlite . && zeek -B broker -b %DIR/sort-stuff.zeek ../two.zeek >../output2"
+# @TEST-EXEC: btest-bg-run master "cp ../*.sqlite . && zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../two.zeek >../output2"
-# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b %DIR/sort-stuff.zeek ../three.zeek >../output3"
+# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../three.zeek >../output3"
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 20
 
 # @TEST-EXEC: btest-diff output1
 # @TEST-EXEC: btest-diff output2
@@ -11,12 +11,7 @@
 # @TEST-EXEC: diff output1 output2
 # @TEST-EXEC: diff output2 output3
 
-# the first test writes out the sqlite files...
+@TEST-START-FILE common.zeek
-
-@TEST-START-FILE one.zeek
-
-module TestModule;
-
 global tablestore: opaque of Broker::Store;
 global setstore: opaque of Broker::Store;
 global recordstore: opaque of Broker::Store;
@@ -24,12 +19,18 @@ global recordstore: opaque of Broker::Store;
 type testrec: record {
 	a: count;
 	b: string;
-	c: set[string];
+	c: vector of string;
 };
 
 global t: table[string] of count &broker_store="table";
 global s: set[string] &broker_store="set";
 global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
+@TEST-END-FILE
+
+# the first test writes out the sqlite files...
+
+@TEST-START-FILE one.zeek
+redef exit_only_after_terminate = T;
 
 event zeek_init()
 	{
@@ -39,41 +40,25 @@ event zeek_init()
 	t["a"] = 5;
 	t["b"] = 3;
 	t["c"] = 4;
-	t["whatever"] = 5;
 	delete t["c"];
+	t["whatever"] = 5;
 	add s["I am a set!"];
 	add s["I am really a set!"];
 	add s["Believe me - I am a set"];
-	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
-	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
-	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
+	r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
 	print sort_table(t);
 	print sort_set(s);
 	print sort_table(r);
+	terminate();
 	}
 
 @TEST-END-FILE
 @TEST-START-FILE two.zeek
-
-# read in again - and serve to clones
-
 redef exit_only_after_terminate = T;
 
-module TestModule;
+# read in again - and serve to clones
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &broker_store="table";
-global s: set[string] &broker_store="set";
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
 
 event zeek_init()
 	{
@@ -94,27 +79,9 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 @TEST-END-FILE
 
 @TEST-START-FILE three.zeek
-
-# get copy from master
-
 redef exit_only_after_terminate = T;
 
-module TestModule;
+# get copy from master
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-
-global t: table[string] of count &broker_store="table";
-global s: set[string] &broker_store="set";
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
 
 event zeek_init()
 	{
@@ -129,12 +96,20 @@ event print_me()
 	terminate();
 	}
 
+event check_all_set()
+	{
+	if ( "whatever" in t && |s| == 3 && "b" in r )
+		event print_me();
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	tablestore = Broker::create_clone("table");
 	setstore = Broker::create_clone("set");
 	recordstore = Broker::create_clone("rec");
-	schedule 2sec { print_me() };
+	schedule 0.1sec { check_all_set() };
 	}
 
 

testing/btest/broker/store/brokerstore-attr-persistence.zeek

@@ -1,15 +1,12 @@
-# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek one.zeek > output1
+# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek common.zeek one.zeek > output1
-# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek two.zeek > output2
+# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek common.zeek two.zeek > output2
 # @TEST-EXEC: btest-diff output1
 # @TEST-EXEC: btest-diff output2
 # @TEST-EXEC: diff output1 output2
 
 # the first test writes out the sqlite files...
 
-@TEST-START-FILE one.zeek
+@TEST-START-FILE common.zeek
-
-module TestModule;
-
 global tablestore: opaque of Broker::Store;
 global setstore: opaque of Broker::Store;
 global recordstore: opaque of Broker::Store;
@@ -17,12 +14,15 @@ global recordstore: opaque of Broker::Store;
 type testrec: record {
 	a: count;
 	b: string;
-	c: set[string];
+	c: vector of string;
 };
 
 global t: table[string] of count &broker_store="table";
 global s: set[string] &broker_store="set";
 global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
+@TEST-END-FILE
+
+@TEST-START-FILE one.zeek
 
 event zeek_init()
 	{
@@ -37,9 +37,9 @@ event zeek_init()
 	add s["I am a set!"];
 	add s["I am really a set!"];
 	add s["Believe me - I am a set"];
-	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
-	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
-	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
+	r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
 	print sort_table(t);
 	print sort_set(s);
 	print sort_table(r);
@@ -50,22 +50,6 @@ event zeek_init()
 
 # the second one reads them in again
 
-module TestModule;
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &broker_store="table";
-global s: set[string] &broker_store="set";
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
-
 event zeek_init()
 	{
 	tablestore = Broker::create_master("table", Broker::SQLITE);

testing/btest/broker/store/brokerstore-attr-simple.zeek

@@ -1,16 +1,14 @@
 # @TEST-PORT: BROKER_PORT
 
-# @TEST-EXEC: btest-bg-run master "zeek -b -B broker -b %DIR/sort-stuff.zeek ../master.zeek >../master.out"
+# @TEST-EXEC: btest-bg-run master "zeek -b -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
-# @TEST-EXEC: btest-bg-run clone "zeek -b -B broker -b %DIR/sort-stuff.zeek ../clone.zeek >../clone.out"
+# @TEST-EXEC: btest-bg-run clone "zeek -b -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone.out"
-# @TEST-EXEC: btest-bg-wait 15
+# @TEST-EXEC: btest-bg-wait 20
 #
 # @TEST-EXEC: btest-diff clone.out
 
-@TEST-START-FILE master.zeek
+@TEST-START-FILE common.zeek
 redef exit_only_after_terminate = T;
 
-module TestModule;
-
 global tablestore: opaque of Broker::Store;
 global setstore: opaque of Broker::Store;
 global recordstore: opaque of Broker::Store;
@@ -24,13 +22,15 @@ type testrec: record {
 global t: table[string] of count &broker_store="table";
 global s: set[string] &broker_store="set";
 global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
+@TEST-END-FILE
 
+@TEST-START-FILE master.zeek
 event zeek_init()
 	{
-	Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
 	tablestore = Broker::create_master("table");
 	setstore = Broker::create_master("set");
 	recordstore = Broker::create_master("rec");
+	Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
 	}
 
 event insert_stuff()
@@ -43,8 +43,8 @@ event insert_stuff()
 	t["a"] = 3;
 	t["b"] = 3;
 	t["c"] = 4;
-	t["whatever"] = 5;
 	delete t["c"];
+	t["whatever"] = 5;
 	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
 	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
 	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
@@ -56,7 +56,7 @@ event insert_stuff()
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Peer added ", endpoint;
-  schedule 3secs { insert_stuff() };
+	event insert_stuff();
 	}
 
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
@@ -67,24 +67,6 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 @TEST-END-FILE
 
 @TEST-START-FILE clone.zeek
-redef exit_only_after_terminate = T;
-
-module TestModule;
-
-global tablestore: opaque of Broker::Store;
-global setstore: opaque of Broker::Store;
-global recordstore: opaque of Broker::Store;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &broker_store="table";
-global s: set[string] &broker_store="set";
-global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
-
 event zeek_init()
 	{
 	Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
@@ -98,12 +80,20 @@ event dump_tables()
 	terminate();
 	}
 
+event check_all_set()
+	{
+	if ( "whatever" in t && "hi" in s && "b" in r )
+		event dump_tables();
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	print "Peer added";
 	tablestore = Broker::create_clone("table");
 	setstore = Broker::create_clone("set");
 	recordstore = Broker::create_clone("rec");
-	schedule 5secs { dump_tables() };
+	schedule 0.1sec { check_all_set() };
 	}
 @TEST-END-FILE

testing/btest/broker/store/brokerstore-backend-simple-incompatible.zeek

@@ -5,10 +5,10 @@
 # @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker ../master.zeek >../master.out"
 # @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker ../clone.zeek >../clone.out"
 # @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker ../clone.zeek >../clone2.out"
-# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-bg-wait 30
 #
-# @TEST-EXEC: grep -v PEER_UNAVAILABLE worker-1/.stderr > worker-1-stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-sort" btest-diff worker-1/err.log
-# @TEST-EXEC: btest-diff worker-1-stderr
+# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-sort" btest-diff worker-2/err.log
 
 @TEST-START-FILE cluster-layout.zeek
 redef Cluster::nodes = {
@@ -22,23 +22,33 @@ redef Cluster::nodes = {
 @TEST-START-FILE master.zeek
 @load base/frameworks/cluster
 redef exit_only_after_terminate = T;
-redef Log::enable_local_logging = T;
-redef Log::default_rotation_interval = 0secs;
 
 module TestModule;
 
 global t: table[string] of count &backend=Broker::MEMORY;
 global s: table[string] of string &backend=Broker::MEMORY;
 
-event zeek_init()
+event add_stuff()
 	{
 	t["a"] = 5;
 	s["a"] = "b";
 	print t;
 	}
 
+global peers = 0;
+event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	++peers;
+
+	if ( peers == 2 )
+		schedule 2sec { add_stuff() };
+	}
+
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
+	--peers;
+
+	if ( peers == 0 )
 		terminate();
 	}
 
@@ -46,25 +56,26 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 
 @TEST-START-FILE clone.zeek
 @load base/frameworks/cluster
+@load base/frameworks/reporter
 redef exit_only_after_terminate = T;
-redef Log::enable_local_logging = T;
-redef Log::default_rotation_interval = 0secs;
 
 module TestModule;
 
 global t: table[count] of count &backend=Broker::MEMORY;
 global s: table[string] of count &backend=Broker::MEMORY;
 
-event dump_tables()
+global errlog = open("err.log");
+
+global errors = 0;
+event reporter_error(t: time, msg: string, location: string)
 	{
-	print t;
+	if ( /ProcessStoreEvent/ in msg )
-	print s;
+		{
-	terminate();
+		print errlog, msg;
+		++errors;
 		}
 
-event Cluster::node_up(name: string, id: string)
+	if ( errors == 2 )
-	{
+		terminate();
-	#print "node up", name;
-	schedule 4secs { dump_tables() };
 	}
 @TEST-END-FILE

testing/btest/broker/store/brokerstore-backend-simple-reverse.zeek

@@ -2,9 +2,9 @@
 # @TEST-PORT: BROKER_PORT2
 # @TEST-PORT: BROKER_PORT3
 
-# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../master.zeek >../master.out"
+# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
-# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../clone.zeek >../clone.out"
+# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone.out"
-# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../clone2.zeek >../clone2.out"
+# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone2.zeek >../clone2.out"
 # @TEST-EXEC: btest-bg-wait 40
 #
 # @TEST-EXEC: btest-diff master.out
@@ -20,32 +20,36 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
-
+@TEST-START-FILE common.zeek
-@TEST-START-FILE master.zeek
 @load base/frameworks/cluster
 
 redef exit_only_after_terminate = T;
 redef Log::enable_local_logging = T;
 redef Log::default_rotation_interval = 0secs;
 
-module TestModule;
-
 type testrec: record {
 	a: count;
 	b: string;
-	c: set[string];
+	c: vector of string;
 };
 
 global t: table[string] of count &backend=Broker::MEMORY;
 global s: set[string] &backend=Broker::MEMORY;
 global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
 
-global terminate_count = 0;
+event go_away()
-
-event zeek_init()
 	{
+	terminate();
 	}
 
+function all_stores_set(): bool
+	{
+	return "whatever" in t && "hi" in s && "b" in r;
+	}
+@TEST-END-FILE
+
+@TEST-START-FILE master.zeek
+
 event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=1
 	{
 	Reporter::info(fmt("Peer added: %s", cat(endpoint)));
@@ -54,41 +58,15 @@ event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
 	Reporter::info(fmt("Peer lost: %s", cat(endpoint)));
-	terminate_count += 1;
-	if ( terminate_count == 2)
-		{
-		terminate();
 	print sort_table(t);
 	print sort_set(s);
 	print sort_table(r);
-		}
+	terminate();
 	}
 
 @TEST-END-FILE
 
 @TEST-START-FILE clone.zeek
-@load base/frameworks/cluster
-
-redef exit_only_after_terminate = T;
-redef Log::enable_local_logging = T;
-redef Log::default_rotation_interval = 0secs;
-
-module TestModule;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &backend=Broker::MEMORY;
-global s: set[string] &backend=Broker::MEMORY;
-global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
-
-event terminate_me()
-	{
-	terminate();
-	}
 
 event dump_tables()
 	{
@@ -99,49 +77,34 @@ event dump_tables()
 	t["a"] = 3;
 	t["b"] = 3;
 	t["c"] = 4;
-	t["whatever"] = 5;
 	delete t["c"];
-	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
+	t["whatever"] = 5;
-	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
-	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
+	r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
 	print sort_table(t);
 	print sort_set(s);
 	print sort_table(r);
-	schedule 10sec { terminate_me() };
 	}
 
 event Cluster::node_up(name: string, id: string)
 	{
 	Reporter::info(fmt("Node Up: %s", name));
-	schedule 5secs { dump_tables() };
+	event dump_tables();
+	}
+
+event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
+	{
+	terminate();
 	}
 
 event Broker::announce_masters(masters: set[string])
 	{
 	Reporter::info(fmt("Received announce_masters: %s", cat(masters)));
 	}
-
 @TEST-END-FILE
 
 @TEST-START-FILE clone2.zeek
-@load base/frameworks/cluster
-
-redef exit_only_after_terminate = T;
-redef Log::enable_local_logging = T;
-redef Log::default_rotation_interval = 0secs;
-
-module TestModule;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &backend=Broker::MEMORY;
-global s: set[string] &backend=Broker::MEMORY;
-global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
-
 event dump_tables()
 	{
 	print sort_table(t);
@@ -155,10 +118,18 @@ event Broker::announce_masters(masters: set[string])
 	Reporter::info(fmt("Received announce_masters: %s", cat(masters)));
 	}
 
+event check_all_set()
+	{
+	if ( all_stores_set() )
+		event dump_tables();
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
 event Cluster::node_up(name: string, id: string)
 	{
 	Reporter::info(fmt("Node Up: %s", name));
-	schedule 20secs { dump_tables() };
+	schedule 0.1sec { check_all_set() };
 	}
 @TEST-END-FILE
 

testing/btest/broker/store/brokerstore-backend-simple.zeek

@@ -5,7 +5,7 @@
 # @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker ../common.zeek ../master.zeek >../master.out"
 # @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker ../common.zeek ../clone.zeek >../clone.out"
 # @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker ../common.zeek ../clone.zeek >../clone2.out"
-# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-bg-wait 30
 #
 # @TEST-EXEC: btest-diff master.out
 # @TEST-EXEC: btest-diff clone.out
@@ -24,6 +24,20 @@ redef Cluster::nodes = {
 @load base/frameworks/cluster
 @load base/frameworks/broker
 
+redef exit_only_after_terminate = T;
+redef Log::enable_local_logging = T;
+redef Log::default_rotation_interval = 0secs;
+
+type testrec: record {
+	a: count;
+	b: string;
+	c: vector of string;
+};
+
+global t: table[string] of count &backend=Broker::MEMORY;
+global s: set[string] &backend=Broker::MEMORY;
+global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
+
 function sort_set(s: set[string]): vector of string
 	{
 	local v: vector of string = vector();
@@ -58,22 +72,6 @@ function sort_table(t: table[string] of any): vector of TableEntry
 @TEST-END-FILE
 
 @TEST-START-FILE master.zeek
-redef exit_only_after_terminate = T;
-redef Log::enable_local_logging = T;
-redef Log::default_rotation_interval = 0secs;
-
-module TestModule;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &backend=Broker::MEMORY;
-global s: set[string] &backend=Broker::MEMORY;
-global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
-
 event zeek_init()
 	{
 	t["a"] = 5;
@@ -83,40 +81,28 @@ event zeek_init()
 	t["a"] = 3;
 	t["b"] = 3;
 	t["c"] = 4;
-	t["whatever"] = 5;
 	delete t["c"];
-	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
+	t["whatever"] = 5;
-	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
-	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
+	r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
+	r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
 	print sort_table(t);
 	print sort_set(s);
 	print sort_table(r);
 	}
 
+global peers_lost = 0;
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
+	++peers_lost;
+
+	if ( peers_lost == 2 )
 		terminate();
 	}
 
 @TEST-END-FILE
 
 @TEST-START-FILE clone.zeek
-redef exit_only_after_terminate = T;
-redef Log::enable_local_logging = T;
-redef Log::default_rotation_interval = 0secs;
-
-module TestModule;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
-global t: table[string] of count &backend=Broker::MEMORY;
-global s: set[string] &backend=Broker::MEMORY;
-global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
-
 
 event dump_tables()
 	{
@@ -126,9 +112,17 @@ event dump_tables()
 	terminate();
 	}
 
+event check_all_set()
+	{
+	if ( "whatever" in t && "hi" in s && "b" in r )
+		event dump_tables();
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
+
 event Cluster::node_up(name: string, id: string)
 	{
-	#print "node up", name;
+	schedule 0.1sec { check_all_set() };
-	schedule 4secs { dump_tables() };
 	}
 @TEST-END-FILE

testing/btest/broker/store/brokerstore-backend-sqlite.zeek

@@ -2,10 +2,10 @@
 # @TEST-PORT: BROKER_PORT2
 # @TEST-PORT: BROKER_PORT3
 
-# @TEST-EXEC: zeek -b %DIR/sort-stuff.zeek preseed-sqlite.zeek;
+# @TEST-EXEC: zeek -b %DIR/sort-stuff.zeek common.zeek preseed-sqlite.zeek
-# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../master.zeek >../master.out"
+# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
-# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../clone.zeek >../clone.out"
+# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone.out"
-# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../clone.zeek >../clone2.out"
+# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone2.out"
 # @TEST-EXEC: btest-bg-wait 40
 #
 # @TEST-EXEC: btest-diff master.out
@@ -21,16 +21,15 @@ redef Cluster::nodes = {
 };
 @TEST-END-FILE
 
-@TEST-START-FILE preseed-sqlite.zeek
+@TEST-START-FILE common.zeek
-
-module TestModule;
-
 type testrec: record {
 	a: count;
 	b: string;
 	c: set[string];
 };
+@TEST-END-FILE
 
+@TEST-START-FILE preseed-sqlite.zeek
 global t: table[string] of count &backend=Broker::SQLITE;
 global s: set[string] &backend=Broker::SQLITE;
 global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::SQLITE;
@@ -44,8 +43,8 @@ event zeek_init()
 	t["a"] = 3;
 	t["b"] = 3;
 	t["c"] = 4;
-	t["whatever"] = 5;
 	delete t["c"];
+	t["whatever"] = 5;
 	r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
 	r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
 	r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
@@ -63,14 +62,6 @@ redef exit_only_after_terminate = T;
 redef Log::enable_local_logging = T;
 redef Log::default_rotation_interval = 0secs;
 
-module TestModule;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
 function change_function(t: table[string] of count, tpe: TableChange, idxa: string, val: count)
 	{
 	print "This should not print";
@@ -90,8 +81,12 @@ event zeek_init()
 	print sort_table(r);
 	}
 
+global peers_lost = 0;
 event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
 	{
+	++peers_lost;
+
+	if ( peers_lost == 2 )
 		terminate();
 	}
 
@@ -104,19 +99,10 @@ redef exit_only_after_terminate = T;
 redef Log::enable_local_logging = T;
 redef Log::default_rotation_interval = 0secs;
 
-module TestModule;
-
-type testrec: record {
-	a: count;
-	b: string;
-	c: set[string];
-};
-
 global t: table[string] of count &backend=Broker::MEMORY;
 global s: set[string] &backend=Broker::MEMORY;
 global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
 
-
 event dump_tables()
 	{
 	print sort_table(t);
@@ -125,9 +111,16 @@ event dump_tables()
 	terminate();
 	}
 
+event check_all_set()
+	{
+	if ( "whatever" in t && "hi" in s && "b" in r )
+		event dump_tables();
+	else
+		schedule 0.1sec { check_all_set() };
+	}
+
 event Cluster::node_up(name: string, id: string)
 	{
-	#print "node up", name;
+	schedule 0.1sec { check_all_set() };
-	schedule 15secs { dump_tables() };
 	}
 @TEST-END-FILE