Improve termination reliability/speed for brokerstore btests

This commit is contained in:
Jon Siwek 2020-08-08 10:23:54 -07:00 committed by Tim Wojtulewicz
parent 7967a5b0aa
commit ec8c24b13b
23 changed files with 287 additions and 499 deletions

View file

@ -1,10 +1,4 @@
Peer added
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -2,19 +2,4 @@ Peer added
Peer added
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
[Believe me - I am a set, I am a set!, I am really a set!]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
[Believe me - I am a set, I am a set!, I am really a set!]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
[Believe me - I am a set, I am a set!, I am really a set!]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
[Believe me - I am a set, I am a set!, I am really a set!]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=5], [key=b, val=3], [key=whatever, val=5]]
[Believe me - I am a set, I am a set!, I am really a set!]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,3 +0,0 @@
error: ProcessStoreEvent Insert: could not convert value "b" for key "a" in store "___sync_store_TestModule::s" while receiving remote data. This probably means the tables have different types on different nodes.
error: ProcessStoreEvent Insert: could not convert key "a" for store "___sync_store_TestModule::t" while receiving remote data. This probably means the tables have different types on different nodes.
received termination signal

View file

@ -0,0 +1,2 @@
ProcessStoreEvent Insert: could not convert key "a" for store "___sync_store_TestModule::t" while receiving remote data. This probably means the tables have different types on different nodes.
ProcessStoreEvent Insert: could not convert value "b" for key "a" in store "___sync_store_TestModule::s" while receiving remote data. This probably means the tables have different types on different nodes.

View file

@ -0,0 +1,2 @@
ProcessStoreEvent Insert: could not convert key "a" for store "___sync_store_TestModule::t" while receiving remote data. This probably means the tables have different types on different nodes.
ProcessStoreEvent Insert: could not convert value "b" for key "a" in store "___sync_store_TestModule::s" while receiving remote data. This probably means the tables have different types on different nodes.

View file

@ -1,9 +1,3 @@
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -1,9 +1,3 @@
[[key=a, val=3], [key=b, val=3], [key=whatever, val=5]]
[hi]
[[key=a, val=[a=1, b=c, c={
elem1,
elem2
}]], [key=b, val=[a=2, b=d, c={
elem1,
elem2
}]]]
[[key=a, val=[a=1, b=c, c=[elem1, elem2]]], [key=b, val=[a=2, b=d, c=[elem1, elem2]]]]

View file

@ -2,19 +2,17 @@
# @TEST-PORT: BROKER_PORT
# @TEST-EXEC: btest-bg-run master "zeek -B broker -b %DIR/sort-stuff.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run cloneone "zeek -B broker -b %DIR/sort-stuff.zeek ../cloneone.zeek >../cloneone.out"
# @TEST-EXEC: btest-bg-run clonetwo "zeek -B broker -b %DIR/sort-stuff.zeek ../clonetwo.zeek >../clonetwo.out"
# @TEST-EXEC: btest-bg-wait 15
# @TEST-EXEC: btest-bg-run master "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run cloneone "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../cloneone.zeek >../cloneone.out"
# @TEST-EXEC: btest-bg-run clonetwo "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../clonetwo.zeek >../clonetwo.out"
# @TEST-EXEC: btest-bg-wait 20
#
# @TEST-EXEC: btest-diff master.out
# @TEST-EXEC: btest-diff clonetwo.out
@TEST-START-FILE master.zeek
@TEST-START-FILE common.zeek
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
@ -22,21 +20,13 @@ global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
c: vector of string;
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
event zeek_init()
{
Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
tablestore = Broker::create_master("table");
setstore = Broker::create_master("set");
recordstore = Broker::create_master("rec");
}
event dump_tables()
{
print sort_table(t);
@ -44,39 +34,62 @@ event dump_tables()
print sort_table(r);
}
event do_terminate()
{ terminate(); }
@TEST-END-FILE
@TEST-START-FILE master.zeek
event zeek_init()
{
Broker::subscribe("master");
tablestore = Broker::create_master("table");
setstore = Broker::create_master("set");
recordstore = Broker::create_master("rec");
Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
}
global peers = 0;
event check_all_set()
{
if ( "whatever" in t && "hi" in s && "b" in r )
{
event dump_tables();
Broker::publish("cloneone", do_terminate);
Broker::publish("clonetwo", check_all_set);
}
else
schedule 0.1sec { check_all_set() };
}
global send_stuff_over: event();
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
++peers;
print "Peer added ";
schedule 5secs { dump_tables() };
if ( peers == 2 )
{
Broker::publish("cloneone", send_stuff_over);
schedule 0.1sec { check_all_set() };
}
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
terminate();
--peers;
if ( peers == 0 )
terminate();
}
@TEST-END-FILE
@TEST-START-FILE cloneone.zeek
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
event zeek_init()
{
Broker::subscribe("cloneone");
Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
}
@ -90,62 +103,41 @@ event send_stuff_over()
t["a"] = 3;
t["b"] = 3;
t["c"] = 4;
t["whatever"] = 5;
delete t["c"];
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
t["whatever"] = 5;
r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
print sort_table(t);
print sort_set(s);
print sort_table(r);
}
event killmeplease()
{
terminate();
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print "Peer added", endpoint;
tablestore = Broker::create_clone("table");
setstore = Broker::create_clone("set");
recordstore = Broker::create_clone("rec");
schedule 2secs { send_stuff_over() };
schedule 5secs { killmeplease() };
}
@TEST-END-FILE
@TEST-START-FILE clonetwo.zeek
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
event zeek_init()
{
Broker::subscribe("clonetwo");
Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
}
event dump_tables()
event check_all_set()
{
print sort_table(t);
print sort_set(s);
print sort_table(r);
terminate();
if ( "whatever" in t && "hi" in s && "b" in r )
{
event dump_tables();
terminate();
}
else
schedule 0.1sec { check_all_set() };
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
@ -154,6 +146,5 @@ event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
tablestore = Broker::create_clone("table");
setstore = Broker::create_clone("set");
recordstore = Broker::create_clone("rec");
schedule 5secs { dump_tables() };
}
@TEST-END-FILE

View file

@ -8,25 +8,16 @@
# @TEST-PORT: BROKER_PORT
# @TEST-EXEC: btest-bg-run master "zeek -B broker -b ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run master "zeek -B broker -b ../common.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b ../common.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-wait 20
#
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone.out
@TEST-START-FILE master.zeek
@TEST-START-FILE common.zeek
redef exit_only_after_terminate = T;
redef table_expire_interval = 0.5sec;
module TestModule;
global start_time: time;
function time_past(): interval
{
return network_time() - start_time;
}
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
@ -34,7 +25,7 @@ global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
c: vector of string;
};
function change_t(tbl: any, tpe: TableChange, idx: string, idxb: count)
@ -51,27 +42,16 @@ function change_s(tbl: any, tpe: TableChange, idx: string, idbx: count)
function change_r(tbl: any, tpe: TableChange, idx: string, idxb: testrec)
{
if ( tpe == TABLE_ELEMENT_EXPIRED )
print fmt("Expiring r: %s", idx);
}
function print_keys()
{
print "Printing keys";
when ( local s = Broker::keys(tablestore) )
{
print "keys", s;
}
timeout 2sec
{
print fmt("<timeout for print keys>");
}
if ( tpe == TABLE_ELEMENT_EXPIRED )
print fmt("Expiring r: %s", idx);
}
global t: table[string] of count &broker_store="table" &create_expire=4sec &on_change=change_t;
global s: table[string] of count &broker_store="set" &write_expire=3sec &on_change=change_s;
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec" &write_expire=5sec &on_change=change_r;
@TEST-END-FILE
@TEST-START-FILE master.zeek
event zeek_init()
{
Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
@ -85,13 +65,12 @@ event update_stuff()
t["a"] = 3;
t["expire_later_in_t_not_with_a"] = 4;
s["expire_later"] = 2;
r["reca"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["reca"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
}
event insert_stuff()
{
print "Inserting stuff";
start_time = network_time();
t["a"] = 5;
delete t["a"];
s["expire_first"] = 0;
@ -99,8 +78,8 @@ event insert_stuff()
t["a"] = 2;
t["b"] = 3;
t["whatever"] = 5;
r["reca"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["recb"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
r["reca"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
r["recb"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
print t;
print s;
print r;
@ -110,59 +89,17 @@ event insert_stuff()
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print "Peer added ", endpoint;
schedule 3secs { insert_stuff() };
}
event terminate_me()
{
print "Terminating";
terminate();
event insert_stuff();
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
print_keys();
schedule 3secs { terminate_me() };
print "Terminating";
terminate();
}
@TEST-END-FILE
@TEST-START-FILE clone.zeek
redef exit_only_after_terminate = T;
redef table_expire_interval = 0.5sec;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
function change_t(tbl: any, tpe: TableChange, idx: string, idxb: count)
{
if ( tpe == TABLE_ELEMENT_EXPIRED )
print fmt("Expiring t: %s", idx);
}
function change_s(tbl: any, tpe: TableChange, idx: string, idbx: count)
{
if ( tpe == TABLE_ELEMENT_EXPIRED )
print fmt("Expiring s: %s", idx);
}
function change_r(tbl: any, tpe: TableChange, idx: string, idxb: testrec)
{
if ( tpe == TABLE_ELEMENT_EXPIRED )
print fmt("Expiring r: %s", idx);
}
global t: table[string] of count &broker_store="table" &create_expire=4sec &on_change=change_t;
global s: table[string] of count &broker_store="set" &write_expire=3sec &on_change=change_s;
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec" &write_expire=5sec &on_change=change_r;
event zeek_init()
{
@ -178,12 +115,28 @@ event dump_tables()
terminate();
}
event check_all_unset()
{
if ( |t| == 0 && |s| == 0 && |r| == 0 )
event dump_tables();
else
schedule 0.1sec { check_all_unset() };
}
event check_all_set()
{
if ( "whatever" in t && "expire_later" in s && "recb" in r )
schedule 0.1sec { check_all_unset() };
else
schedule 0.1sec { check_all_set() };
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print "Peer added";
tablestore = Broker::create_clone("table");
setstore = Broker::create_clone("set");
recordstore = Broker::create_clone("rec");
schedule 15secs { dump_tables() };
schedule 0.1sec { check_all_set() };
}
@TEST-END-FILE

View file

@ -1,9 +1,9 @@
# @TEST-PORT: BROKER_PORT
# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek one.zeek > output1
# @TEST-EXEC: btest-bg-run master "cp ../*.sqlite . && zeek -B broker -b %DIR/sort-stuff.zeek ../two.zeek >../output2"
# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b %DIR/sort-stuff.zeek ../three.zeek >../output3"
# @TEST-EXEC: btest-bg-wait 15
# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek common.zeek one.zeek > output1
# @TEST-EXEC: btest-bg-run master "cp ../*.sqlite . && zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../two.zeek >../output2"
# @TEST-EXEC: btest-bg-run clone "zeek -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../three.zeek >../output3"
# @TEST-EXEC: btest-bg-wait 20
# @TEST-EXEC: btest-diff output1
# @TEST-EXEC: btest-diff output2
@ -11,12 +11,7 @@
# @TEST-EXEC: diff output1 output2
# @TEST-EXEC: diff output2 output3
# the first test writes out the sqlite files...
@TEST-START-FILE one.zeek
module TestModule;
@TEST-START-FILE common.zeek
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
@ -24,12 +19,18 @@ global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
c: vector of string;
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
@TEST-END-FILE
# the first test writes out the sqlite files...
@TEST-START-FILE one.zeek
redef exit_only_after_terminate = T;
event zeek_init()
{
@ -39,41 +40,25 @@ event zeek_init()
t["a"] = 5;
t["b"] = 3;
t["c"] = 4;
t["whatever"] = 5;
delete t["c"];
t["whatever"] = 5;
add s["I am a set!"];
add s["I am really a set!"];
add s["Believe me - I am a set"];
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
print sort_table(t);
print sort_set(s);
print sort_table(r);
terminate();
}
@TEST-END-FILE
@TEST-START-FILE two.zeek
# read in again - and serve to clones
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
# read in again - and serve to clones
event zeek_init()
{
@ -94,27 +79,9 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
@TEST-END-FILE
@TEST-START-FILE three.zeek
# get copy from master
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
# get copy from master
event zeek_init()
{
@ -129,12 +96,20 @@ event print_me()
terminate();
}
event check_all_set()
{
if ( "whatever" in t && |s| == 3 && "b" in r )
event print_me();
else
schedule 0.1sec { check_all_set() };
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
tablestore = Broker::create_clone("table");
setstore = Broker::create_clone("set");
recordstore = Broker::create_clone("rec");
schedule 2sec { print_me() };
schedule 0.1sec { check_all_set() };
}

View file

@ -1,15 +1,12 @@
# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek one.zeek > output1
# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek two.zeek > output2
# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek common.zeek one.zeek > output1
# @TEST-EXEC: zeek -B broker -b %DIR/sort-stuff.zeek common.zeek two.zeek > output2
# @TEST-EXEC: btest-diff output1
# @TEST-EXEC: btest-diff output2
# @TEST-EXEC: diff output1 output2
# the first test writes out the sqlite files...
@TEST-START-FILE one.zeek
module TestModule;
@TEST-START-FILE common.zeek
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
@ -17,12 +14,15 @@ global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
c: vector of string;
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
@TEST-END-FILE
@TEST-START-FILE one.zeek
event zeek_init()
{
@ -37,9 +37,9 @@ event zeek_init()
add s["I am a set!"];
add s["I am really a set!"];
add s["Believe me - I am a set"];
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
print sort_table(t);
print sort_set(s);
print sort_table(r);
@ -50,22 +50,6 @@ event zeek_init()
# the second one reads them in again
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
event zeek_init()
{
tablestore = Broker::create_master("table", Broker::SQLITE);

View file

@ -1,16 +1,14 @@
# @TEST-PORT: BROKER_PORT
# @TEST-EXEC: btest-bg-run master "zeek -b -B broker -b %DIR/sort-stuff.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run clone "zeek -b -B broker -b %DIR/sort-stuff.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-wait 15
# @TEST-EXEC: btest-bg-run master "zeek -b -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run clone "zeek -b -B broker -b %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-wait 20
#
# @TEST-EXEC: btest-diff clone.out
@TEST-START-FILE master.zeek
@TEST-START-FILE common.zeek
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
@ -24,13 +22,15 @@ type testrec: record {
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
@TEST-END-FILE
@TEST-START-FILE master.zeek
event zeek_init()
{
Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
tablestore = Broker::create_master("table");
setstore = Broker::create_master("set");
recordstore = Broker::create_master("rec");
Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
}
event insert_stuff()
@ -43,8 +43,8 @@ event insert_stuff()
t["a"] = 3;
t["b"] = 3;
t["c"] = 4;
t["whatever"] = 5;
delete t["c"];
t["whatever"] = 5;
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
@ -56,7 +56,7 @@ event insert_stuff()
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print "Peer added ", endpoint;
schedule 3secs { insert_stuff() };
event insert_stuff();
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
@ -67,24 +67,6 @@ event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
@TEST-END-FILE
@TEST-START-FILE clone.zeek
redef exit_only_after_terminate = T;
module TestModule;
global tablestore: opaque of Broker::Store;
global setstore: opaque of Broker::Store;
global recordstore: opaque of Broker::Store;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &broker_store="table";
global s: set[string] &broker_store="set";
global r: table[string] of testrec &broker_allow_complex_type &broker_store="rec";
event zeek_init()
{
Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
@ -98,12 +80,20 @@ event dump_tables()
terminate();
}
event check_all_set()
{
if ( "whatever" in t && "hi" in s && "b" in r )
event dump_tables();
else
schedule 0.1sec { check_all_set() };
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print "Peer added";
tablestore = Broker::create_clone("table");
setstore = Broker::create_clone("set");
recordstore = Broker::create_clone("rec");
schedule 5secs { dump_tables() };
schedule 0.1sec { check_all_set() };
}
@TEST-END-FILE

View file

@ -5,10 +5,10 @@
# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker ../clone.zeek >../clone2.out"
# @TEST-EXEC: btest-bg-wait 20
# @TEST-EXEC: btest-bg-wait 30
#
# @TEST-EXEC: grep -v PEER_UNAVAILABLE worker-1/.stderr > worker-1-stderr
# @TEST-EXEC: btest-diff worker-1-stderr
# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-sort" btest-diff worker-1/err.log
# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-sort" btest-diff worker-2/err.log
@TEST-START-FILE cluster-layout.zeek
redef Cluster::nodes = {
@ -22,49 +22,60 @@ redef Cluster::nodes = {
@TEST-START-FILE master.zeek
@load base/frameworks/cluster
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
global t: table[string] of count &backend=Broker::MEMORY;
global s: table[string] of string &backend=Broker::MEMORY;
event zeek_init()
event add_stuff()
{
t["a"] = 5;
s["a"] = "b";
print t;
}
global peers = 0;
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
++peers;
if ( peers == 2 )
schedule 2sec { add_stuff() };
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
terminate();
--peers;
if ( peers == 0 )
terminate();
}
@TEST-END-FILE
@TEST-START-FILE clone.zeek
@load base/frameworks/cluster
@load base/frameworks/reporter
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
global t: table[count] of count &backend=Broker::MEMORY;
global s: table[string] of count &backend=Broker::MEMORY;
event dump_tables()
{
print t;
print s;
terminate();
}
global errlog = open("err.log");
event Cluster::node_up(name: string, id: string)
global errors = 0;
event reporter_error(t: time, msg: string, location: string)
{
#print "node up", name;
schedule 4secs { dump_tables() };
if ( /ProcessStoreEvent/ in msg )
{
print errlog, msg;
++errors;
}
if ( errors == 2 )
terminate();
}
@TEST-END-FILE

View file

@ -2,9 +2,9 @@
# @TEST-PORT: BROKER_PORT2
# @TEST-PORT: BROKER_PORT3
# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../clone2.zeek >../clone2.out"
# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone2.zeek >../clone2.out"
# @TEST-EXEC: btest-bg-wait 40
#
# @TEST-EXEC: btest-diff master.out
@ -20,32 +20,36 @@ redef Cluster::nodes = {
};
@TEST-END-FILE
@TEST-START-FILE master.zeek
@TEST-START-FILE common.zeek
@load base/frameworks/cluster
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
c: vector of string;
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
global terminate_count = 0;
event zeek_init()
event go_away()
{
terminate();
}
function all_stores_set(): bool
{
return "whatever" in t && "hi" in s && "b" in r;
}
@TEST-END-FILE
@TEST-START-FILE master.zeek
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=1
{
Reporter::info(fmt("Peer added: %s", cat(endpoint)));
@ -54,41 +58,15 @@ event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) &priority=
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
Reporter::info(fmt("Peer lost: %s", cat(endpoint)));
terminate_count += 1;
if ( terminate_count == 2)
{
terminate();
print sort_table(t);
print sort_set(s);
print sort_table(r);
}
print sort_table(t);
print sort_set(s);
print sort_table(r);
terminate();
}
@TEST-END-FILE
@TEST-START-FILE clone.zeek
@load base/frameworks/cluster
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
event terminate_me()
{
terminate();
}
event dump_tables()
{
@ -99,49 +77,34 @@ event dump_tables()
t["a"] = 3;
t["b"] = 3;
t["c"] = 4;
t["whatever"] = 5;
delete t["c"];
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
t["whatever"] = 5;
r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
print sort_table(t);
print sort_set(s);
print sort_table(r);
schedule 10sec { terminate_me() };
}
event Cluster::node_up(name: string, id: string)
{
Reporter::info(fmt("Node Up: %s", name));
schedule 5secs { dump_tables() };
event dump_tables();
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
terminate();
}
event Broker::announce_masters(masters: set[string])
{
Reporter::info(fmt("Received announce_masters: %s", cat(masters)));
}
@TEST-END-FILE
@TEST-START-FILE clone2.zeek
@load base/frameworks/cluster
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
event dump_tables()
{
print sort_table(t);
@ -155,10 +118,18 @@ event Broker::announce_masters(masters: set[string])
Reporter::info(fmt("Received announce_masters: %s", cat(masters)));
}
event check_all_set()
{
if ( all_stores_set() )
event dump_tables();
else
schedule 0.1sec { check_all_set() };
}
event Cluster::node_up(name: string, id: string)
{
Reporter::info(fmt("Node Up: %s", name));
schedule 20secs { dump_tables() };
schedule 0.1sec { check_all_set() };
}
@TEST-END-FILE

View file

@ -5,7 +5,7 @@
# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker ../common.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker ../common.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker ../common.zeek ../clone.zeek >../clone2.out"
# @TEST-EXEC: btest-bg-wait 20
# @TEST-EXEC: btest-bg-wait 30
#
# @TEST-EXEC: btest-diff master.out
# @TEST-EXEC: btest-diff clone.out
@ -24,6 +24,20 @@ redef Cluster::nodes = {
@load base/frameworks/cluster
@load base/frameworks/broker
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
type testrec: record {
a: count;
b: string;
c: vector of string;
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
function sort_set(s: set[string]): vector of string
{
local v: vector of string = vector();
@ -58,22 +72,6 @@ function sort_table(t: table[string] of any): vector of TableEntry
@TEST-END-FILE
@TEST-START-FILE master.zeek
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
event zeek_init()
{
t["a"] = 5;
@ -83,40 +81,28 @@ event zeek_init()
t["a"] = 3;
t["b"] = 3;
t["c"] = 4;
t["whatever"] = 5;
delete t["c"];
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
t["whatever"] = 5;
r["a"] = testrec($a=1, $b="b", $c=vector("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=vector("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=vector("elem1", "elem2"));
print sort_table(t);
print sort_set(s);
print sort_table(r);
}
global peers_lost = 0;
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
terminate();
++peers_lost;
if ( peers_lost == 2 )
terminate();
}
@TEST-END-FILE
@TEST-START-FILE clone.zeek
redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
event dump_tables()
{
@ -126,9 +112,17 @@ event dump_tables()
terminate();
}
event check_all_set()
{
if ( "whatever" in t && "hi" in s && "b" in r )
event dump_tables();
else
schedule 0.1sec { check_all_set() };
}
event Cluster::node_up(name: string, id: string)
{
#print "node up", name;
schedule 4secs { dump_tables() };
schedule 0.1sec { check_all_set() };
}
@TEST-END-FILE

View file

@ -2,10 +2,10 @@
# @TEST-PORT: BROKER_PORT2
# @TEST-PORT: BROKER_PORT3
# @TEST-EXEC: zeek -b %DIR/sort-stuff.zeek preseed-sqlite.zeek;
# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../clone.zeek >../clone2.out"
# @TEST-EXEC: zeek -b %DIR/sort-stuff.zeek common.zeek preseed-sqlite.zeek
# @TEST-EXEC: btest-bg-run manager-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=manager-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../master.zeek >../master.out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone.out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-2 zeek -b -B broker %DIR/sort-stuff.zeek ../common.zeek ../clone.zeek >../clone2.out"
# @TEST-EXEC: btest-bg-wait 40
#
# @TEST-EXEC: btest-diff master.out
@ -21,16 +21,15 @@ redef Cluster::nodes = {
};
@TEST-END-FILE
@TEST-START-FILE preseed-sqlite.zeek
module TestModule;
@TEST-START-FILE common.zeek
type testrec: record {
a: count;
b: string;
c: set[string];
};
@TEST-END-FILE
@TEST-START-FILE preseed-sqlite.zeek
global t: table[string] of count &backend=Broker::SQLITE;
global s: set[string] &backend=Broker::SQLITE;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::SQLITE;
@ -44,8 +43,8 @@ event zeek_init()
t["a"] = 3;
t["b"] = 3;
t["c"] = 4;
t["whatever"] = 5;
delete t["c"];
t["whatever"] = 5;
r["a"] = testrec($a=1, $b="b", $c=set("elem1", "elem2"));
r["a"] = testrec($a=1, $b="c", $c=set("elem1", "elem2"));
r["b"] = testrec($a=2, $b="d", $c=set("elem1", "elem2"));
@ -63,14 +62,6 @@ redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
};
function change_function(t: table[string] of count, tpe: TableChange, idxa: string, val: count)
{
print "This should not print";
@ -90,9 +81,13 @@ event zeek_init()
print sort_table(r);
}
global peers_lost = 0;
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
terminate();
++peers_lost;
if ( peers_lost == 2 )
terminate();
}
@TEST-END-FILE
@ -104,19 +99,10 @@ redef exit_only_after_terminate = T;
redef Log::enable_local_logging = T;
redef Log::default_rotation_interval = 0secs;
module TestModule;
type testrec: record {
a: count;
b: string;
c: set[string];
};
global t: table[string] of count &backend=Broker::MEMORY;
global s: set[string] &backend=Broker::MEMORY;
global r: table[string] of testrec &broker_allow_complex_type &backend=Broker::MEMORY;
event dump_tables()
{
print sort_table(t);
@ -125,9 +111,16 @@ event dump_tables()
terminate();
}
event check_all_set()
{
if ( "whatever" in t && "hi" in s && "b" in r )
event dump_tables();
else
schedule 0.1sec { check_all_set() };
}
event Cluster::node_up(name: string, id: string)
{
#print "node up", name;
schedule 15secs { dump_tables() };
schedule 0.1sec { check_all_set() };
}
@TEST-END-FILE