btest/generic: Add publish_hrw(), publish_rr() and logging tests

They currently use zeromq, but technically they should be valid for
any other backend, too, even broker.
This commit is contained in:
Arne Welzel 2024-11-22 12:13:08 +01:00
parent df69ec1279
commit d816bfb249
11 changed files with 724 additions and 0 deletions

View file

@ -0,0 +1,4 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
error in <...>/cluster-publish-errors.zeek, line 58: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish(topic, <internal>::#0))
error in <...>/cluster-publish-errors.zeek, line 65: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_hrw(Cluster::proxy_pool, key, <internal>::#0))
error in <...>/cluster-publish-errors.zeek, line 72: Publish of unknown record type 'Cluster::MyEvent' (Cluster::publish_rr(Cluster::proxy_pool, key, <internal>::#0))

View file

@ -0,0 +1,201 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
0 worker-1
0 worker-2
1 worker-1
1 worker-2
2 worker-1
2 worker-2
3 worker-1
3 worker-2
4 worker-1
4 worker-2
5 worker-1
5 worker-2
6 worker-1
6 worker-2
7 worker-1
7 worker-2
8 worker-1
8 worker-2
9 worker-1
9 worker-2
10 worker-1
10 worker-2
11 worker-1
11 worker-2
12 worker-1
12 worker-2
13 worker-1
13 worker-2
14 worker-1
14 worker-2
15 worker-1
15 worker-2
16 worker-1
16 worker-2
17 worker-1
17 worker-2
18 worker-1
18 worker-2
19 worker-1
19 worker-2
20 worker-1
20 worker-2
21 worker-1
21 worker-2
22 worker-1
22 worker-2
23 worker-1
23 worker-2
24 worker-1
24 worker-2
25 worker-1
25 worker-2
26 worker-1
26 worker-2
27 worker-1
27 worker-2
28 worker-1
28 worker-2
29 worker-1
29 worker-2
30 worker-1
30 worker-2
31 worker-1
31 worker-2
32 worker-1
32 worker-2
33 worker-1
33 worker-2
34 worker-1
34 worker-2
35 worker-1
35 worker-2
36 worker-1
36 worker-2
37 worker-1
37 worker-2
38 worker-1
38 worker-2
39 worker-1
39 worker-2
40 worker-1
40 worker-2
41 worker-1
41 worker-2
42 worker-1
42 worker-2
43 worker-1
43 worker-2
44 worker-1
44 worker-2
45 worker-1
45 worker-2
46 worker-1
46 worker-2
47 worker-1
47 worker-2
48 worker-1
48 worker-2
49 worker-1
49 worker-2
50 worker-1
50 worker-2
51 worker-1
51 worker-2
52 worker-1
52 worker-2
53 worker-1
53 worker-2
54 worker-1
54 worker-2
55 worker-1
55 worker-2
56 worker-1
56 worker-2
57 worker-1
57 worker-2
58 worker-1
58 worker-2
59 worker-1
59 worker-2
60 worker-1
60 worker-2
61 worker-1
61 worker-2
62 worker-1
62 worker-2
63 worker-1
63 worker-2
64 worker-1
64 worker-2
65 worker-1
65 worker-2
66 worker-1
66 worker-2
67 worker-1
67 worker-2
68 worker-1
68 worker-2
69 worker-1
69 worker-2
70 worker-1
70 worker-2
71 worker-1
71 worker-2
72 worker-1
72 worker-2
73 worker-1
73 worker-2
74 worker-1
74 worker-2
75 worker-1
75 worker-2
76 worker-1
76 worker-2
77 worker-1
77 worker-2
78 worker-1
78 worker-2
79 worker-1
79 worker-2
80 worker-1
80 worker-2
81 worker-1
81 worker-2
82 worker-1
82 worker-2
83 worker-1
83 worker-2
84 worker-1
84 worker-2
85 worker-1
85 worker-2
86 worker-1
86 worker-2
87 worker-1
87 worker-2
88 worker-1
88 worker-2
89 worker-1
89 worker-2
90 worker-1
90 worker-2
91 worker-1
91 worker-2
92 worker-1
92 worker-2
93 worker-1
93 worker-2
94 worker-1
94 worker-2
95 worker-1
95 worker-2
96 worker-1
96 worker-2
97 worker-1
97 worker-2
98 worker-1
98 worker-2
99 worker-1
99 worker-2

View file

@ -0,0 +1,42 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
got pong, 0, args, worker-2, args
got pong, 0, args, worker-2, make_event
got pong, 0, make_event, worker-2, args
got pong, 0, make_event, worker-2, make_event
got pong, 1, args, worker-1, args
got pong, 1, args, worker-1, make_event
got pong, 1, make_event, worker-1, args
got pong, 1, make_event, worker-1, make_event
got pong, 2, args, worker-2, args
got pong, 2, args, worker-2, make_event
got pong, 2, make_event, worker-2, args
got pong, 2, make_event, worker-2, make_event
got pong, 3, args, worker-1, args
got pong, 3, args, worker-1, make_event
got pong, 3, make_event, worker-1, args
got pong, 3, make_event, worker-1, make_event
got pong, 4, args, worker-1, args
got pong, 4, args, worker-1, make_event
got pong, 4, make_event, worker-1, args
got pong, 4, make_event, worker-1, make_event
got pong, 5, args, worker-2, args
got pong, 5, args, worker-2, make_event
got pong, 5, make_event, worker-2, args
got pong, 5, make_event, worker-2, make_event
got pong, 6, args, worker-1, args
got pong, 6, args, worker-1, make_event
got pong, 6, make_event, worker-1, args
got pong, 6, make_event, worker-1, make_event
got pong, 7, args, worker-2, args
got pong, 7, args, worker-2, make_event
got pong, 7, make_event, worker-2, args
got pong, 7, make_event, worker-2, make_event
got pong, 8, args, worker-1, args
got pong, 8, args, worker-1, make_event
got pong, 8, make_event, worker-1, args
got pong, 8, make_event, worker-1, make_event
got pong, 9, args, worker-1, args
got pong, 9, args, worker-1, make_event
got pong, 9, make_event, worker-1, args
got pong, 9, make_event, worker-1, make_event
have 40, finish!

View file

@ -0,0 +1,16 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
a node_up, manager
a node_up, worker-2
got ping, 1, args
got ping, 1, make_event
got ping, 3, args
got ping, 3, make_event
got ping, 4, args
got ping, 4, make_event
got ping, 6, args
got ping, 6, make_event
got ping, 8, args
got ping, 8, make_event
got ping, 9, args
got ping, 9, make_event
z got finish!

View file

@ -0,0 +1,12 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
a node_up, manager
a node_up, worker-1
got ping, 0, args
got ping, 0, make_event
got ping, 2, args
got ping, 2, make_event
got ping, 5, args
got ping, 5, make_event
got ping, 7, args
got ping, 7, make_event
z got finish!

View file

@ -0,0 +1,42 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
got pong, 0, args, worker-1, args
got pong, 0, args, worker-1, make_event
got pong, 0, make_event, worker-1, args
got pong, 0, make_event, worker-1, make_event
got pong, 1, args, worker-2, args
got pong, 1, args, worker-2, make_event
got pong, 1, make_event, worker-2, args
got pong, 1, make_event, worker-2, make_event
got pong, 2, args, worker-1, args
got pong, 2, args, worker-1, make_event
got pong, 2, make_event, worker-1, args
got pong, 2, make_event, worker-1, make_event
got pong, 3, args, worker-2, args
got pong, 3, args, worker-2, make_event
got pong, 3, make_event, worker-2, args
got pong, 3, make_event, worker-2, make_event
got pong, 4, args, worker-1, args
got pong, 4, args, worker-1, make_event
got pong, 4, make_event, worker-1, args
got pong, 4, make_event, worker-1, make_event
got pong, 5, args, worker-2, args
got pong, 5, args, worker-2, make_event
got pong, 5, make_event, worker-2, args
got pong, 5, make_event, worker-2, make_event
got pong, 6, args, worker-1, args
got pong, 6, args, worker-1, make_event
got pong, 6, make_event, worker-1, args
got pong, 6, make_event, worker-1, make_event
got pong, 7, args, worker-2, args
got pong, 7, args, worker-2, make_event
got pong, 7, make_event, worker-2, args
got pong, 7, make_event, worker-2, make_event
got pong, 8, args, worker-1, args
got pong, 8, args, worker-1, make_event
got pong, 8, make_event, worker-1, args
got pong, 8, make_event, worker-1, make_event
got pong, 9, args, worker-2, args
got pong, 9, args, worker-2, make_event
got pong, 9, make_event, worker-2, args
got pong, 9, make_event, worker-2, make_event
have 40, finish!

View file

@ -0,0 +1,14 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
a node_up, manager
a node_up, worker-2
got ping, 0, args
got ping, 0, make_event
got ping, 2, args
got ping, 2, make_event
got ping, 4, args
got ping, 4, make_event
got ping, 6, args
got ping, 6, make_event
got ping, 8, args
got ping, 8, make_event
z got finish!

View file

@ -0,0 +1,14 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
a node_up, manager
a node_up, worker-1
got ping, 1, args
got ping, 1, make_event
got ping, 3, args
got ping, 3, make_event
got ping, 5, args
got ping, 5, make_event
got ping, 7, args
got ping, 7, make_event
got ping, 9, args
got ping, 9, make_event
z got finish!

View file

@ -0,0 +1,177 @@
# @TEST-DOC: Testing round-robin of Log::write() across two loggers.
#
# @TEST-REQUIRES: have-zeromq
#
# @TEST-PORT: XPUB_PORT
# @TEST-PORT: XSUB_PORT
# @TEST-PORT: LOG_PULL_PORT_1
# @TEST-PORT: LOG_PULL_PORT_2
#
# @TEST-EXEC: chmod +x ./check-log.sh
#
# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-two-loggers.zeek cluster-layout.zeek
# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
#
# @TEST-EXEC: zeek -b --parse-only common.zeek manager.zeek worker.zeek
#
# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out"
# @TEST-EXEC: btest-bg-run logger-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=logger-1 zeek -b ../common.zeek >out"
# @TEST-EXEC: btest-bg-run logger-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=logger-2 zeek -b ../common.zeek >out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../worker.zeek >out"
#
# @TEST-EXEC: btest-bg-wait 10
#
# @TEST-EXEC: test $(grep -c worker-1 logger-1/rr2.log) -gt 10
# @TEST-EXEC: test $(grep -c worker-2 logger-1/rr2.log) -gt 10
# @TEST-EXEC: test $(grep -c worker-1 logger-2/rr2.log) -gt 10
# @TEST-EXEC: test $(grep -c worker-2 logger-2/rr2.log) -gt 10
# @TEST-EXEC: zeek-cut < logger-1/rr2.log > rr2.log
# @TEST-EXEC: zeek-cut < logger-2/rr2.log >> rr2.log
# @TEST-EXEC: sort -n rr2.log > rr2.log.sorted
# @TEST-EXEC: btest-diff rr2.log.sorted
# @TEST-START-FILE common.zeek
@load ./zeromq-test-bootstrap.zeek
redef Log::default_rotation_interval = 0sec;
redef Log::flush_interval = 0.03sec;
redef Log::write_buffer_size = 7;
module LogRR;
export {
redef enum Log::ID += { LOG1, LOG2 };
type Info: record {
c: count &log;
from: string &log &default=Cluster::node;
};
global go: event();
global finish: event();
}
event zeek_init()
{
Log::create_stream(LOG1, [$columns=Info, $path="rr1"]);
Log::create_stream(LOG2, [$columns=Info, $path="rr2"]);
}
event finish()
{
terminate();
}
# @TEST-END-FILE
# @TEST-START-FILE manager.zeek
@load ./common.zeek
event check_ready()
{
if ( ! piped_exec("../check-log.sh", "") )
{
Reporter::error("check-log.sh failed");
terminate();
}
if ( file_size("DONE") >= 0 )
{
Cluster::publish(Cluster::worker_topic, LogRR::go);
return;
}
schedule 0.1sec { check_ready() };
}
event zeek_init()
{
event check_ready();
}
global nodes_down: set[string];
event Cluster::node_down(name: string, id: string)
{
print current_time(), "node_down", name;
add nodes_down[name];
if ( |nodes_down| == 2 ) # workers down
Cluster::publish(Cluster::logger_topic, LogRR::finish);
if ( |nodes_down| == 4 ) # both loggers down
terminate();
}
# @TEST-END-FILE
# @TEST-START-FILE worker.zeek
@load ./common.zeek
global do_write2 = F;
event write_log1(c: count)
{
if ( do_write2 )
{
Log::write(LogRR::LOG1, [$c=10000000]);
return;
}
Log::write(LogRR::LOG1, [$c=c]);
Log::flush(LogRR::LOG1);
schedule 0.05sec { write_log1(++c) };
}
event write_log2(c: count)
{
if ( c == 100 )
{
terminate();
return;
}
Log::write(LogRR::LOG2, [$c=c]);
schedule 0.012sec { write_log2(++c) };
}
event LogRR::go()
{
do_write2 = T;
event write_log2(0);
}
event zeek_init()
{
event write_log1(0);
}
# @TEST-END-FILE
@TEST-START-FILE check-log.sh
#!/usr/bin/env bash
#
# This script regularly checks for the loggers rr1.log file until
# both workers appear. Once this happens, creates a READY file
# which will result in workers getting the "go" and sending writes
# to rr2.log
set -eux
LOGGERS="logger-1 logger-2"
WORKERS="worker-1 worker-2"
for logger in $LOGGERS; do
for worker in $WORKERS; do
date +%s
echo check $logger $worker
if ! grep -q "${worker}" ../${logger}/rr1.log; then
exit 0
fi
done
done
echo "DONE"
echo "DONE" > DONE
exit 0
@TEST-END-FILE

View file

@ -0,0 +1,101 @@
# @TEST-DOC: Send ping/pong using publish_hrw(), publish() and make_event()
#
# @TEST-REQUIRES: have-zeromq
#
# @TEST-PORT: XPUB_PORT
# @TEST-PORT: XSUB_PORT
# @TEST-PORT: LOG_PULL_PORT
#
# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-no-logger.zeek cluster-layout.zeek
# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
#
# @TEST-EXEC: zeek -b --parse-only common.zeek manager.zeek worker.zeek
#
# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../worker.zeek >out"
#
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: sort < ./manager/out > ./manager.sorted
# @TEST-EXEC: sort < ./worker-1/out > ./worker-1.sorted
# @TEST-EXEC: sort < ./worker-2/out > ./worker-2.sorted
# @TEST-EXEC: btest-diff manager.sorted
# @TEST-EXEC: btest-diff worker-1.sorted
# @TEST-EXEC: btest-diff worker-2.sorted
# @TEST-START-FILE common.zeek
@load ./zeromq-test-bootstrap.zeek
global finish: event();
global ping: event(c: count, how: string);
global pong: event(c: count, how: string, from: string, from_how: string);
# @TEST-END-FILE
# @TEST-START-FILE manager.zeek
@load ./common.zeek
global nodes_up: set[string];
global nodes_down: set[string];
global pongs: set[count, string, string, string];
global i = 0;
event send_hrw()
{
if (i >= 10 )
return;
Cluster::publish_hrw(Cluster::worker_pool, cat(i), ping, i, "args");
local e = Cluster::make_event(ping, i, "make_event");
Cluster::publish_hrw(Cluster::worker_pool, cat(i), e);
++i;
schedule 0.01sec { send_hrw() };
}
event pong(c: count, how: string, from: string, from_how: string)
{
print "got pong", c, how, from, from_how;
add pongs[c, how, from, from_how];
if ( |pongs| == 40 )
{
print "have 40, finish!";
Cluster::publish(Cluster::worker_topic, finish);
}
}
event Cluster::node_up(name: string, id: string) {
add nodes_up[name];
if ( |nodes_up| == 2 ) {
event send_hrw();
}
}
event Cluster::node_down(name: string, id: string) {
add nodes_down[name];
if ( |nodes_down| == 2 )
terminate();
}
# @TEST-END-FILE
# @TEST-START-FILE worker.zeek
@load ./common.zeek
event ping(c: count, how: string) {
print "got ping", c, how;
Cluster::publish(Cluster::manager_topic, pong, c, how, Cluster::node, "args");
local e = Cluster::make_event(pong, c, how, Cluster::node, "make_event");
Cluster::publish(Cluster::manager_topic, e);
}
event Cluster::node_up(name: string, id: string) {
print "a node_up", name;
}
event finish() &is_used {
print "z got finish!";
terminate();
}
# @TEST-END-FILE

View file

@ -0,0 +1,101 @@
# @TEST-DOC: Send ping/pong using publish_rr(), publish() and make_event()
#
# @TEST-REQUIRES: have-zeromq
#
# @TEST-PORT: XPUB_PORT
# @TEST-PORT: XSUB_PORT
# @TEST-PORT: LOG_PULL_PORT
#
# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-no-logger.zeek cluster-layout.zeek
# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
#
# @TEST-EXEC: zeek -b --parse-only common.zeek manager.zeek worker.zeek
#
# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out"
# @TEST-EXEC: btest-bg-run worker-1 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out"
# @TEST-EXEC: btest-bg-run worker-2 "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-2 zeek -b ../worker.zeek >out"
#
# @TEST-EXEC: btest-bg-wait 10
# @TEST-EXEC: sort < ./manager/out > ./manager.sorted
# @TEST-EXEC: sort < ./worker-1/out > ./worker-1.sorted
# @TEST-EXEC: sort < ./worker-2/out > ./worker-2.sorted
# @TEST-EXEC: btest-diff manager.sorted
# @TEST-EXEC: btest-diff worker-1.sorted
# @TEST-EXEC: btest-diff worker-2.sorted
# @TEST-START-FILE common.zeek
@load ./zeromq-test-bootstrap.zeek
global finish: event();
global ping: event(c: count, how: string);
global pong: event(c: count, how: string, from: string, from_how: string);
# @TEST-END-FILE
# @TEST-START-FILE manager.zeek
@load ./common.zeek
global nodes_up: set[string];
global nodes_down: set[string];
global pongs: set[count, string, string, string];
global i = 0;
event send_rr()
{
if (i >= 10 )
return;
Cluster::publish_rr(Cluster::worker_pool, "ping-key-args", ping, i, "args");
local e = Cluster::make_event(ping, i, "make_event");
Cluster::publish_rr(Cluster::worker_pool, "ping-key-event", e);
++i;
schedule 0.01sec { send_rr() };
}
event pong(c: count, how: string, from: string, from_how: string)
{
print "got pong", c, how, from, from_how;
add pongs[c, how, from, from_how];
if ( |pongs| == 40 )
{
print "have 40, finish!";
Cluster::publish(Cluster::worker_topic, finish);
}
}
event Cluster::node_up(name: string, id: string) {
add nodes_up[name];
if ( |nodes_up| == 2 ) {
event send_rr();
}
}
event Cluster::node_down(name: string, id: string) {
add nodes_down[name];
if ( |nodes_down| == 2 )
terminate();
}
# @TEST-END-FILE
# @TEST-START-FILE worker.zeek
@load ./common.zeek
event ping(c: count, how: string) {
print "got ping", c, how;
Cluster::publish(Cluster::manager_topic, pong, c, how, Cluster::node, "args");
local e = Cluster::make_event(pong, c, how, Cluster::node, "make_event");
Cluster::publish(Cluster::manager_topic, e);
}
event Cluster::node_up(name: string, id: string) {
print "a node_up", name;
}
event finish() &is_used {
print "z got finish!";
terminate();
}
# @TEST-END-FILE