Merge remote-tracking branch 'origin/topic/awelzel/pluggable-cluster-backends-part2'

* origin/topic/awelzel/pluggable-cluster-backends-part2:
  ci/test.sh: Run doctest with TZ=UTC
  cluster/setup-connections: Switch to Cluster::subscribe(), short-circuit broker
  cluster/serializer: Add Broker based event serializers
  cluster: Add Cluster scoped bifs
  Reporter: Add ScriptLocationScope helper
  init-bare/zeek-setup: Add Cluster::backend const &redef
  broker: Implement cluster::Backend interface
  Broker: Fix some error messages
  broker: Remove MakeEvent(ArgsSpan)
This commit is contained in:
Arne Welzel 2024-11-26 17:45:08 +01:00
commit 3ce41f1eff
36 changed files with 1107 additions and 64 deletions

View file

@ -0,0 +1,12 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
receiver added peer: endpoint=127.0.0.1 msg=handshake successful
is_remote should be T, and is, T
receiver got ping: my-message, 1
is_remote should be T, and is, T
receiver got ping: my-message, 2
is_remote should be T, and is, T
receiver got ping: my-message, 3
is_remote should be T, and is, T
receiver got ping: my-message, 4
is_remote should be T, and is, T
receiver got ping: my-message, 5

View file

@ -0,0 +1,11 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
sender added peer: endpoint=127.0.0.1 msg=handshake successful
is_remote should be T, and is, T
sender got pong: my-message, 1
is_remote should be T, and is, T
sender got pong: my-message, 2
is_remote should be T, and is, T
sender got pong: my-message, 3
is_remote should be T, and is, T
sender got pong: my-message, 4
sender lost peer: endpoint=127.0.0.1 msg=lost connection to remote peer

View file

@ -0,0 +1,15 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
error in <...>/errors.zeek, line 20: no event arguments given (Cluster::publish(topic))
error in <...>/errors.zeek, line 21: not enough arguments (Cluster::make_event())
error in <...>/errors.zeek, line 24: bad number of arguments for ping1: got 0, expect 2
error in <...>/errors.zeek, line 25: bad number of arguments for ping1: got 0, expect 2
error in <...>/errors.zeek, line 28: bad number of arguments for ping1: got 1, expect 2
error in <...>/errors.zeek, line 29: bad number of arguments for ping1: got 1, expect 2
error in <...>/errors.zeek, line 32: bad number of arguments for ping1: got 3, expect 2
error in <...>/errors.zeek, line 33: bad number of arguments for ping1: got 3, expect 2
error in <...>/errors.zeek, line 41: event parameter #2 type mismatch, got count, expecting string
error in <...>/errors.zeek, line 42: event parameter #2 type mismatch, got count, expecting string
error in <...>/errors.zeek, line 45: unexpected function type for hook1: hook
error in <...>/errors.zeek, line 46: unexpected function type for hook1: hook
error in <...>/errors.zeek, line 49: expected function or record as first argument, got count (Cluster::publish(topic, 1))
error in <...>/errors.zeek, line 50: got non-event type 'count' (Cluster::make_event(1))

View file

@ -0,0 +1,10 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
wrong number of args
r1, [ev=<uninitialized>, args=[]]
r2, [ev=<uninitialized>, args=[]]
r3, [ev=<uninitialized>, args=[]]
r4, [ev=<uninitialized>, args=[]]
wrong types
r1, [ev=<uninitialized>, args=[]]
r2, [ev=<uninitialized>, args=[]]
r3, [ev=<uninitialized>, args=[]]

View file

@ -0,0 +1,7 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
error in <...>/make_event.zeek, line 32: not enough arguments (Cluster::make_event())
error in <...>/make_event.zeek, line 37: got non-event type 'string' (Cluster::make_event(a))
error in <...>/make_event.zeek, line 42: unexpected function type for test_fun: function
error in <...>/make_event.zeek, line 47: unexpected function type for test_hook: hook
error in <...>/make_event.zeek, line 52: bad number of arguments for test_event2: got 0, expect 1
error in <...>/make_event.zeek, line 57: bad number of arguments for test_event2: got 2, expect 1

View file

@ -0,0 +1,3 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
event(), []
event(s:string), [abc]

View file

@ -0,0 +1,7 @@
### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
Zeek::Broker_Serializer - Event serialization using Broker event formats (binary and json) (built-in)
[Event Serializer] BROKER_BIN_V1 (Cluster::EVENT_SERIALIZER_BROKER_BIN_V1)
[Event Serializer] BROKER_JSON_V1 (Cluster::EVENT_SERIALIZER_BROKER_JSON_V1)
Cluster::EVENT_SERIALIZER_BROKER_BIN_V1, Cluster::EventSerializerTag
Cluster::EVENT_SERIALIZER_BROKER_JSON_V1, Cluster::EventSerializerTag

View file

@ -135,6 +135,7 @@ scripts/base/init-frameworks-and-bifs.zeek
scripts/base/frameworks/cluster/main.zeek
scripts/base/frameworks/control/__load__.zeek
scripts/base/frameworks/control/main.zeek
build/scripts/base/bif/cluster.bif.zeek
scripts/base/frameworks/cluster/pools.zeek
scripts/base/utils/hash_hrw.zeek
scripts/base/frameworks/config/__load__.zeek

View file

@ -135,6 +135,7 @@ scripts/base/init-frameworks-and-bifs.zeek
scripts/base/frameworks/cluster/main.zeek
scripts/base/frameworks/control/__load__.zeek
scripts/base/frameworks/control/main.zeek
build/scripts/base/bif/cluster.bif.zeek
scripts/base/frameworks/cluster/pools.zeek
scripts/base/utils/hash_hrw.zeek
scripts/base/frameworks/config/__load__.zeek

View file

@ -462,6 +462,7 @@
0.000000 MetaHookPost LoadFile(0, ./api, <...>/api.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek) -> -1
@ -540,6 +541,7 @@
0.000000 MetaHookPost LoadFile(0, base<...>/ayiya, <...>/ayiya) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/broker, <...>/broker) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/cluster, <...>/cluster) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/cluster.bif, <...>/cluster.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> -1
0.000000 MetaHookPost LoadFile(0, base<...>/config, <...>/config) -> -1
@ -763,6 +765,7 @@
0.000000 MetaHookPost LoadFileExtended(0, ./api, <...>/api.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek) -> (-1, <no content>)
@ -841,6 +844,7 @@
0.000000 MetaHookPost LoadFileExtended(0, base<...>/ayiya, <...>/ayiya) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/broker, <...>/broker) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/cluster, <...>/cluster) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/cluster.bif, <...>/cluster.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek) -> (-1, <no content>)
0.000000 MetaHookPost LoadFileExtended(0, base<...>/config, <...>/config) -> (-1, <no content>)
@ -1396,6 +1400,7 @@
0.000000 MetaHookPre LoadFile(0, ./api, <...>/api.zeek)
0.000000 MetaHookPre LoadFile(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFile(0, ./const.bif.zeek, <...>/const.bif.zeek)
@ -1474,6 +1479,7 @@
0.000000 MetaHookPre LoadFile(0, base<...>/ayiya, <...>/ayiya)
0.000000 MetaHookPre LoadFile(0, base<...>/broker, <...>/broker)
0.000000 MetaHookPre LoadFile(0, base<...>/cluster, <...>/cluster)
0.000000 MetaHookPre LoadFile(0, base<...>/cluster.bif, <...>/cluster.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/comm.bif, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/communityid.bif, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFile(0, base<...>/config, <...>/config)
@ -1697,6 +1703,7 @@
0.000000 MetaHookPre LoadFileExtended(0, ./api, <...>/api.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./bloom-filter.bif.zeek, <...>/bloom-filter.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./cardinality-counter.bif.zeek, <...>/cardinality-counter.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./cluster.bif.zeek, <...>/cluster.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./comm.bif.zeek, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./communityid.bif.zeek, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, ./const.bif.zeek, <...>/const.bif.zeek)
@ -1775,6 +1782,7 @@
0.000000 MetaHookPre LoadFileExtended(0, base<...>/ayiya, <...>/ayiya)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/broker, <...>/broker)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/cluster, <...>/cluster)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/cluster.bif, <...>/cluster.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/comm.bif, <...>/comm.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/communityid.bif, <...>/communityid.bif.zeek)
0.000000 MetaHookPre LoadFileExtended(0, base<...>/config, <...>/config)
@ -2331,6 +2339,7 @@
0.000000 | HookLoadFile ./audio <...>/audio.sig
0.000000 | HookLoadFile ./bloom-filter.bif.zeek <...>/bloom-filter.bif.zeek
0.000000 | HookLoadFile ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek
0.000000 | HookLoadFile ./cluster.bif.zeek <...>/cluster.bif.zeek
0.000000 | HookLoadFile ./comm.bif.zeek <...>/comm.bif.zeek
0.000000 | HookLoadFile ./communityid.bif.zeek <...>/communityid.bif.zeek
0.000000 | HookLoadFile ./const.bif.zeek <...>/const.bif.zeek
@ -2419,6 +2428,7 @@
0.000000 | HookLoadFile base<...>/ayiya <...>/ayiya
0.000000 | HookLoadFile base<...>/broker <...>/broker
0.000000 | HookLoadFile base<...>/cluster <...>/cluster
0.000000 | HookLoadFile base<...>/cluster.bif <...>/cluster.bif.zeek
0.000000 | HookLoadFile base<...>/comm.bif <...>/comm.bif.zeek
0.000000 | HookLoadFile base<...>/communityid.bif <...>/communityid.bif.zeek
0.000000 | HookLoadFile base<...>/config <...>/config
@ -2632,6 +2642,7 @@
0.000000 | HookLoadFileExtended ./audio <...>/audio.sig
0.000000 | HookLoadFileExtended ./bloom-filter.bif.zeek <...>/bloom-filter.bif.zeek
0.000000 | HookLoadFileExtended ./cardinality-counter.bif.zeek <...>/cardinality-counter.bif.zeek
0.000000 | HookLoadFileExtended ./cluster.bif.zeek <...>/cluster.bif.zeek
0.000000 | HookLoadFileExtended ./comm.bif.zeek <...>/comm.bif.zeek
0.000000 | HookLoadFileExtended ./communityid.bif.zeek <...>/communityid.bif.zeek
0.000000 | HookLoadFileExtended ./const.bif.zeek <...>/const.bif.zeek
@ -2720,6 +2731,7 @@
0.000000 | HookLoadFileExtended base<...>/ayiya <...>/ayiya
0.000000 | HookLoadFileExtended base<...>/broker <...>/broker
0.000000 | HookLoadFileExtended base<...>/cluster <...>/cluster
0.000000 | HookLoadFileExtended base<...>/cluster.bif <...>/cluster.bif.zeek
0.000000 | HookLoadFileExtended base<...>/comm.bif <...>/comm.bif.zeek
0.000000 | HookLoadFileExtended base<...>/communityid.bif <...>/communityid.bif.zeek
0.000000 | HookLoadFileExtended base<...>/config <...>/config

View file

@ -0,0 +1,106 @@
# @TEST-DOC: Use Cluster::subscribe() and Cluster::publish() with Broker
# @TEST-GROUP: cluster
#
# @TEST-PORT: BROKER_PORT
#
# @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"
# @TEST-EXEC: btest-bg-run send "zeek -b ../send.zeek >send.out"
#
# @TEST-EXEC: btest-bg-wait 30
# @TEST-EXEC: btest-diff recv/recv.out
# @TEST-EXEC: btest-diff send/send.out
@TEST-START-FILE send.zeek
redef Cluster::backend = Cluster::CLUSTER_BACKEND_BROKER;
redef exit_only_after_terminate = T;
global event_count = 0;
global ping: event(msg: string, c: count);
event zeek_init()
{
Cluster::init();
Broker::peer("127.0.0.1", to_port(getenv("BROKER_PORT")));
Cluster::subscribe("zeek/event/my_topic");
}
function send_event()
{
++event_count;
local e = Cluster::make_event(ping, "my-message", event_count);
Cluster::publish("zeek/event/my_topic", e);
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print fmt("sender added peer: endpoint=%s msg=%s",
endpoint$network$address, msg);
send_event();
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
print fmt("sender lost peer: endpoint=%s msg=%s",
endpoint$network$address, msg);
Cluster::unsubscribe("zeek/event/my_topic");
terminate();
}
event pong(msg: string, n: count)
{
print "is_remote should be T, and is", is_remote_event();
print fmt("sender got pong: %s, %s", msg, n);
send_event();
}
@TEST-END-FILE
@TEST-START-FILE recv.zeek
redef Cluster::backend = Cluster::CLUSTER_BACKEND_BROKER;
redef exit_only_after_terminate = T;
const events_to_recv = 5;
global handler: event(msg: string, c: count);
global auto_handler: event(msg: string, c: count);
global pong: event(msg: string, c: count);
event zeek_init()
{
Cluster::init();
Broker::listen("127.0.0.1", to_port(getenv("BROKER_PORT")));
Cluster::subscribe("zeek/event/my_topic");
}
event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string)
{
print fmt("receiver added peer: endpoint=%s msg=%s", endpoint$network$address, msg);
}
event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string)
{
print fmt("receiver lost peer: endpoint=%s msg=%s", endpoint$network$address, msg);
}
event ping(msg: string, n: count)
{
print "is_remote should be T, and is", is_remote_event();
print fmt("receiver got ping: %s, %s", msg, n);
if ( n == events_to_recv )
{
Cluster::unsubscribe("zeek/event/my_topic");
terminate();
return;
}
Cluster::publish("zeek/event/my_topic", pong, msg, n);
}
@TEST-END-FILE

View file

@ -0,0 +1,52 @@
# @TEST-DOC: Test some validation errors of cluster bifs
#
# @TEST-EXEC: zeek --parse-only -b %INPUT
# @TEST-EXEC: zeek -b %INPUT
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stdout
event ping1(c: count, how: string) &is_used
{
}
hook hook1(c: count, how: string) &is_used
{
}
event zeek_init() &priority=-1
{
print "wrong number of args";
Cluster::publish("topic");
local r1 = Cluster::make_event();
print "r1", r1;
Cluster::publish("topic", ping1);
local r2 = Cluster::make_event(ping1);
print "r2", r2;
Cluster::publish("topic", ping1, 1);
local r3 = Cluster::make_event(ping1, 1);
print "r3", r3;
Cluster::publish("topic", ping1, 1, "args", 1.2.3.4);
local r4 = Cluster::make_event(ping1, 1, "event", 1.2.3.4);
print "r4", r4;
}
event zeek_init() &priority=-2
{
print "wrong types";
Cluster::publish("topic", ping1, 1, 2);
local r1 = Cluster::make_event(ping1, 1, 2);
print "r1", r1;
Cluster::publish("topic", hook1, 1, "hook");
local r2 = Cluster::make_event(hook1, 1, "hook");
print "r2", r2;
Cluster::publish("topic", 1);
local r3 = Cluster::make_event(1);
print "r3", r2;
}

View file

@ -0,0 +1,58 @@
# @TEST-DOC: Test make_event behavior.
#
# @TEST-EXEC: zeek -b %INPUT >out
#
# @TEST-EXEC: btest-diff out
# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderr
function test_fun() { }
hook test_hook() { }
event test_event() { }
event test_event2(s: string) { }
function as_cluster_event(e: any): Cluster::Event
{
assert e is Cluster::Event;
return e as Cluster::Event;
}
event zeek_init() &priority=10
{
local e1 = Cluster::make_event(test_event);
local ce1 = as_cluster_event(e1);
print type_name(ce1$ev), ce1$args;
local e2 = Cluster::make_event(test_event2, "abc");
local ce2 = as_cluster_event(e2);
print type_name(ce2$ev), ce2$args;
}
event zeek_init() &priority=-10
{
local e = Cluster::make_event();
}
event zeek_init() &priority=-11
{
local e = Cluster::make_event("a");
}
event zeek_init() &priority=-12
{
local e = Cluster::make_event(test_fun);
}
event zeek_init() &priority=-13
{
local e = Cluster::make_event(test_hook);
}
event zeek_init() &priority=-14
{
local e = Cluster::make_event(test_event2);
}
event zeek_init() &priority=-15
{
local e = Cluster::make_event(test_event2, "a", "b");
}

View file

@ -0,0 +1,11 @@
# @TEST-DOC: Test cluster backend enum
#
# @TEST-EXEC: zeek -NN Zeek::Broker_Serializer >>out
# @TEST-EXEC: zeek -b %INPUT >>out
# @TEST-EXEC: btest-diff out
event zeek_init()
{
print Cluster::EVENT_SERIALIZER_BROKER_BIN_V1, type_name(Cluster::EVENT_SERIALIZER_BROKER_BIN_V1);
print Cluster::EVENT_SERIALIZER_BROKER_JSON_V1, type_name(Cluster::EVENT_SERIALIZER_BROKER_JSON_V1);
}