diff --git a/CHANGES b/CHANGES index c895151485..2b1c6304f4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,20 @@ +7.1.0-dev.766 | 2024-12-11 11:07:53 -0700 + + * btest: Avoid loading ZeroMQ if not compiled in (Arne Welzel, Corelight) + + ...at the same time, add some `TEST-REQUIRES: have-zeromq` which + unfortunately means that developers will usually want libzmq + installed on their system. + + * btest/coverage: Avoid warnings in test-all-policy-cluster (Arne Welzel, Corelight) + + * Broker::publish: Warn on using Broker::publish() when inactive (Arne Welzel, Corelight) + + This is mostly for transitioning base scripts to Cluster::publish() and + avoid silent surprises why certain things don't work when using ZeroMQ. + + * Update zeek-testing and zeek-testing-cluster commit hashes (Tim Wojtulewicz, Corelight) + 7.1.0-dev.760 | 2024-12-11 09:28:04 +0100 * Update ZAM BiF-tracking (Arne Welzel, Corelight) diff --git a/VERSION b/VERSION index 0fbaee588f..b10412627e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.1.0-dev.760 +7.1.0-dev.766 diff --git a/scripts/test-all-policy.zeek b/scripts/test-all-policy.zeek index 58930dc194..324d6745d6 100644 --- a/scripts/test-all-policy.zeek +++ b/scripts/test-all-policy.zeek @@ -11,9 +11,11 @@ # @load frameworks/control/controllee.zeek # @load frameworks/control/controller.zeek +@ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ ) @load frameworks/cluster/backend/zeromq/__load__.zeek # @load frameworks/cluster/backend/zeromq/connect.zeek @load frameworks/cluster/backend/zeromq/main.zeek +@endif @load frameworks/cluster/experimental.zeek # Loaded via the above through test-all-policy-cluster.test # when running as a manager, creates cluster.log entries diff --git a/scripts/zeekygen/__load__.zeek b/scripts/zeekygen/__load__.zeek index d22dba2a97..b7ac6992a6 100644 --- a/scripts/zeekygen/__load__.zeek +++ b/scripts/zeekygen/__load__.zeek @@ -2,7 +2,9 @@ # Scripts which are commented out in test-all-policy.zeek. @load protocols/ssl/decryption.zeek +@ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ ) @load frameworks/cluster/backend/zeromq/connect.zeek +@endif @load frameworks/cluster/nodes-experimental/manager.zeek @load frameworks/control/controllee.zeek @load frameworks/control/controller.zeek diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 51974685d9..5b02a0ba71 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -109,7 +109,7 @@ public: /** * Returns true if any Broker communication is currently active. */ - [[deprecated("Remove with v8.1 - unused")]] bool Active(); + bool Active(); /** * Advances time. Broker data store expiration is driven by this diff --git a/src/broker/messaging.bif b/src/broker/messaging.bif index 96008861de..4347616a97 100644 --- a/src/broker/messaging.bif +++ b/src/broker/messaging.bif @@ -57,8 +57,14 @@ static bool publish_event_args(ArgsSpan args, const zeek::String* topic, zeek::detail::Frame* frame) { zeek::Broker::Manager::ScriptScopeGuard ssg; + zeek::ScriptLocationScope scope{frame}; + auto rval = false; + if ( zeek::broker_mgr != zeek::cluster::backend && ! zeek::broker_mgr->Active() ) + zeek::reporter->Warning("Non-broker cluster backend configured and Broker manager inactive. " + "Did you mean to use Cluster::publish() instead of Broker::publish()?"); + if ( args[0]->GetType()->Tag() == zeek::TYPE_RECORD ) rval = zeek::broker_mgr->PublishEvent(topic->CheckString(), args[0]->AsRecordVal()); diff --git a/testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr b/testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr new file mode 100644 index 0000000000..9e7871896a --- /dev/null +++ b/testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +warning in ../manager.zeek, line 8: Non-broker cluster backend configured and Broker manager inactive. Did you mean to use Cluster::publish() instead of Broker::publish()? +warning in ../manager.zeek, line 16: Non-broker cluster backend configured and Broker manager inactive. Did you mean to use Cluster::publish() instead of Broker::publish()? +received termination signal diff --git a/testing/btest/Baseline/cluster.broker.publish-warning/..manager.out b/testing/btest/Baseline/cluster.broker.publish-warning/..manager.out new file mode 100644 index 0000000000..7e67339b4d --- /dev/null +++ b/testing/btest/Baseline/cluster.broker.publish-warning/..manager.out @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +node_up, worker-1 +node_down, worker-1 diff --git a/testing/btest/Baseline/cluster.broker.publish-warning/..worker.out b/testing/btest/Baseline/cluster.broker.publish-warning/..worker.out new file mode 100644 index 0000000000..386f8ae30f --- /dev/null +++ b/testing/btest/Baseline/cluster.broker.publish-warning/..worker.out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +node_up, manager diff --git a/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr b/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr index 49d861c74c..bff9a64e41 100644 --- a/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr +++ b/testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr @@ -1 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +received termination signal +received termination signal +received termination signal +received termination signal diff --git a/testing/btest/cluster/broker/publish-warning.zeek b/testing/btest/cluster/broker/publish-warning.zeek new file mode 100644 index 0000000000..38dff8e980 --- /dev/null +++ b/testing/btest/cluster/broker/publish-warning.zeek @@ -0,0 +1,61 @@ +# @TEST-DOC: When using ZeroMQ, Broker::publish() produces a warning. +# +# @TEST-REQUIRES: have-zeromq +# +# @TEST-GROUP: cluster-zeromq +# +# @TEST-PORT: XPUB_PORT +# @TEST-PORT: XSUB_PORT +# @TEST-PORT: LOG_PULL_PORT +# +# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek +# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek +# +# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out" +# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out" +# +# @TEST-EXEC: btest-bg-wait 30 +# @TEST-EXEC: btest-diff ./manager/out +# @TEST-EXEC: btest-diff ./manager/.stderr +# @TEST-EXEC: btest-diff ./worker/out + + +# @TEST-START-FILE common.zeek +@load ./zeromq-test-bootstrap + +global finish: event(name: string); +# @TEST-END-FILE + +# @TEST-START-FILE manager.zeek +@load ./common.zeek +# If a node comes up that isn't us, send it a finish event. +event Cluster::node_up(name: string, id: string) { + print "node_up", name; + Cluster::publish(Cluster::nodeid_topic(id), finish, Cluster::node); + + # Also via broker, but this produces a warning which we test for. + Broker::publish(Cluster::nodeid_topic(id), finish, Cluster::node); +} + +# If the worker vanishes, finish the test. +event Cluster::node_down(name: string, id: string) { + print "node_down", name; + + # Do another Broker::publish(), just for the kicks. + Broker::publish(Cluster::nodeid_topic(id), finish, Cluster::node); + + terminate(); +} +# @TEST-END-FILE + +# @TEST-START-FILE worker.zeek +@load ./common.zeek + +event Cluster::node_up(name: string, id: string) { + print "node_up", name; +} + +event finish(name: string) &is_used { + terminate(); +} +# @TEST-END-FILE diff --git a/testing/btest/coverage/bare-load-baseline.test b/testing/btest/coverage/bare-load-baseline.test index 8b73fb125c..95f06a12eb 100644 --- a/testing/btest/coverage/bare-load-baseline.test +++ b/testing/btest/coverage/bare-load-baseline.test @@ -9,6 +9,7 @@ # below does. Don't ask. :-) # @TEST-REQUIRES: $SCRIPTS/have-spicy # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable. +# @TEST-REQUIRES: have-zeromq # Require ZeroMQ so that the plugin's bif file is loaded. # @TEST-REQUIRES: ! have-spicy-ssl # Enabling Spicy SSL changes the loaded scripts, skip in this case # @TEST-EXEC: zeek -b misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index ea5b66955e..748a3f95e5 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -6,6 +6,7 @@ # # Require Spicy, otherwise its scripts cannot be loaded. # @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: have-zeromq # # @TEST-EXEC: test -d $DIST/scripts # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.zeek`; do zeek -b --parse-only $script >>errors 2>&1; done diff --git a/testing/btest/coverage/default-load-baseline.test b/testing/btest/coverage/default-load-baseline.test index 2d098279fe..bbf79b6a52 100644 --- a/testing/btest/coverage/default-load-baseline.test +++ b/testing/btest/coverage/default-load-baseline.test @@ -7,7 +7,8 @@ # prefix to make the test work everywhere. That's what the sed magic # below does. Don't ask. :-) -# @TEST-REQUIRES: ${SCRIPTS}/have-spicy +# @TEST-REQUIRES: have-spicy +# @TEST-REQUIRES: have-zeromq # @TEST-REQUIRES: ! have-spicy-ssl # Enabling Spicy SSL changes the loaded scripts, skip in this case # @TEST-EXEC: zeek misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log diff --git a/testing/btest/coverage/test-all-policy-cluster.test b/testing/btest/coverage/test-all-policy-cluster.test index dc5e14e385..6aab29a1e7 100644 --- a/testing/btest/coverage/test-all-policy-cluster.test +++ b/testing/btest/coverage/test-all-policy-cluster.test @@ -16,6 +16,10 @@ @load test-all-policy +# Flip this to broker to avoid warnings() due Broker::publish() +# calls in some scripts. +redef Cluster::backend = Cluster::CLUSTER_BACKEND_BROKER; + @TEST-START-FILE cluster-layout.zeek redef Cluster::nodes = { ["manager"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1"))], diff --git a/testing/btest/plugins/hooks.zeek b/testing/btest/plugins/hooks.zeek index b65d321230..25e636a411 100644 --- a/testing/btest/plugins/hooks.zeek +++ b/testing/btest/plugins/hooks.zeek @@ -1,5 +1,6 @@ # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" -# @TEST-REQUIRES: ${SCRIPTS}/have-spicy # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable. +# @TEST-REQUIRES: have-spicy # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable. +# @TEST-REQUIRES: have-zeromq # This test logs loaded scripts, so disable it if ZeroMQ isn't available. # @TEST-REQUIRES: ! have-spicy-ssl # Enabling Spicy SSL changes baselines and thus changes raised events. Skip in this case. # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* .