Merge remote-tracking branch 'origin/topic/awelzel/broker-publish-warn'

* origin/topic/awelzel/broker-publish-warn:
  btest: Avoid loading ZeroMQ if not compiled in
  btest/coverage: Avoid warnings in test-all-policy-cluster
  Broker::publish: Warn on using Broker::publish() when inactive

CHANGES

@@ -1,3 +1,20 @@
+7.1.0-dev.766 | 2024-12-11 11:07:53 -0700
+
+  * btest: Avoid loading ZeroMQ if not compiled in (Arne Welzel, Corelight)
+
+    ...at the same time, add some `TEST-REQUIRES: have-zeromq` which
+    unfortunately means that developers will usually want libzmq
+    installed on their system.
+
+  * btest/coverage: Avoid warnings in test-all-policy-cluster (Arne Welzel, Corelight)
+
+  * Broker::publish: Warn on using Broker::publish() when inactive (Arne Welzel, Corelight)
+
+    This is mostly for transitioning base scripts to Cluster::publish() and
+    avoid silent surprises why certain things don't work when using ZeroMQ.
+
+  * Update zeek-testing and zeek-testing-cluster commit hashes (Tim Wojtulewicz, Corelight)
+
 7.1.0-dev.760 | 2024-12-11 09:28:04 +0100
 
   * Update ZAM BiF-tracking (Arne Welzel, Corelight)

VERSION

@@ -1 +1 @@
-7.1.0-dev.760
+7.1.0-dev.766

scripts/test-all-policy.zeek

@@ -11,9 +11,11 @@
 
 # @load frameworks/control/controllee.zeek
 # @load frameworks/control/controller.zeek
+@ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ )
 @load frameworks/cluster/backend/zeromq/__load__.zeek
 # @load frameworks/cluster/backend/zeromq/connect.zeek
 @load frameworks/cluster/backend/zeromq/main.zeek
+@endif
 @load frameworks/cluster/experimental.zeek
 # Loaded via the above through test-all-policy-cluster.test
 # when running as a manager, creates cluster.log entries

scripts/zeekygen/__load__.zeek

@@ -2,7 +2,9 @@
 
 # Scripts which are commented out in test-all-policy.zeek.
 @load protocols/ssl/decryption.zeek
+@ifdef ( Cluster::CLUSTER_BACKEND_ZEROMQ )
 @load frameworks/cluster/backend/zeromq/connect.zeek
+@endif
 @load frameworks/cluster/nodes-experimental/manager.zeek
 @load frameworks/control/controllee.zeek
 @load frameworks/control/controller.zeek

src/broker/Manager.h

@@ -109,7 +109,7 @@ public:
     /**
      * Returns true if any Broker communication is currently active.
      */
-    [[deprecated("Remove with v8.1 - unused")]] bool Active();
+    bool Active();
 
     /**
      * Advances time.  Broker data store expiration is driven by this

src/broker/messaging.bif

@@ -57,8 +57,14 @@ static bool publish_event_args(ArgsSpan args, const zeek::String* topic,
                                zeek::detail::Frame* frame)
 	{
 	zeek::Broker::Manager::ScriptScopeGuard ssg;
+	zeek::ScriptLocationScope scope{frame};
+
 	auto rval = false;
 
+	if ( zeek::broker_mgr != zeek::cluster::backend && ! zeek::broker_mgr->Active() )
+		zeek::reporter->Warning("Non-broker cluster backend configured and Broker manager inactive. "
+                                        "Did you mean to use Cluster::publish() instead of Broker::publish()?");
+
 	if ( args[0]->GetType()->Tag() == zeek::TYPE_RECORD )
 		rval = zeek::broker_mgr->PublishEvent(topic->CheckString(),
 		                                      args[0]->AsRecordVal());

testing/btest/Baseline/cluster.broker.publish-warning/..manager..stderr

@@ -0,0 +1,4 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+warning in ../manager.zeek, line 8: Non-broker cluster backend configured and Broker manager inactive. Did you mean to use Cluster::publish() instead of Broker::publish()?
+warning in ../manager.zeek, line 16: Non-broker cluster backend configured and Broker manager inactive. Did you mean to use Cluster::publish() instead of Broker::publish()?
+received termination signal

testing/btest/Baseline/cluster.broker.publish-warning/..manager.out

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+node_up, worker-1
+node_down, worker-1

testing/btest/Baseline/cluster.broker.publish-warning/..worker.out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+node_up, manager

testing/btest/Baseline/coverage.test-all-policy-cluster/.stderr

@@ -1 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+received termination signal
+received termination signal
+received termination signal
+received termination signal

testing/btest/cluster/broker/publish-warning.zeek

@@ -0,0 +1,61 @@
+# @TEST-DOC: When using ZeroMQ, Broker::publish() produces a warning.
+#
+# @TEST-REQUIRES: have-zeromq
+#
+# @TEST-GROUP: cluster-zeromq
+#
+# @TEST-PORT: XPUB_PORT
+# @TEST-PORT: XSUB_PORT
+# @TEST-PORT: LOG_PULL_PORT
+#
+# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek
+# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
+#
+# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out"
+# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out"
+#
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: btest-diff ./manager/out
+# @TEST-EXEC: btest-diff ./manager/.stderr
+# @TEST-EXEC: btest-diff ./worker/out
+
+
+# @TEST-START-FILE common.zeek
+@load ./zeromq-test-bootstrap
+
+global finish: event(name: string);
+# @TEST-END-FILE
+
+# @TEST-START-FILE manager.zeek
+@load ./common.zeek
+# If a node comes up that isn't us, send it a finish event.
+event Cluster::node_up(name: string, id: string) {
+	print "node_up", name;
+	Cluster::publish(Cluster::nodeid_topic(id), finish, Cluster::node);
+
+	# Also via broker, but this produces a warning which we test for.
+	Broker::publish(Cluster::nodeid_topic(id), finish, Cluster::node);
+}
+
+# If the worker vanishes, finish the test.
+event Cluster::node_down(name: string, id: string) {
+	print "node_down", name;
+
+	# Do another Broker::publish(), just for the kicks.
+	Broker::publish(Cluster::nodeid_topic(id), finish, Cluster::node);
+
+	terminate();
+}
+# @TEST-END-FILE
+
+# @TEST-START-FILE worker.zeek
+@load ./common.zeek
+
+event Cluster::node_up(name: string, id: string) {
+	print "node_up", name;
+}
+
+event finish(name: string) &is_used {
+	terminate();
+}
+# @TEST-END-FILE

testing/btest/coverage/bare-load-baseline.test

@@ -9,6 +9,7 @@
 # below does. Don't ask. :-)
 
 # @TEST-REQUIRES: $SCRIPTS/have-spicy  # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable.
+# @TEST-REQUIRES: have-zeromq  # Require ZeroMQ so that the plugin's bif file is loaded.
 # @TEST-REQUIRES: ! have-spicy-ssl  # Enabling Spicy SSL changes the loaded scripts, skip in this case
 # @TEST-EXEC: zeek -b misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log

testing/btest/coverage/bare-mode-errors.test

@@ -6,6 +6,7 @@
 #
 # Require Spicy, otherwise its scripts cannot be loaded.
 # @TEST-REQUIRES: have-spicy
+# @TEST-REQUIRES: have-zeromq
 #
 # @TEST-EXEC: test -d $DIST/scripts
 # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.zeek`; do zeek -b --parse-only $script >>errors 2>&1; done

testing/btest/coverage/default-load-baseline.test

@@ -7,7 +7,8 @@
 # prefix to make the test work everywhere. That's what the sed magic
 # below does. Don't ask. :-)
 
-# @TEST-REQUIRES: ${SCRIPTS}/have-spicy
+# @TEST-REQUIRES: have-spicy
+# @TEST-REQUIRES: have-zeromq
 # @TEST-REQUIRES: ! have-spicy-ssl  # Enabling Spicy SSL changes the loaded scripts, skip in this case
 # @TEST-EXEC: zeek misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log

testing/btest/coverage/test-all-policy-cluster.test

@@ -16,6 +16,10 @@
 
 @load test-all-policy
 
+# Flip this to broker to avoid warnings() due Broker::publish()
+# calls in some scripts.
+redef Cluster::backend = Cluster::CLUSTER_BACKEND_BROKER;
+
 @TEST-START-FILE cluster-layout.zeek
 redef Cluster::nodes = {
 	["manager"]  = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1"))],

testing/btest/plugins/hooks.zeek

@@ -1,5 +1,6 @@
 # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1"
-# @TEST-REQUIRES: ${SCRIPTS}/have-spicy  # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable.
+# @TEST-REQUIRES: have-spicy  # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable.
+# @TEST-REQUIRES: have-zeromq  # This test logs loaded scripts, so disable it if ZeroMQ isn't available.
 # @TEST-REQUIRES: ! have-spicy-ssl  # Enabling Spicy SSL changes baselines and thus changes raised events. Skip in this case.
 # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks
 # @TEST-EXEC: cp -r %DIR/hooks-plugin/* .