btest/cluster/telemetry: Add smoke testing for telemetry

testing/btest/Baseline/cluster.telemetry.two-nodes/..manager.out

@@ -0,0 +1,9 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+zeek, zeek_cluster_core_incoming_events_total, [node], [manager], 2.0
+zeek, zeek_cluster_core_outgoing_events_total, [node], [manager], 3.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [Cluster::Backend::ZeroMQ::hello, manager, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [Cluster::hello, manager, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [Cluster::Backend::ZeroMQ::hello, manager, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [Cluster::hello, manager, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [finish, manager, zeek.cluster.nodeid.__normalized__], 1.0

testing/btest/Baseline/cluster.telemetry.two-nodes/..worker.out

@@ -0,0 +1,9 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+zeek, zeek_cluster_core_incoming_events_total, [node], [worker-1], 3.0
+zeek, zeek_cluster_core_outgoing_events_total, [node], [worker-1], 2.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [Cluster::Backend::ZeroMQ::hello, worker-1, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [Cluster::hello, worker-1, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [finish, worker-1, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [Cluster::Backend::ZeroMQ::hello, worker-1, zeek.cluster.nodeid.__normalized__], 1.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [Cluster::hello, worker-1, zeek.cluster.nodeid.__normalized__], 1.0

testing/btest/Baseline/cluster.telemetry.ws-app/..manager..stderr

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+received termination signal

testing/btest/Baseline/cluster.telemetry.ws-app/..manager..stdout

@@ -0,0 +1,33 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+Cluster::websocket_client_added, [/test/pings, /zeek/wstest/ws1/]
+zeek, zeek_cluster_core_debug_incoming_event_sizes, [handler, node, topic], [ping, manager, /test/pings/0], [0.0, 1.0, 0.0, 3.0, 4.0, 17.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_core_debug_incoming_event_sizes, [handler, node, topic], [ping, manager, /test/pings/1], [0.0, 0.0, 1.0, 3.0, 4.0, 17.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_core_debug_incoming_event_sizes, [handler, node, topic], [ping, manager, /test/pings/2], [0.0, 0.0, 0.0, 4.0, 3.0, 18.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_core_debug_incoming_event_sizes, [handler, node, topic], [ping, manager, /test/pings/3], [0.0, 0.0, 0.0, 3.0, 4.0, 18.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_core_debug_outgoing_event_sizes, [handler, node, script_location, topic], [ping, manager, ./manager.zeek:32, /test/pings/0], [0.0, 1.0, 0.0, 3.0, 4.0, 17.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_core_debug_outgoing_event_sizes, [handler, node, script_location, topic], [ping, manager, ./manager.zeek:32, /test/pings/2], [0.0, 0.0, 0.0, 4.0, 3.0, 18.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_core_incoming_events_total, [node], [manager], 100.0
+zeek, zeek_cluster_core_outgoing_events_total, [node], [manager], 50.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/0], 25.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/1], 25.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/2], 25.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/3], 25.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [ping, manager, /test/pings/0], 25.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [ping, manager, /test/pings/2], 25.0
+zeek, zeek_cluster_websocket_debug_incoming_event_sizes, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/0], [0.0, 1.0, 0.0, 3.0, 4.0, 17.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_websocket_debug_incoming_event_sizes, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/2], [0.0, 0.0, 0.0, 4.0, 3.0, 18.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_websocket_debug_outgoing_event_sizes, [app, handler, node, script_location, topic], [btest-python-client, ping, manager, none, /test/pings/0], [0.0, 1.0, 0.0, 3.0, 4.0, 17.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_websocket_debug_outgoing_event_sizes, [app, handler, node, script_location, topic], [btest-python-client, ping, manager, none, /test/pings/1], [0.0, 0.0, 1.0, 3.0, 4.0, 17.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_websocket_debug_outgoing_event_sizes, [app, handler, node, script_location, topic], [btest-python-client, ping, manager, none, /test/pings/2], [0.0, 0.0, 0.0, 4.0, 3.0, 18.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_websocket_debug_outgoing_event_sizes, [app, handler, node, script_location, topic], [btest-python-client, ping, manager, none, /test/pings/3], [0.0, 0.0, 0.0, 3.0, 4.0, 18.0, 0.0, 0.0, 0.0]
+zeek, zeek_cluster_websocket_incoming_events_total, [app, node], [btest-python-client, manager], 50.0
+zeek, zeek_cluster_websocket_outgoing_events_total, [app, node], [btest-python-client, manager], 100.0
+zeek, zeek_cluster_websocket_verbose_incoming_events_total, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/0], 25.0
+zeek, zeek_cluster_websocket_verbose_incoming_events_total, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/2], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/0], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/1], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/2], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [btest-python-client, ping, manager, /test/pings/3], 25.0
+zeek_cluster_* histogram metrics, 12
+zeek_cluster_* metrics, 16

testing/btest/Baseline/cluster.telemetry.ws/..manager..stderr

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+received termination signal

testing/btest/Baseline/cluster.telemetry.ws/..manager..stdout

@@ -0,0 +1,18 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+### NOTE: This file has been sorted with diff-sort.
+Cluster::websocket_client_added, [/test/pings, /zeek/wstest/ws1/]
+zeek, zeek_cluster_core_incoming_events_total, [node], [manager], 100.0
+zeek, zeek_cluster_core_outgoing_events_total, [node], [manager], 50.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/0], 25.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/1], 25.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/2], 25.0
+zeek, zeek_cluster_core_verbose_incoming_events_total, [handler, node, topic], [ping, manager, /test/pings/3], 25.0
+zeek, zeek_cluster_core_verbose_outgoing_events_total, [handler, node, topic], [ping, manager, /test/pings], 50.0
+zeek, zeek_cluster_websocket_incoming_events_total, [app, node], [unknown, manager], 50.0
+zeek, zeek_cluster_websocket_outgoing_events_total, [app, node], [unknown, manager], 100.0
+zeek, zeek_cluster_websocket_verbose_incoming_events_total, [app, handler, node, topic], [unknown, ping, manager, /test/pings], 50.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [unknown, ping, manager, /test/pings/0], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [unknown, ping, manager, /test/pings/1], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [unknown, ping, manager, /test/pings/2], 25.0
+zeek, zeek_cluster_websocket_verbose_outgoing_events_total, [app, handler, node, topic], [unknown, ping, manager, /test/pings/3], 25.0
+zeek_cluster_* metrics, 14

testing/btest/Files/ws/wstest.py

@@ -90,14 +90,18 @@ class TestClient:
         return self.__name
 
 
-def connect(name: str, url: Optional[str] = None) -> TestClient:
+def connect(
+    name: str,
+    url: Optional[str] = None,
+    additional_headers: Optional[dict[str, str]] = None,
+) -> TestClient:
     """
     Connect to a WebSocket server and return a TestClient instance.
     """
     if url is None:
         url = WS4_URL_V1
 
-    cc = websockets.sync.client.connect(url)
+    cc = websockets.sync.client.connect(url, additional_headers=additional_headers)
     return TestClient(name, cc)
 
 

testing/btest/cluster/telemetry/two-nodes.zeek

@@ -0,0 +1,68 @@
+# @TEST-DOC: All parties log their cluster metrics at zeek_done() time.
+#
+# @TEST-REQUIRES: have-zeromq
+#
+# @TEST-GROUP: cluster-zeromq
+#
+# @TEST-PORT: XPUB_PORT
+# @TEST-PORT: XSUB_PORT
+# @TEST-PORT: LOG_PULL_PORT
+#
+# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek
+# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
+#
+# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek >out"
+# @TEST-EXEC: btest-bg-run worker "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=worker-1 zeek -b ../worker.zeek >out"
+#
+# @TEST-EXEC: btest-bg-wait 30
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff ./manager/out
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff ./worker/out
+
+
+# @TEST-START-FILE common.zeek
+@load base/frameworks/telemetry
+
+@load ./zeromq-test-bootstrap
+
+redef Cluster::Telemetry::core_metrics += {
+	Cluster::Telemetry::VERBOSE,
+};
+
+redef Cluster::Telemetry::websocket_metrics += {
+	Cluster::Telemetry::VERBOSE,
+};
+
+global finish: event(name: string);
+
+event zeek_done()
+	{
+	local ms = Telemetry::collect_metrics("zeek", "cluster_core_*");
+	ms += Telemetry::collect_metrics("zeek", "cluster_websocket_*");
+	for ( _, m in ms )
+		print m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
+	}
+# @TEST-END-FILE
+
+# @TEST-START-FILE manager.zeek
+@load ./common.zeek
+# If a node comes up that isn't us, send it a finish event.
+event Cluster::node_up(name: string, id: string) {
+	Cluster::publish(Cluster::nodeid_topic(id), finish, Cluster::node);
+}
+
+# If the worker vanishes, finish the test.
+event Cluster::node_down(name: string, id: string) {
+	terminate();
+}
+# @TEST-END-FILE
+
+# @TEST-START-FILE worker.zeek
+@load ./common.zeek
+
+event Cluster::node_up(name: string, id: string) {
+}
+
+event finish(name: string) &is_used {
+	terminate();
+}
+# @TEST-END-FILE

testing/btest/cluster/telemetry/ws-app.zeek

@@ -0,0 +1,105 @@
+# @TEST-DOC: Output cluster telemetry after working with a WebSocket client. The WebSocket client sends an X-Application-Name header. Also include debug metrics as histograms in the output.
+#
+# @TEST-REQUIRES: have-zeromq
+# @TEST-REQUIRES: python3 -c 'import websockets.sync'
+#
+# @TEST-GROUP: cluster-zeromq
+#
+# @TEST-PORT: XPUB_PORT
+# @TEST-PORT: XSUB_PORT
+# @TEST-PORT: LOG_PULL_PORT
+# @TEST-PORT: WEBSOCKET_PORT
+#
+# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek
+# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
+# @TEST-EXEC: cp $FILES/ws/wstest.py .
+#
+# @TEST-EXEC: zeek -b --parse-only manager.zeek
+# @TEST-EXEC: python3 -m py_compile client.py
+#
+# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek"
+# @TEST-EXEC: python3 client.py
+#
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff ./manager/.stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff ./manager/.stderr
+
+# @TEST-START-FILE manager.zeek
+@load base/frameworks/telemetry
+
+@load ./zeromq-test-bootstrap
+
+redef Cluster::Telemetry::core_metrics += {
+	Cluster::Telemetry::VERBOSE,
+	Cluster::Telemetry::DEBUG,
+};
+
+redef Cluster::Telemetry::websocket_metrics += {
+	Cluster::Telemetry::VERBOSE,
+	Cluster::Telemetry::DEBUG,
+};
+
+redef exit_only_after_terminate = T;
+
+global expected_ping_count = 100;
+global ping_count = 0;
+
+global ping: event(msg: string, c: count) &is_used;
+
+event zeek_init()
+	{
+	Cluster::subscribe("/test/pings");
+	Cluster::listen_websocket([$listen_addr=127.0.0.1, $listen_port=to_port(getenv("WEBSOCKET_PORT"))]);
+	}
+
+event ping(msg: string, n: count) &is_used
+	{
+	if ( ping_count % 2 == 0)  # Reply every other ping.
+		{
+		Cluster::publish(fmt("/test/pings/%s", ping_count % 4), ping, msg, n);
+		}
+
+	++ping_count;
+
+	if ( ping_count == expected_ping_count )
+		terminate();
+	}
+
+event Cluster::websocket_client_added(info: Cluster::EndpointInfo, subscriptions: string_vec)
+	{
+	print "Cluster::websocket_client_added", subscriptions;
+	}
+
+event zeek_done()
+	{
+	local ms = Telemetry::collect_metrics("zeek", "cluster_core_*");
+	ms += Telemetry::collect_metrics("zeek", "cluster_websocket_*");
+	print "zeek_cluster_* metrics", |ms|;
+	for ( _, m in ms )
+		print m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
+
+	local hms = Telemetry::collect_histogram_metrics("zeek", "cluster_core_*");
+	hms += Telemetry::collect_histogram_metrics("zeek", "cluster_websocket_*");
+
+	print "zeek_cluster_* histogram metrics", |hms|;
+	for ( _, hm in hms )
+		print hm$opts$prefix, hm$opts$name, hm$label_names, hm$label_values, hm$values;
+	}
+# @TEST-END-FILE
+
+
+# @TEST-START-FILE client.py
+import wstest
+
+def run(ws_url):
+    with wstest.connect("ws1", ws_url, additional_headers={"X-Application-Name": "btest-python-client"}) as tc:
+        tc.hello_v1(["/test/pings"])
+        for i in range(0, 100):
+            msg = f"ping {i}" + (i * 32 * "A")
+            tc.send_json(wstest.build_event_v1(f"/test/pings/{i % 4}", "ping", [msg, i]))
+            if i % 2 == 0:  # Wait for a reply for every other ping
+                tc.recv_json()
+
+if __name__ == "__main__":
+    wstest.main(run, wstest.WS4_URL_V1)
+# @TEST-END-FILE

testing/btest/cluster/telemetry/ws.zeek

@@ -0,0 +1,93 @@
+# @TEST-DOC: Output cluster telemetry after working with a WebSocket client.
+#
+# @TEST-REQUIRES: have-zeromq
+# @TEST-REQUIRES: python3 -c 'import websockets.sync'
+#
+# @TEST-GROUP: cluster-zeromq
+#
+# @TEST-PORT: XPUB_PORT
+# @TEST-PORT: XSUB_PORT
+# @TEST-PORT: LOG_PULL_PORT
+# @TEST-PORT: WEBSOCKET_PORT
+#
+# @TEST-EXEC: cp $FILES/zeromq/cluster-layout-simple.zeek cluster-layout.zeek
+# @TEST-EXEC: cp $FILES/zeromq/test-bootstrap.zeek zeromq-test-bootstrap.zeek
+# @TEST-EXEC: cp $FILES/ws/wstest.py .
+#
+# @TEST-EXEC: zeek -b --parse-only manager.zeek
+# @TEST-EXEC: python3 -m py_compile client.py
+#
+# @TEST-EXEC: btest-bg-run manager "ZEEKPATH=$ZEEKPATH:.. && CLUSTER_NODE=manager zeek -b ../manager.zeek"
+# @TEST-EXEC: python3 client.py
+#
+# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff ./manager/.stdout
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff ./manager/.stderr
+
+# @TEST-START-FILE manager.zeek
+@load base/frameworks/telemetry
+
+@load ./zeromq-test-bootstrap
+
+redef Cluster::Telemetry::core_metrics += {
+	Cluster::Telemetry::VERBOSE,
+};
+
+redef Cluster::Telemetry::websocket_metrics += {
+	Cluster::Telemetry::VERBOSE,
+};
+
+redef exit_only_after_terminate = T;
+
+global expected_ping_count = 100;
+global ping_count = 0;
+
+global ping: event(msg: string, c: count) &is_used;
+
+event zeek_init()
+	{
+	Cluster::subscribe("/test/pings");
+	Cluster::listen_websocket([$listen_addr=127.0.0.1, $listen_port=to_port(getenv("WEBSOCKET_PORT"))]);
+	}
+
+event ping(msg: string, n: count) &is_used
+	{
+	if ( ping_count % 2 == 0)  # Reply every other ping.
+		Cluster::publish("/test/pings", ping, msg, n);
+
+	++ping_count;
+
+	if ( ping_count == expected_ping_count )
+		terminate();
+	}
+
+event Cluster::websocket_client_added(info: Cluster::EndpointInfo, subscriptions: string_vec)
+	{
+	print "Cluster::websocket_client_added", subscriptions;
+	}
+
+event zeek_done()
+	{
+	local ms = Telemetry::collect_metrics("zeek", "cluster_core_*");
+	ms += Telemetry::collect_metrics("zeek", "cluster_websocket_*");
+	print "zeek_cluster_* metrics", |ms|;
+	for ( _, m in ms )
+		print m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
+	}
+# @TEST-END-FILE
+
+
+# @TEST-START-FILE client.py
+import wstest
+
+def run(ws_url):
+    with wstest.connect("ws1", ws_url) as tc:
+        tc.hello_v1(["/test/pings"])
+        for i in range(0, 100):
+            tc.send_json(wstest.build_event_v1(f"/test/pings/{i % 4}", "ping", [f"ping {i}", i]))
+            if i % 2 == 0:  # Wait for a reply for every other ping
+                tc.recv_json()
+
+if __name__ == "__main__":
+    wstest.main(run, wstest.WS4_URL_V1)
+# @TEST-END-FILE