Merge remote-tracking branch 'origin/topic/awelzel/3947-telemetry-hook-scrape'

* origin/topic/awelzel/3947-telemetry-hook-scrape:
  btest/telemetry: Fix "Note compilable" typo
  misc/stats: Add zeek_net_timestamp_seconds
  telemetry/Manager: Remove variant include
  telemetry: Invoke Telemetry::sync() only at scrape/collection time

testing/btest/scripts/base/frameworks/telemetry/basic.zeek

@@ -1,5 +1,5 @@
 # @TEST-DOC: Using and listing of counters and gauges using the telemetry module.
-# Note compilable to C++ due to globals being initialized to a record that
+# Not compilable to C++ due to globals being initialized to a record that
 # has an opaque type as a field.
 # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
 # @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out

testing/btest/scripts/base/frameworks/telemetry/conn-duration-histogram.zeek

@@ -1,4 +1,4 @@
-# Note compilable to C++ due to globals being initialized to a record that
+# Not compilable to C++ due to globals being initialized to a record that
 # has an opaque type as a field.
 # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
 # @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out

testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek

@@ -1,6 +1,6 @@
 # @TEST-DOC: Query for zeek event-handler-invocations metrics counting number of times handlers were called.
 
-# Note compilable to C++ due to globals being initialized to a record that
+# Not compilable to C++ due to globals being initialized to a record that
 # has an opaque type as a field.
 # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
 # @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out

testing/btest/scripts/base/frameworks/telemetry/internal-metrics.zeek

@@ -1,5 +1,5 @@
 # @TEST-DOC: Query Broker's telemetry to verify it ends up in Zeek's registry.
-# Note compilable to C++ due to globals being initialized to a record that
+# Not compilable to C++ due to globals being initialized to a record that
 # has an opaque type as a field.
 # @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
 # @TEST-EXEC: zcat <$TRACES/echo-connections.pcap.gz | zeek -b -Cr - %INPUT > out

testing/btest/scripts/base/frameworks/telemetry/sync-collect.zeek

@@ -0,0 +1,48 @@
+# @TEST-DOC: Calling collect_metrics() invokes Telemetry::sync.
+# Not compilable to C++ due to globals being initialized to a record that
+# has an opaque type as a field.
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+#
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+
+@load base/frameworks/telemetry
+
+global connections_by_proto_cf = Telemetry::register_counter_family([
+	$prefix="btest",
+	$name="connections",
+	$unit="",
+	$help_text="Total number of monitored connections",
+	$label_names=vector("proto")
+]);
+
+function print_metrics(ms: vector of Telemetry::Metric) {
+	for (_, m in ms) {
+		print m$opts$name, m$label_values, m$value;
+	}
+}
+
+event zeek_init()
+	{
+	print "node up";
+	local ms = Telemetry::collect_metrics("btest");
+	print_metrics(ms);
+	ms = Telemetry::collect_metrics("btest");
+	print_metrics(ms);
+	ms = Telemetry::collect_metrics("btest");
+	print_metrics(ms);
+	local hm = Telemetry::collect_histogram_metrics("btest");
+	print_metrics(ms);
+	}
+
+
+global sync_calls = 0;
+
+hook Telemetry::sync()
+	{
+	++sync_calls;
+	local proto = sync_calls == 1 ? "tcp" : "udp";
+	print "sync", sync_calls, proto;
+	Telemetry::counter_family_inc(connections_by_proto_cf, vector(proto));
+	}

testing/btest/scripts/base/frameworks/telemetry/sync-warnings.zeek

@@ -0,0 +1,43 @@
+# @TEST-DOC: Breaking and recursive Telemetry::sync() warning
+# Not compilable to C++ due to globals being initialized to a record that
+# has an opaque type as a field.
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+#
+# @TEST-EXEC: zeek -b %INPUT >out
+# @TEST-EXEC: btest-diff out
+# @TEST-EXEC: btest-diff .stderr
+
+
+@load base/frameworks/telemetry
+
+global connections_by_proto_cf = Telemetry::register_counter_family([
+	$prefix="btest",
+	$name="connections",
+	$unit="",
+	$help_text="Total number of monitored connections",
+	$label_names=vector("proto")
+]);
+
+event zeek_init()
+	{
+	print "node up";
+	Telemetry::counter_family_inc(connections_by_proto_cf, vector("tcp"));
+	local ms = Telemetry::collect_metrics("btest");
+	}
+
+
+hook Telemetry::sync()
+	{
+	# Calling collect_metrics() in Telemetry::sync() is not good as
+	# it would invoke Telemetry::sync() recursively. The manager will
+	# emit a warning and not run the second Telemetry::sync() invocation.
+	local ms = Telemetry::collect_metrics("btest");
+	}
+
+hook Telemetry::sync() &priority=-100
+	{
+	# break is not good as it prevents other Telemetry::sync() hooks
+	# from running. This will produce a warning.
+	# We could find this via script validation if we wanted to.
+	break;
+	}

testing/btest/scripts/base/frameworks/telemetry/sync.zeek

@@ -0,0 +1,68 @@
+# @TEST-DOC: Verify Telemetry::sync() is invoked for metric scraping via the Prometheus HTTP endpoint.
+# Not compilable to C++ due to globals being initialized to a record that
+# has an opaque type as a field.
+# @TEST-REQUIRES: test "${ZEEK_USE_CPP}" != "1"
+# @TEST-REQUIRES: which jq
+# @TEST-REQUIRES: which curl
+#
+# @TEST-PORT: METRICS_PORT
+#
+# @TEST-EXEC: chmod +x fetch-metrics.sh
+# @TEST-EXEC: zeek --parse-only %INPUT
+# @TEST-EXEC: btest-bg-run zeek ZEEKPATH=$ZEEKPATH:.. zeek -b %INPUT
+# @TEST-EXEC: $SCRIPTS/wait-for-file zeek/up 5 || (btest-bg-wait -k 1 && false)
+# @TEST-EXEC: ./fetch-metrics.sh 1.trace metrics1.txt
+# @TEST-EXEC: ./fetch-metrics.sh 2.trace metrics2.txt
+# @TEST-EXEC: ./fetch-metrics.sh 3.trace metrics3.txt
+# @TEST-EXEC: btest-bg-wait 10
+#
+# @TEST-EXEC: btest-diff zeek/.stdout
+# @TEST-EXEC: btest-diff metrics1.txt
+# @TEST-EXEC: btest-diff metrics2.txt
+# @TEST-EXEC: btest-diff metrics3.txt
+
+@TEST-START-FILE fetch-metrics.sh
+#! /usr/bin/env bash
+set -ux
+trace_file=$1
+output_file=$2
+
+PORT=$(echo ${METRICS_PORT} | cut -d '/' -f 1)
+URL=http://localhost:${PORT}/metrics
+
+curl -m 5 --trace $trace_file $URL | grep ^btest > $output_file
+
+exit 0
+@TEST-END-FILE
+
+@load base/frameworks/telemetry
+
+redef exit_only_after_terminate = T;
+redef Telemetry::metrics_port = to_port(getenv("METRICS_PORT"));
+
+event zeek_init()
+	{
+	print "node up";
+	system("touch up");
+	}
+
+global connections_by_proto_cf = Telemetry::register_counter_family([
+	$prefix="btest",
+	$name="connections",
+	$unit="",
+	$help_text="Total number of monitored connections",
+	$label_names=vector("proto")
+]);
+
+global sync_calls = 0;
+
+hook Telemetry::sync()
+	{
+	++sync_calls;
+	local proto = sync_calls == 1 ? "tcp" : "udp";
+	print "sync", sync_calls, proto;
+	Telemetry::counter_family_inc(connections_by_proto_cf, vector(proto));
+
+	if ( sync_calls == 3 )
+		terminate();
+	}