Add necessary script-land changes

scripts/base/frameworks/broker/main.zeek

@@ -139,18 +139,18 @@ export {
 	## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek
 	## overrides any value provided in zeek_init or earlier at startup if the
 	## environment variable BROKER_METRICS_PORT is defined.
-	const metrics_port = 0/unknown &redef;
+	const metrics_port = 0/unknown &redef &deprecated="Remove in 7.1. Use Telemetry::metrics_port.";
 
 	## Frequency for publishing scraped metrics to the target topic. Zeek
 	## overrides any value provided in zeek_init or earlier at startup if the
 	## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined.
-	option metrics_export_interval = 1 sec;
+	option metrics_export_interval = 1 sec &deprecated="Remove in 7.1. Use Telemetry::metrics_export_interval";
 
 	## Target topic for the metrics. Setting a non-empty string starts the
 	## periodic publishing of local metrics. Zeek overrides any value provided in
 	## zeek_init or earlier at startup if the environment variable
 	## BROKER_METRICS_EXPORT_TOPIC is defined.
-	option metrics_export_topic = "";
+	option metrics_export_topic = "" &deprecated="Remove in 7.1. Use Telemetry::metrics_export_topic";
 
 	## Topics for the Prometheus exporter for collecting metrics from other
 	## peers in the network and including them in the output. Has no effect when
@@ -158,7 +158,7 @@ export {
 	##
 	## Zeek overrides any value provided in zeek_init or earlier at startup if
 	## the environment variable BROKER_METRICS_IMPORT_TOPICS is defined.
-	option metrics_import_topics: vector of string = vector();
+	option metrics_import_topics: vector of string = vector() &deprecated="Remove in 7.1. Use Telemetry::metrics_import_topics";
 
 	## ID for the metrics exporter. When setting a target topic for the
 	## exporter, Broker sets this option to the suffix of the new topic *unless*
@@ -167,12 +167,12 @@ export {
 	## setting it at all if the topic suffix serves as a good-enough ID. Zeek
 	## overrides any value provided in zeek_init or earlier at startup if the
 	## environment variable BROKER_METRICS_ENDPOINT_NAME is defined.
-	option metrics_export_endpoint_name = "";
+	option metrics_export_endpoint_name = "" &deprecated="Remove in 7.1. Use Telemetry::metrics_export_endpoint_name";
 
 	## Selects prefixes from the local metrics. Only metrics with prefixes
 	## listed in this variable are included when publishing local metrics.
 	## Setting an empty vector selects *all* metrics.
-	option metrics_export_prefixes: vector of string = vector();
+	option metrics_export_prefixes: vector of string = vector() &deprecated="Remove in 7.1. Use Telemetry::metrics_export_prefixes";
 
 	## The default topic prefix where logs will be published.  The log's stream
 	## id is appended when writing to a particular stream.
@@ -458,31 +458,31 @@ event Broker::log_flush() &priority=10
 	schedule Broker::log_batch_interval { Broker::log_flush() };
 	}
 
-function update_metrics_export_interval(id: string, val: interval): interval
+function update_metrics_export_interval(id: string, val: interval): interval &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_interval."
 	{
 	Broker::__set_metrics_export_interval(val);
 	return val;
 	}
 
-function update_metrics_export_topic(id: string, val: string): string
+function update_metrics_export_topic(id: string, val: string): string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_topic."
 	{
 	Broker::__set_metrics_export_topic(val);
 	return val;
 	}
 
-function update_metrics_import_topics(id: string, topics: vector of string): vector of string
+function update_metrics_import_topics(id: string, topics: vector of string): vector of string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_import_topics."
 	{
 	Broker::__set_metrics_import_topics(topics);
 	return topics;
 	}
 
-function update_metrics_export_endpoint_name(id: string, val: string): string
+function update_metrics_export_endpoint_name(id: string, val: string): string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_endpoint_name."
 	{
 	Broker::__set_metrics_export_endpoint_name(val);
 	return val;
 	}
 
-function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
+function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_prefixes."
 	{
 	Broker::__set_metrics_export_prefixes(filter);
 	return filter;
@@ -491,6 +491,9 @@ function update_metrics_export_prefixes(id: string, filter: vector of string): v
 event zeek_init()
 	{
 	schedule Broker::log_batch_interval { Broker::log_flush() };
+
+	# Remove in v7.1.
+@pragma push ignore-deprecations
 	# interval
 	update_metrics_export_interval("Broker::metrics_export_interval",
 	                               Broker::metrics_export_interval);
@@ -516,6 +519,7 @@ event zeek_init()
 	                               Broker::metrics_export_prefixes);
 	Option::set_change_handler("Broker::metrics_export_prefixes",
 	                           update_metrics_export_prefixes);
+@pragma pop
 	}
 
 event retry_listen(a: string, p: port, retry: interval)

scripts/base/frameworks/telemetry/main.zeek

@@ -6,6 +6,7 @@
 ##! `BROKER_METRICS_PORT` environment variable.
 
 @load base/misc/version
+@load base/frameworks/telemetry/options
 
 module Telemetry;
 
@@ -21,7 +22,7 @@ export {
 		## The human-readable name of the metric.
 		name: string;
 
-		## The unit of the metric. Use the pseudo-unit "1" if this is a unit-less metric.
+		## The unit of the metric. Set to a blank string if this is a unit-less metric.
 		unit: string;
 
 		## Documentation for this metric.
@@ -361,6 +362,8 @@ export {
 		count_sum: count &optional;
 	};
 
+	type MetricVector : vector of Metric;
+
 	## Collect all counter and gauge metrics matching the given *name* and *prefix*.
 	##
 	## For histogram metrics, use the :zeek:see:`Telemetry::collect_histogram_metrics`.
@@ -406,7 +409,7 @@ function register_counter_family(opts: MetricOpts): CounterFamily
 global error_counter_cf = register_counter_family([
 	$prefix="zeek",
 	$name="telemetry_counter_usage_error",
-	$unit="1",
+	$unit="",
 	$help_text="This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero."
 ]);
 
@@ -466,7 +469,7 @@ function register_gauge_family(opts: MetricOpts): GaugeFamily
 global error_gauge_cf = register_gauge_family([
 	$prefix="zeek",
 	$name="telemetry_gauge_usage_error",
-	$unit="1",
+	$unit="",
 	$help_text="This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero."
 ]);
 
@@ -536,7 +539,7 @@ function register_histogram_family(opts: MetricOpts): HistogramFamily
 global error_histogram_hf = register_histogram_family([
 	$prefix="zeek",
 	$name="telemetry_histogram_usage_error",
-	$unit="1",
+	$unit="",
 	$help_text="This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero.",
 	$bounds=vector(1.0)
 ]);
@@ -580,16 +583,11 @@ event run_sync_hook()
 	schedule sync_interval { run_sync_hook() };
 	}
 
-event zeek_init()
-	{
-	schedule sync_interval { run_sync_hook() };
-	}
-
 # Expose the Zeek version as Prometheus style info metric
 global version_gauge_family = Telemetry::register_gauge_family([
 	$prefix="zeek",
 	$name="version_info",
-	$unit="1",
+	$unit="",
 	$help_text="The Zeek version",
 	$labels=vector("version_number", "major", "minor", "patch", "commit",
                        "beta", "debug","version_string")
@@ -597,6 +595,8 @@ global version_gauge_family = Telemetry::register_gauge_family([
 
 event zeek_init()
 	{
+	schedule sync_interval { run_sync_hook() };
+
 	local v = Version::info;
 	local labels = vector(cat(v$version_number),
 	                      cat(v$major), cat(v$minor), cat (v$patch),

scripts/base/frameworks/telemetry/options.zeek

@@ -0,0 +1,109 @@
+module Telemetry;
+
+# This file contains the options for the Telemetry framework. These are kept
+# separately so that they can be loaded in bare mode without loading all of
+# the rest of the framework. This allows things like the plugins.hooks test
+# to see the options without needing the rest.
+
+export {
+	## Port used to make metric data available to Prometheus scrapers via
+	## HTTP.  Zeek overrides any value provided in zeek_init or earlier at
+	## startup if the environment variable ZEEK_METRICS_PORT is defined.
+	const metrics_port = 0/unknown &redef;
+
+	## Frequency for publishing scraped metrics to the target topic. Zeek
+	## overrides any value provided in zeek_init or earlier at startup if
+	## the environment variable ZEEK_METRICS_EXPORT_INTERVAL is defined.
+	option metrics_export_interval = 1 sec;
+
+	## Target topic for the metrics. Setting a non-empty string starts the
+	## periodic publishing of local metrics. Zeek overrides any value
+	## provided in zeek_init or earlier at startup if the environment
+	## variable ZEEK_METRICS_EXPORT_TOPIC is defined.
+	option metrics_export_topic = "";
+
+	## Topics for the telmeetry framework for collecting metrics from other
+	## peers in the network and including them in the output. Has no effect
+	## when not exporting the metrics to Prometheus.
+	##
+	## Zeek overrides any value provided in zeek_init or earlier at startup
+	## if the environment variable ZEEK_METRICS_IMPORT_TOPICS is defined.
+	option metrics_import_topics: vector of string = vector();
+
+	## ID for the metrics exporter. When setting a target topic for the
+	## exporter, Broker sets this option to the suffix of the new topic
+	## *unless* the ID is a non-empty string. Since setting a topic starts
+	## the periodic publishing of events, we recommend setting the ID always
+	## first or avoid setting it at all if the topic suffix serves as a
+	## good-enough ID. Zeek overrides any value provided in zeek_init or
+	## earlier at startup if the environment variable
+	## ZEEK_METRICS_ENDPOINT_NAME is defined.
+	option metrics_export_endpoint_name = "";
+
+	## Selects prefixes from the local metrics. Only metrics with prefixes
+	## listed in this variable are included when publishing local metrics.
+	## Setting an empty vector selects *all* metrics.
+	option metrics_export_prefixes: vector of string = vector();
+}
+
+# Needed for the __set methods below
+@load base/bif/telemetry.bif
+
+function update_metrics_export_interval(id: string, val: interval): interval
+	{
+	Telemetry::__set_metrics_export_interval(val);
+	return val;
+	}
+
+function update_metrics_export_topic(id: string, val: string): string
+	{
+	Telemetry::__set_metrics_export_topic(val);
+	return val;
+	}
+
+function update_metrics_import_topics(id: string, topics: vector of string): vector of string
+	{
+	Telemetry::__set_metrics_import_topics(topics);
+	return topics;
+	}
+
+function update_metrics_export_endpoint_name(id: string, val: string): string
+	{
+	Telemetry::__set_metrics_export_endpoint_name(val);
+	return val;
+	}
+
+function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string
+	{
+	Telemetry::__set_metrics_export_prefixes(filter);
+	return filter;
+	}
+
+event zeek_init()
+	{
+	# interval
+	update_metrics_export_interval("Telemetry::metrics_export_interval",
+	                               Telemetry::metrics_export_interval);
+	Option::set_change_handler("Telemetry::metrics_export_interval",
+	                           update_metrics_export_interval);
+	# topic
+	update_metrics_export_topic("Telemetry::metrics_export_topic",
+	                            Telemetry::metrics_export_topic);
+	Option::set_change_handler("Telemetry::metrics_export_topic",
+	                           update_metrics_export_topic);
+	# import topics
+	update_metrics_import_topics("Telemetry::metrics_import_topics",
+	                             Telemetry::metrics_import_topics);
+	Option::set_change_handler("Telemetry::metrics_import_topics",
+	                           update_metrics_import_topics);
+	# endpoint name
+	update_metrics_export_endpoint_name("Telemetry::metrics_export_endpoint_name",
+	                                    Telemetry::metrics_export_endpoint_name);
+	Option::set_change_handler("Telemetry::metrics_export_endpoint_name",
+	                           update_metrics_export_endpoint_name);
+	# prefixes
+	update_metrics_export_prefixes("Telemetry::metrics_export_prefixes",
+	                               Telemetry::metrics_export_prefixes);
+	Option::set_change_handler("Telemetry::metrics_export_prefixes",
+	                           update_metrics_export_prefixes);
+	}

scripts/base/init-frameworks-and-bifs.zeek

@@ -11,6 +11,7 @@
 @load base/frameworks/config
 @load base/frameworks/analyzer
 @load base/frameworks/files
+@load base/frameworks/telemetry/options
 
 @load base/bif
 

scripts/policy/frameworks/telemetry/prometheus.zeek

@@ -2,9 +2,9 @@
 ##! Prometheus exposition and import all metrics from the
 ##! `zeek/cluster/metrics/...` topic.
 ##!
-##! For customization or disabling, redef the involved Broker options again.
+##! For customization or disabling, redef the involved Telemetry options
-##! Specifically, to disable listening on port 9911, set
+##! again. Specifically, to disable listening on port 9911, set
-##! :zeek:see:`Broker::metrics_port` to `0/unknown` again.
+##! :zeek:see:`Telemetry::metrics_port` to `0/unknown` again.
 ##!
 ##! Note that in large clusters, metrics import may cause significant
 ##! communication overhead as well as load on the manager.
@@ -14,15 +14,15 @@
 @if ( Cluster::is_enabled() )
 
 # Use Cluster::node as "endpoint" label
-redef Broker::metrics_export_endpoint_name = Cluster::node;
+redef Telemetry::metrics_export_endpoint_name = Cluster::node;
 
 # The manager opens port 9911 and imports metrics from all nodes by default.
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
-redef Broker::metrics_port = 9911/tcp;
+redef Telemetry::metrics_port = 9911/tcp;
-redef Broker::metrics_import_topics = vector("zeek/cluster/metrics/");
+redef Telemetry::metrics_import_topics = vector("zeek/cluster/metrics/");
 
 @else
-redef Broker::metrics_export_topic = "zeek/cluster/metrics/";
+redef Telemetry::metrics_export_topic = "zeek/cluster/metrics/";
 @endif
 
 @endif

testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek

@@ -36,7 +36,7 @@ redef Cluster::nodes = {
 # Query the Prometheus endpoint using ActiveHTTP for testing, oh my.
 event run_test()
 	{
-	local url = fmt("http://localhost:%s/metrics", port_to_count(Broker::metrics_port));
+	local url = fmt("http://localhost:%s/metrics", port_to_count(Telemetry::metrics_port));
 	when [url] ( local response = ActiveHTTP::request([$url=url]) )
 		{
 		if  ( response$code != 200 )
@@ -70,19 +70,19 @@ event run_test()
 @if ( Cluster::node == "manager-1" )
 # Use a dynamic metrics port for testing to avoid colliding on 9911/tcp
 # when running tests in parallel.
-global orig_metrics_port = Broker::metrics_port;
+global orig_metrics_port = Telemetry::metrics_port;
-redef Broker::metrics_port = to_port(getenv("BROKER_TEST_METRICS_PORT"));
+redef Telemetry::metrics_port = to_port(getenv("BROKER_TEST_METRICS_PORT"));
 
 event zeek_init()
 	{
-	print Cluster::node, "original Broker::metrics_port", orig_metrics_port;
+	print Cluster::node, "original Telemetry::metrics_port", orig_metrics_port;
 	}
 
 event Cluster::Experimental::cluster_started()
 	{
 	# Run the test once all nodes are up and metrics_export_interval
 	# has passed at least once.
-	schedule 2 * Broker::metrics_export_interval { run_test() };
+	schedule 2 * Telemetry::metrics_export_interval { run_test() };
 	}
 @endif