From 73192c0f6363fee0b01bf1390dd6e950bd0cdca5 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 29 Feb 2024 16:00:44 -0700 Subject: [PATCH] Add necessary script-land changes --- scripts/base/frameworks/broker/main.zeek | 26 +++-- scripts/base/frameworks/telemetry/main.zeek | 22 ++-- .../base/frameworks/telemetry/options.zeek | 109 ++++++++++++++++++ scripts/base/init-frameworks-and-bifs.zeek | 1 + .../frameworks/telemetry/prometheus.zeek | 14 +-- .../frameworks/telemetry/prometheus.zeek | 10 +- 6 files changed, 148 insertions(+), 34 deletions(-) create mode 100644 scripts/base/frameworks/telemetry/options.zeek diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index 63828e3ba7..5e449c5c8d 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -139,18 +139,18 @@ export { ## Broker to make metrics available to Prometheus scrapers via HTTP. Zeek ## overrides any value provided in zeek_init or earlier at startup if the ## environment variable BROKER_METRICS_PORT is defined. - const metrics_port = 0/unknown &redef; + const metrics_port = 0/unknown &redef &deprecated="Remove in 7.1. Use Telemetry::metrics_port."; ## Frequency for publishing scraped metrics to the target topic. Zeek ## overrides any value provided in zeek_init or earlier at startup if the ## environment variable BROKER_METRICS_EXPORT_INTERVAL is defined. - option metrics_export_interval = 1 sec; + option metrics_export_interval = 1 sec &deprecated="Remove in 7.1. Use Telemetry::metrics_export_interval"; ## Target topic for the metrics. Setting a non-empty string starts the ## periodic publishing of local metrics. Zeek overrides any value provided in ## zeek_init or earlier at startup if the environment variable ## BROKER_METRICS_EXPORT_TOPIC is defined. - option metrics_export_topic = ""; + option metrics_export_topic = "" &deprecated="Remove in 7.1. Use Telemetry::metrics_export_topic"; ## Topics for the Prometheus exporter for collecting metrics from other ## peers in the network and including them in the output. Has no effect when @@ -158,7 +158,7 @@ export { ## ## Zeek overrides any value provided in zeek_init or earlier at startup if ## the environment variable BROKER_METRICS_IMPORT_TOPICS is defined. - option metrics_import_topics: vector of string = vector(); + option metrics_import_topics: vector of string = vector() &deprecated="Remove in 7.1. Use Telemetry::metrics_import_topics"; ## ID for the metrics exporter. When setting a target topic for the ## exporter, Broker sets this option to the suffix of the new topic *unless* @@ -167,12 +167,12 @@ export { ## setting it at all if the topic suffix serves as a good-enough ID. Zeek ## overrides any value provided in zeek_init or earlier at startup if the ## environment variable BROKER_METRICS_ENDPOINT_NAME is defined. - option metrics_export_endpoint_name = ""; + option metrics_export_endpoint_name = "" &deprecated="Remove in 7.1. Use Telemetry::metrics_export_endpoint_name"; ## Selects prefixes from the local metrics. Only metrics with prefixes ## listed in this variable are included when publishing local metrics. ## Setting an empty vector selects *all* metrics. - option metrics_export_prefixes: vector of string = vector(); + option metrics_export_prefixes: vector of string = vector() &deprecated="Remove in 7.1. Use Telemetry::metrics_export_prefixes"; ## The default topic prefix where logs will be published. The log's stream ## id is appended when writing to a particular stream. @@ -458,31 +458,31 @@ event Broker::log_flush() &priority=10 schedule Broker::log_batch_interval { Broker::log_flush() }; } -function update_metrics_export_interval(id: string, val: interval): interval +function update_metrics_export_interval(id: string, val: interval): interval &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_interval." { Broker::__set_metrics_export_interval(val); return val; } -function update_metrics_export_topic(id: string, val: string): string +function update_metrics_export_topic(id: string, val: string): string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_topic." { Broker::__set_metrics_export_topic(val); return val; } -function update_metrics_import_topics(id: string, topics: vector of string): vector of string +function update_metrics_import_topics(id: string, topics: vector of string): vector of string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_import_topics." { Broker::__set_metrics_import_topics(topics); return topics; } -function update_metrics_export_endpoint_name(id: string, val: string): string +function update_metrics_export_endpoint_name(id: string, val: string): string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_endpoint_name." { Broker::__set_metrics_export_endpoint_name(val); return val; } -function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string +function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string &deprecated="Remove in v7.1. Use Telemetry::update_metrics_export_prefixes." { Broker::__set_metrics_export_prefixes(filter); return filter; @@ -491,6 +491,9 @@ function update_metrics_export_prefixes(id: string, filter: vector of string): v event zeek_init() { schedule Broker::log_batch_interval { Broker::log_flush() }; + + # Remove in v7.1. +@pragma push ignore-deprecations # interval update_metrics_export_interval("Broker::metrics_export_interval", Broker::metrics_export_interval); @@ -516,6 +519,7 @@ event zeek_init() Broker::metrics_export_prefixes); Option::set_change_handler("Broker::metrics_export_prefixes", update_metrics_export_prefixes); +@pragma pop } event retry_listen(a: string, p: port, retry: interval) diff --git a/scripts/base/frameworks/telemetry/main.zeek b/scripts/base/frameworks/telemetry/main.zeek index f6e22b0901..8e7a3dd938 100644 --- a/scripts/base/frameworks/telemetry/main.zeek +++ b/scripts/base/frameworks/telemetry/main.zeek @@ -6,6 +6,7 @@ ##! `BROKER_METRICS_PORT` environment variable. @load base/misc/version +@load base/frameworks/telemetry/options module Telemetry; @@ -21,7 +22,7 @@ export { ## The human-readable name of the metric. name: string; - ## The unit of the metric. Use the pseudo-unit "1" if this is a unit-less metric. + ## The unit of the metric. Set to a blank string if this is a unit-less metric. unit: string; ## Documentation for this metric. @@ -361,6 +362,8 @@ export { count_sum: count &optional; }; + type MetricVector : vector of Metric; + ## Collect all counter and gauge metrics matching the given *name* and *prefix*. ## ## For histogram metrics, use the :zeek:see:`Telemetry::collect_histogram_metrics`. @@ -406,7 +409,7 @@ function register_counter_family(opts: MetricOpts): CounterFamily global error_counter_cf = register_counter_family([ $prefix="zeek", $name="telemetry_counter_usage_error", - $unit="1", + $unit="", $help_text="This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero." ]); @@ -466,7 +469,7 @@ function register_gauge_family(opts: MetricOpts): GaugeFamily global error_gauge_cf = register_gauge_family([ $prefix="zeek", $name="telemetry_gauge_usage_error", - $unit="1", + $unit="", $help_text="This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero." ]); @@ -536,7 +539,7 @@ function register_histogram_family(opts: MetricOpts): HistogramFamily global error_histogram_hf = register_histogram_family([ $prefix="zeek", $name="telemetry_histogram_usage_error", - $unit="1", + $unit="", $help_text="This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero.", $bounds=vector(1.0) ]); @@ -559,7 +562,7 @@ function histogram_observe(h: Histogram, measurement: double): bool return Telemetry::__dbl_histogram_observe(h$__metric, measurement); } -function histogram_family_observe(hf: HistogramFamily, label_values: labels_vector, measurement: double): bool +function histogram_family_observe(hf: HistogramFamily, label_values: labels_vector, measurement: double): bool { return histogram_observe(histogram_with(hf, label_values), measurement); } @@ -580,16 +583,11 @@ event run_sync_hook() schedule sync_interval { run_sync_hook() }; } -event zeek_init() - { - schedule sync_interval { run_sync_hook() }; - } - # Expose the Zeek version as Prometheus style info metric global version_gauge_family = Telemetry::register_gauge_family([ $prefix="zeek", $name="version_info", - $unit="1", + $unit="", $help_text="The Zeek version", $labels=vector("version_number", "major", "minor", "patch", "commit", "beta", "debug","version_string") @@ -597,6 +595,8 @@ global version_gauge_family = Telemetry::register_gauge_family([ event zeek_init() { + schedule sync_interval { run_sync_hook() }; + local v = Version::info; local labels = vector(cat(v$version_number), cat(v$major), cat(v$minor), cat (v$patch), diff --git a/scripts/base/frameworks/telemetry/options.zeek b/scripts/base/frameworks/telemetry/options.zeek new file mode 100644 index 0000000000..e2a82eae92 --- /dev/null +++ b/scripts/base/frameworks/telemetry/options.zeek @@ -0,0 +1,109 @@ +module Telemetry; + +# This file contains the options for the Telemetry framework. These are kept +# separately so that they can be loaded in bare mode without loading all of +# the rest of the framework. This allows things like the plugins.hooks test +# to see the options without needing the rest. + +export { + ## Port used to make metric data available to Prometheus scrapers via + ## HTTP. Zeek overrides any value provided in zeek_init or earlier at + ## startup if the environment variable ZEEK_METRICS_PORT is defined. + const metrics_port = 0/unknown &redef; + + ## Frequency for publishing scraped metrics to the target topic. Zeek + ## overrides any value provided in zeek_init or earlier at startup if + ## the environment variable ZEEK_METRICS_EXPORT_INTERVAL is defined. + option metrics_export_interval = 1 sec; + + ## Target topic for the metrics. Setting a non-empty string starts the + ## periodic publishing of local metrics. Zeek overrides any value + ## provided in zeek_init or earlier at startup if the environment + ## variable ZEEK_METRICS_EXPORT_TOPIC is defined. + option metrics_export_topic = ""; + + ## Topics for the telmeetry framework for collecting metrics from other + ## peers in the network and including them in the output. Has no effect + ## when not exporting the metrics to Prometheus. + ## + ## Zeek overrides any value provided in zeek_init or earlier at startup + ## if the environment variable ZEEK_METRICS_IMPORT_TOPICS is defined. + option metrics_import_topics: vector of string = vector(); + + ## ID for the metrics exporter. When setting a target topic for the + ## exporter, Broker sets this option to the suffix of the new topic + ## *unless* the ID is a non-empty string. Since setting a topic starts + ## the periodic publishing of events, we recommend setting the ID always + ## first or avoid setting it at all if the topic suffix serves as a + ## good-enough ID. Zeek overrides any value provided in zeek_init or + ## earlier at startup if the environment variable + ## ZEEK_METRICS_ENDPOINT_NAME is defined. + option metrics_export_endpoint_name = ""; + + ## Selects prefixes from the local metrics. Only metrics with prefixes + ## listed in this variable are included when publishing local metrics. + ## Setting an empty vector selects *all* metrics. + option metrics_export_prefixes: vector of string = vector(); +} + +# Needed for the __set methods below +@load base/bif/telemetry.bif + +function update_metrics_export_interval(id: string, val: interval): interval + { + Telemetry::__set_metrics_export_interval(val); + return val; + } + +function update_metrics_export_topic(id: string, val: string): string + { + Telemetry::__set_metrics_export_topic(val); + return val; + } + +function update_metrics_import_topics(id: string, topics: vector of string): vector of string + { + Telemetry::__set_metrics_import_topics(topics); + return topics; + } + +function update_metrics_export_endpoint_name(id: string, val: string): string + { + Telemetry::__set_metrics_export_endpoint_name(val); + return val; + } + +function update_metrics_export_prefixes(id: string, filter: vector of string): vector of string + { + Telemetry::__set_metrics_export_prefixes(filter); + return filter; + } + +event zeek_init() + { + # interval + update_metrics_export_interval("Telemetry::metrics_export_interval", + Telemetry::metrics_export_interval); + Option::set_change_handler("Telemetry::metrics_export_interval", + update_metrics_export_interval); + # topic + update_metrics_export_topic("Telemetry::metrics_export_topic", + Telemetry::metrics_export_topic); + Option::set_change_handler("Telemetry::metrics_export_topic", + update_metrics_export_topic); + # import topics + update_metrics_import_topics("Telemetry::metrics_import_topics", + Telemetry::metrics_import_topics); + Option::set_change_handler("Telemetry::metrics_import_topics", + update_metrics_import_topics); + # endpoint name + update_metrics_export_endpoint_name("Telemetry::metrics_export_endpoint_name", + Telemetry::metrics_export_endpoint_name); + Option::set_change_handler("Telemetry::metrics_export_endpoint_name", + update_metrics_export_endpoint_name); + # prefixes + update_metrics_export_prefixes("Telemetry::metrics_export_prefixes", + Telemetry::metrics_export_prefixes); + Option::set_change_handler("Telemetry::metrics_export_prefixes", + update_metrics_export_prefixes); + } diff --git a/scripts/base/init-frameworks-and-bifs.zeek b/scripts/base/init-frameworks-and-bifs.zeek index 62330cf593..7a0588eb40 100644 --- a/scripts/base/init-frameworks-and-bifs.zeek +++ b/scripts/base/init-frameworks-and-bifs.zeek @@ -11,6 +11,7 @@ @load base/frameworks/config @load base/frameworks/analyzer @load base/frameworks/files +@load base/frameworks/telemetry/options @load base/bif diff --git a/scripts/policy/frameworks/telemetry/prometheus.zeek b/scripts/policy/frameworks/telemetry/prometheus.zeek index abc947670a..65ba8470bd 100644 --- a/scripts/policy/frameworks/telemetry/prometheus.zeek +++ b/scripts/policy/frameworks/telemetry/prometheus.zeek @@ -2,9 +2,9 @@ ##! Prometheus exposition and import all metrics from the ##! `zeek/cluster/metrics/...` topic. ##! -##! For customization or disabling, redef the involved Broker options again. -##! Specifically, to disable listening on port 9911, set -##! :zeek:see:`Broker::metrics_port` to `0/unknown` again. +##! For customization or disabling, redef the involved Telemetry options +##! again. Specifically, to disable listening on port 9911, set +##! :zeek:see:`Telemetry::metrics_port` to `0/unknown` again. ##! ##! Note that in large clusters, metrics import may cause significant ##! communication overhead as well as load on the manager. @@ -14,15 +14,15 @@ @if ( Cluster::is_enabled() ) # Use Cluster::node as "endpoint" label -redef Broker::metrics_export_endpoint_name = Cluster::node; +redef Telemetry::metrics_export_endpoint_name = Cluster::node; # The manager opens port 9911 and imports metrics from all nodes by default. @if ( Cluster::local_node_type() == Cluster::MANAGER ) -redef Broker::metrics_port = 9911/tcp; -redef Broker::metrics_import_topics = vector("zeek/cluster/metrics/"); +redef Telemetry::metrics_port = 9911/tcp; +redef Telemetry::metrics_import_topics = vector("zeek/cluster/metrics/"); @else -redef Broker::metrics_export_topic = "zeek/cluster/metrics/"; +redef Telemetry::metrics_export_topic = "zeek/cluster/metrics/"; @endif @endif diff --git a/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek b/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek index 1a0f9568ca..2494358ec2 100644 --- a/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek +++ b/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek @@ -36,7 +36,7 @@ redef Cluster::nodes = { # Query the Prometheus endpoint using ActiveHTTP for testing, oh my. event run_test() { - local url = fmt("http://localhost:%s/metrics", port_to_count(Broker::metrics_port)); + local url = fmt("http://localhost:%s/metrics", port_to_count(Telemetry::metrics_port)); when [url] ( local response = ActiveHTTP::request([$url=url]) ) { if ( response$code != 200 ) @@ -70,19 +70,19 @@ event run_test() @if ( Cluster::node == "manager-1" ) # Use a dynamic metrics port for testing to avoid colliding on 9911/tcp # when running tests in parallel. -global orig_metrics_port = Broker::metrics_port; -redef Broker::metrics_port = to_port(getenv("BROKER_TEST_METRICS_PORT")); +global orig_metrics_port = Telemetry::metrics_port; +redef Telemetry::metrics_port = to_port(getenv("BROKER_TEST_METRICS_PORT")); event zeek_init() { - print Cluster::node, "original Broker::metrics_port", orig_metrics_port; + print Cluster::node, "original Telemetry::metrics_port", orig_metrics_port; } event Cluster::Experimental::cluster_started() { # Run the test once all nodes are up and metrics_export_interval # has passed at least once. - schedule 2 * Broker::metrics_export_interval { run_test() }; + schedule 2 * Telemetry::metrics_export_interval { run_test() }; } @endif