mirror of
https://github.com/zeek/zeek.git
synced 2025-10-02 14:48:21 +00:00
Introduce telemetry framework
Adds base/frameworks/telemetry with wrappers around telemetry.bif and updates telemetry/Manager to support collecting metrics from script land. Add policy/frameworks/telemetry/log for logging of metrics data into a new telemetry.log and telemetry_histogram.log and add into local.zeek by default.
This commit is contained in:
Arne Welzel
parent
95fba8fd29
commit
3fe930dbf2
32 changed files with 1950 additions and 27 deletions
1
scripts/base/frameworks/telemetry/__load__.zeek
Normal file
1
scripts/base/frameworks/telemetry/__load__.zeek
Normal file
|
|
@ -0,0 +1 @@
|
|||
@load ./main
|
||||
610
scripts/base/frameworks/telemetry/main.zeek
Normal file
610
scripts/base/frameworks/telemetry/main.zeek
Normal file
|
|
@ -0,0 +1,610 @@
|
|||
##! Module for recording and querying metrics. This modules wraps
|
||||
##! the lower-level telemetry.bif functions.
|
||||
##!
|
||||
##! Metrics will be exposed through a Prometheus HTTP endpoint when
|
||||
##! enabled by setting :zeek:see:`Broker::metrics_port` or using the
|
||||
##! `BROKER_METRICS_PORT` environment variable.
|
||||
|
||||
@load base/misc/version
|
||||
|
||||
module Telemetry;
|
||||
|
||||
export {
|
||||
## Alias for a vector of label values.
|
||||
type labels_vector: vector of string;
|
||||
|
||||
## Type that captures options used to create metrics.
|
||||
type MetricOpts: record {
|
||||
## The prefix (namespace) of the metric.
|
||||
prefix: string;
|
||||
|
||||
## The human-readable name of the metric.
|
||||
name: string;
|
||||
|
||||
## The unit of the metric. Use the pseudo-unit "1" if this is a unit-less metric.
|
||||
unit: string;
|
||||
|
||||
## Documentation for this metric.
|
||||
help_text: string;
|
||||
|
||||
## The label names (also called dimensions) of the metric. When
|
||||
## instantiating or working with concrete metrics, corresponding
|
||||
## label values have to be provided.
|
||||
labels: vector of string &default=vector();
|
||||
|
||||
## Whether the metric represents something that is accumulating.
|
||||
## Defaults to ``T`` for counters and ``F`` for gauges and
|
||||
## histograms.
|
||||
is_total: bool &optional;
|
||||
|
||||
## When creating a :zeek:see:`Telemetry::HistogramFamily`,
|
||||
## describes the number and bounds of the individual buckets.
|
||||
bounds: vector of double &optional;
|
||||
|
||||
## The same meaning as *bounds*, but as :zeek:type:`count`.
|
||||
## Only set in the return value of
|
||||
## :zeek:see:`Telemetry::collect_histogram_metrics`.
|
||||
## for histograms when the underlying type is ``int64_t``,
|
||||
## otherwise ignored.
|
||||
count_bounds: vector of count &optional;
|
||||
|
||||
## Describes the underlying metric type.
|
||||
## Only set in the return value of
|
||||
## :zeek:see:`Telemetry::collect_metrics` or
|
||||
## :zeek:see:`Telemetry::collect_histogram_metrics`,
|
||||
## otherwise ignored.
|
||||
metric_type: MetricType &optional;
|
||||
};
|
||||
|
||||
## Type representing a family of counters with uninitialized label values.
|
||||
##
|
||||
## To create concrete :zeek:see:`Telemetry::Counter` instances, use
|
||||
## :zeek:see:`Telemetry::counter_with`. To modify counters directly
|
||||
## use :zeek:see:`Telemetry::counter_family_inc`.
|
||||
type CounterFamily: record {
|
||||
__family: opaque of dbl_counter_metric_family;
|
||||
__labels: vector of string;
|
||||
};
|
||||
|
||||
## Type representing a counter metric with initialized label values.
|
||||
##
|
||||
## Counter metrics only ever go up and reset when the process
|
||||
## restarts. Use :zeek:see:`Telemetry::counter_inc` or
|
||||
## :zeek:see:`Telemetry::counter_set` to modify counters.
|
||||
## An example for a counter is the number of log writes
|
||||
## per :zeek:see:`Log::Stream` or number connections broken down
|
||||
## by protocol and service.
|
||||
type Counter: record {
|
||||
__metric: opaque of dbl_counter_metric;
|
||||
};
|
||||
|
||||
## Register a counter family.
|
||||
global register_counter_family: function(opts: MetricOpts): CounterFamily;
|
||||
|
||||
## Get a :zeek:see:`Telemetry::Counter` instance given family and label values.
|
||||
global counter_with: function(cf: CounterFamily,
|
||||
label_values: labels_vector &default=vector()): Counter;
|
||||
|
||||
## Increment a :zeek:see:`Telemetry::Counter` by `amount`.
|
||||
## Using a negative `amount` is an error.
|
||||
##
|
||||
## c: The counter instance.
|
||||
##
|
||||
## amount: The amount by which to increment the counter.
|
||||
##
|
||||
## Returns: True if the counter was incremented successfully.
|
||||
global counter_inc: function(c: Counter, amount: double &default=1.0): bool;
|
||||
|
||||
## Helper to set a :zeek:see:`Telemetry::Counter` to the given `value`.
|
||||
## This can be useful for mirroring counter metrics in an
|
||||
## :zeek:see:`Telemetry::sync` hook implementation.
|
||||
## Setting a value that is less than the current value of the
|
||||
## metric is an error and will be ignored.
|
||||
##
|
||||
## c: The counter instance.
|
||||
##
|
||||
## value: The value to set the counter to.
|
||||
##
|
||||
## Returns: True if the counter value was set successfully.
|
||||
global counter_set: function(c: Counter, value: double): bool;
|
||||
|
||||
## Increment a :zeek:see:`Telemetry::Counter` through the :zeek:see:`Telemetry::CounterFamily`.
|
||||
## This is a short-cut for :zeek:see:`Telemetry::counter_inc`.
|
||||
## Using a negative amount is an error.
|
||||
##
|
||||
## cf: The counter family to use.
|
||||
##
|
||||
## label_values: The label values to use for the counter.
|
||||
##
|
||||
## amount: The amount by which to increment the counter.
|
||||
##
|
||||
## Returns: True if the counter was incremented successfully.
|
||||
global counter_family_inc: function(cf: CounterFamily,
|
||||
label_values: labels_vector &default=vector(),
|
||||
amount: double &default=1.0): bool;
|
||||
|
||||
## Set a :zeek:see:`Telemetry::Counter` through the :zeek:see:`Telemetry::CounterFamily`.
|
||||
## This is a short-cut for :zeek:see:`Telemetry::counter_set`.
|
||||
## Setting a value that is less than the current value of the
|
||||
## metric is an error and will be ignored.
|
||||
##
|
||||
## cf: The counter family to use.
|
||||
##
|
||||
## label_values: The label values to use for the counter.
|
||||
##
|
||||
## value: The value to set the counter to.
|
||||
##
|
||||
## Returns: True if the counter value was set successfully.
|
||||
global counter_family_set: function(cf: CounterFamily,
|
||||
label_values: labels_vector,
|
||||
value: double): bool;
|
||||
|
||||
## Type representing a family of gauges with uninitialized label values.
|
||||
##
|
||||
## Create concrete :zeek:see:`Telemetry::Gauge` instances with
|
||||
## :zeek:see:`Telemetry::gauge_with`, or use
|
||||
## :zeek:see:`Telemetry::gauge_family_inc` or
|
||||
## :zeek:see:`Telemetry::gauge_family_set` directly.
|
||||
type GaugeFamily: record {
|
||||
__family: opaque of dbl_gauge_metric_family;
|
||||
__labels: vector of string;
|
||||
};
|
||||
|
||||
## Type representing a gauge metric with initialized label values.
|
||||
##
|
||||
## Use :zeek:see:`Telemetry::gauge_inc`, :zeek:see:`Telemetry::gauge_dec`,
|
||||
## or :zeek:see:`Telemetry::gauge_set` to modify the gauge.
|
||||
## Example for gauges are process memory usage, table sizes
|
||||
## or footprints of long-lived values as determined by
|
||||
## :zeek:see:`val_footprint`.
|
||||
type Gauge: record {
|
||||
__metric: opaque of dbl_gauge_metric;
|
||||
};
|
||||
|
||||
## Register a gauge family.
|
||||
global register_gauge_family: function(opts: MetricOpts): GaugeFamily;
|
||||
|
||||
|
||||
## Get a :zeek:see:`Telemetry::Gauge` instance given family and label values.
|
||||
global gauge_with: function(gf: GaugeFamily,
|
||||
label_values: labels_vector &default=vector()): Gauge;
|
||||
|
||||
## Increment a :zeek:see:`Telemetry::Gauge` by `amount`.
|
||||
##
|
||||
## g: The gauge instance.
|
||||
##
|
||||
## amount: The amount by which to increment the gauge.
|
||||
##
|
||||
## Returns: True if the gauge was incremented successfully.
|
||||
global gauge_inc: function(g: Gauge, amount: double &default=1.0): bool;
|
||||
|
||||
## Decrement a :zeek:see:`Telemetry::Gauge` by `amount`.
|
||||
##
|
||||
## g: The gauge instance.
|
||||
##
|
||||
## amount: The amount by which to decrement the gauge.
|
||||
##
|
||||
## Returns: True if the gauge was incremented successfully.
|
||||
global gauge_dec: function(g: Gauge, amount: double &default=1.0): bool;
|
||||
|
||||
## Helper to set a :zeek:see:`Telemetry::Gauge` to the given `value`.
|
||||
##
|
||||
## g: The gauge instance.
|
||||
##
|
||||
## value: The value to set the gauge to.
|
||||
##
|
||||
## Returns: True if the gauge value was set successfully.
|
||||
global gauge_set: function(g: Gauge, value: double): bool;
|
||||
|
||||
## Increment a :zeek:see:`Telemetry::Gauge` by the given `amount` through
|
||||
## the :zeek:see:`Telemetry::GaugeFamily`.
|
||||
## This is a short-cut for :zeek:see:`Telemetry::gauge_inc`.
|
||||
## Using a negative amount is an error.
|
||||
##
|
||||
## gf: The gauge family to use.
|
||||
##
|
||||
## label_values: The label values to use for the gauge.
|
||||
##
|
||||
## amount: The amount by which to increment the gauge.
|
||||
##
|
||||
## Returns: True if the gauge was incremented successfully.
|
||||
global gauge_family_inc: function(gf: GaugeFamily,
|
||||
label_values: labels_vector &default=vector(),
|
||||
amount: double &default=1.0): bool;
|
||||
|
||||
## Decrement a :zeek:see:`Telemetry::Gauge` by the given `amount` through
|
||||
## the :zeek:see:`Telemetry::GaugeFamily`.
|
||||
## This is a short-cut for :zeek:see:`Telemetry::gauge_dec`.
|
||||
##
|
||||
## gf: The gauge family to use.
|
||||
##
|
||||
## label_values: The label values to use for the gauge.
|
||||
##
|
||||
## amount: The amount by which to increment the gauge.
|
||||
##
|
||||
## Returns: True if the gauge was incremented successfully.
|
||||
global gauge_family_dec: function(gf: GaugeFamily,
|
||||
label_values: labels_vector &default=vector(),
|
||||
amount: double &default=1.0): bool;
|
||||
|
||||
## Set a :zeek:see:`Telemetry::Gauge` to the given `value` through
|
||||
## the :zeek:see:`Telemetry::GaugeFamily`.
|
||||
## This is a short-cut for :zeek:see:`Telemetry::gauge_set`.
|
||||
##
|
||||
## gf: The gauge family to use.
|
||||
##
|
||||
## label_values: The label values to use for the gauge.
|
||||
##
|
||||
## value: The value to set the gauge to.
|
||||
##
|
||||
## Returns: True if the gauge value was set successfully.
|
||||
global gauge_family_set: function(g: GaugeFamily,
|
||||
label_values: labels_vector,
|
||||
value: double): bool;
|
||||
|
||||
## Type representing a family of histograms with uninitialized label values.
|
||||
## Create concrete :zeek:see:`Telemetry::Histogram` instances with
|
||||
## :zeek:see:`Telemetry::histogram_with` or use
|
||||
## :zeek:see:`Telemetry::histogram_family_observe` directly.
|
||||
type HistogramFamily: record {
|
||||
__family: opaque of dbl_histogram_metric_family;
|
||||
__labels: vector of string;
|
||||
};
|
||||
|
||||
## Type representing a histogram metric with initialized label values.
|
||||
## Use :zeek:see:`Telemetry::histogram_observe` to make observations.
|
||||
type Histogram: record {
|
||||
__metric: opaque of dbl_histogram_metric;
|
||||
};
|
||||
|
||||
## Register a histogram family.
|
||||
global register_histogram_family: function(opts: MetricOpts): HistogramFamily;
|
||||
|
||||
## Get a :zeek:see:`Telemetry::Histogram` instance given family and label values.
|
||||
global histogram_with: function(hf: HistogramFamily,
|
||||
label_values: labels_vector &default=vector()): Histogram;
|
||||
|
||||
## Observe a measurement for a :zeek:see:`Telemetry::Histogram`.
|
||||
##
|
||||
## h: The histogram instance.
|
||||
##
|
||||
## measurement: The value for this observations.
|
||||
##
|
||||
## Returns: True if measurement was observed successfully.
|
||||
global histogram_observe: function(h: Histogram, measurement: double): bool;
|
||||
|
||||
## Observe a measurement for a :zeek:see:`Telemetry::Histogram` through
|
||||
## the :zeek:see:`Telemetry::HistogramFamily`.
|
||||
## This is a short-cut for :zeek:see:`Telemetry::histogram_observe`.
|
||||
##
|
||||
## hf: The histogram family to use.
|
||||
##
|
||||
## label_values: The label values to use for the histogram.
|
||||
##
|
||||
## measurement: The value for this observations.
|
||||
##
|
||||
## Returns: True if measurement was observed successfully.
|
||||
global histogram_family_observe: function(hf: HistogramFamily,
|
||||
label_values: labels_vector,
|
||||
measurement: double): bool;
|
||||
|
||||
## Telemetry sync hook.
|
||||
##
|
||||
## This hook is invoked every :zeek:see:`Telemetry::sync_interval`
|
||||
## for script writers to synchronize or mirror metrics with the
|
||||
## telemetry subsystem. For example, when tracking table or value
|
||||
## footprints with gauges, the value in question can be set on an actual
|
||||
## :zeek:see:`Telemetry::Gauge` instance during execution of this hook.
|
||||
##
|
||||
## Implementations should be lightweight, this hook may be called
|
||||
## multiple times per minute. The interval can increased by changing
|
||||
## :zeek:see:`Telemetry::sync_interval` at the cost of delaying
|
||||
## metric updates and thereby reducing granularity.
|
||||
global sync: hook();
|
||||
|
||||
## Interval at which the :zeek:see:`Telemetry::sync` hook is invoked.
|
||||
option sync_interval = 10sec;
|
||||
|
||||
## Type of elements returned by the :zeek:see:`Telemetry::collect_metrics` function.
|
||||
type Metric: record {
|
||||
## A :zeek:see:`Telemetry::MetricOpts` record describing this metric.
|
||||
opts: MetricOpts;
|
||||
|
||||
## The label values associated with this metric, if any.
|
||||
labels: vector of string;
|
||||
|
||||
## The value of gauge or counter cast to a double
|
||||
## independent of the underlying data type.
|
||||
## This value is set for all counter and gauge metrics,
|
||||
## it is unset for histograms.
|
||||
value: double &optional;
|
||||
|
||||
## The value of the underlying gauge or counter as a double
|
||||
## if the underlying metric type uses ``int64_t``.
|
||||
## Only counters and gauges created with the C++ API may
|
||||
## have this value set.
|
||||
count_value: count &optional;
|
||||
};
|
||||
|
||||
## Type of elements returned by the :zeek:see:`Telemetry::collect_histogram_metrics` function.
|
||||
type HistogramMetric: record {
|
||||
## A :zeek:see:`Telemetry::MetricOpts` record describing this histogram.
|
||||
opts: MetricOpts;
|
||||
|
||||
## The label values associated with this histogram, if any.
|
||||
labels: vector of string;
|
||||
|
||||
## Individual counters for each of the buckets as
|
||||
## described by the *bounds* field in *opts*;
|
||||
values: vector of double;
|
||||
|
||||
## If the underlying data type of the histogram is ``int64_t``,
|
||||
## this vector will hold the values as counts, otherwise it
|
||||
## is unset. Only histograms created with the C++ API have
|
||||
## may have this value set.
|
||||
count_values: vector of count &optional;
|
||||
|
||||
## The number of observations made for this histogram.
|
||||
observations: double;
|
||||
|
||||
## The sum of all observations for this histogram.
|
||||
sum: double;
|
||||
|
||||
## If the underlying data type of the histogram is ``int64_t``,
|
||||
## the number of observations as :zeek:type:`count`, otherwise
|
||||
## unset.
|
||||
count_observations: count &optional;
|
||||
|
||||
## If the underlying data type of the histogram is ``int64_t``,
|
||||
## the sum of all observations as :zeek:type:`count`, otherwise
|
||||
## unset.
|
||||
count_sum: count &optional;
|
||||
};
|
||||
|
||||
## Collect all counter and gauge metrics matching the given *name* and *prefix*.
|
||||
##
|
||||
## For histogram metrics, use the :zeek:see:`Telemetry::collect_histogram_metrics`.
|
||||
##
|
||||
## The *prefix* and *name* parameters support globbing. By default,
|
||||
## all counters and gauges are returned.
|
||||
global collect_metrics: function(prefix: string &default="*",
|
||||
name: string &default="*"): vector of Metric;
|
||||
|
||||
## Collect all histograms and their observations matching the given
|
||||
## *prefix* and *name*.
|
||||
##
|
||||
## The *prefix* and *name* parameters support globbing. By default,
|
||||
## all histogram metrics are returned.
|
||||
global collect_histogram_metrics: function(prefix: string &default="*",
|
||||
name: string &default="*"): vector of HistogramMetric;
|
||||
}
|
||||
|
||||
|
||||
## Internal helper to create the labels table.
|
||||
function make_labels(keys: vector of string, values: labels_vector): table[string] of string
|
||||
{
|
||||
local labels: table[string] of string;
|
||||
for ( i in keys )
|
||||
labels[keys[i]] = values[i];
|
||||
|
||||
return labels;
|
||||
}
|
||||
|
||||
function register_counter_family(opts: MetricOpts): CounterFamily
|
||||
{
|
||||
local f = Telemetry::__dbl_counter_family(
|
||||
opts$prefix,
|
||||
opts$name,
|
||||
opts$labels,
|
||||
opts$help_text,
|
||||
opts$unit,
|
||||
opts?$is_total ? opts$is_total : T
|
||||
);
|
||||
return CounterFamily($__family=f, $__labels=opts$labels);
|
||||
}
|
||||
|
||||
# Fallback Counter returned when there are issues with the labels.
|
||||
global error_counter_cf = register_counter_family([
|
||||
$prefix="zeek",
|
||||
$name="telemetry_counter_usage_error",
|
||||
$unit="1",
|
||||
$help_text="This counter is returned when label usage for counters is wrong. Check reporter.log if non-zero."
|
||||
]);
|
||||
|
||||
function counter_with(cf: CounterFamily, label_values: labels_vector): Counter
|
||||
{
|
||||
if ( |cf$__labels| != |label_values| )
|
||||
{
|
||||
Reporter::error(fmt("Invalid label values expected %s, have %s", |cf$__labels|, |label_values|));
|
||||
return counter_with(error_counter_cf);
|
||||
}
|
||||
|
||||
local labels = make_labels(cf$__labels, label_values);
|
||||
local m = Telemetry::__dbl_counter_metric_get_or_add(cf$__family, labels);
|
||||
return Counter($__metric=m);
|
||||
}
|
||||
|
||||
function counter_inc(c: Counter, amount: double): bool
|
||||
{
|
||||
return Telemetry::__dbl_counter_inc(c$__metric, amount);
|
||||
}
|
||||
|
||||
function counter_set(c: Counter, value: double): bool
|
||||
{
|
||||
local cur_value: double = Telemetry::__dbl_counter_value(c$__metric);
|
||||
if (value < cur_value)
|
||||
{
|
||||
Reporter::error(fmt("Attempted to set lower counter value=%s cur_value=%s", value, cur_value));
|
||||
return F;
|
||||
}
|
||||
return Telemetry::__dbl_counter_inc(c$__metric, value - cur_value);
|
||||
}
|
||||
|
||||
function counter_family_inc(cf: CounterFamily, label_values: labels_vector, amount: double): bool
|
||||
{
|
||||
return counter_inc(counter_with(cf, label_values), amount);
|
||||
}
|
||||
|
||||
function counter_family_set(cf: CounterFamily, label_values: labels_vector, value: double): bool
|
||||
{
|
||||
return counter_set(counter_with(cf, label_values), value);
|
||||
}
|
||||
|
||||
function register_gauge_family(opts: MetricOpts): GaugeFamily
|
||||
{
|
||||
local f = Telemetry::__dbl_gauge_family(
|
||||
opts$prefix,
|
||||
opts$name,
|
||||
opts$labels,
|
||||
opts$help_text,
|
||||
opts$unit,
|
||||
opts?$is_total ? opts$is_total : F
|
||||
);
|
||||
return GaugeFamily($__family=f, $__labels=opts$labels);
|
||||
}
|
||||
|
||||
# Fallback Gauge returned when there are issues with the label usage.
|
||||
global error_gauge_cf = register_gauge_family([
|
||||
$prefix="zeek",
|
||||
$name="telemetry_gauge_usage_error",
|
||||
$unit="1",
|
||||
$help_text="This gauge is returned when label usage for gauges is wrong. Check reporter.log if non-zero."
|
||||
]);
|
||||
|
||||
function gauge_with(gf: GaugeFamily, label_values: labels_vector): Gauge
|
||||
{
|
||||
if ( |gf$__labels| != |label_values| )
|
||||
{
|
||||
Reporter::error(fmt("Invalid label values expected %s, have %s", |gf$__labels|, |label_values|));
|
||||
return gauge_with(error_gauge_cf);
|
||||
}
|
||||
local labels = make_labels(gf$__labels, label_values);
|
||||
local m = Telemetry::__dbl_gauge_metric_get_or_add(gf$__family, labels);
|
||||
return Gauge($__metric=m);
|
||||
}
|
||||
|
||||
function gauge_inc(g: Gauge, amount: double &default=1.0): bool
|
||||
{
|
||||
return Telemetry::__dbl_gauge_inc(g$__metric, amount);
|
||||
}
|
||||
|
||||
function gauge_dec(g: Gauge, amount: double &default=1.0): bool
|
||||
{
|
||||
return Telemetry::__dbl_gauge_dec(g$__metric, amount);
|
||||
}
|
||||
|
||||
function gauge_set(g: Gauge, value: double): bool
|
||||
{
|
||||
# Telemetry currently does not implement __dbl_gauge_set(), do
|
||||
# it by hand here.
|
||||
local cur_value: double = Telemetry::__dbl_gauge_value(g$__metric);
|
||||
if (value > cur_value)
|
||||
return Telemetry::__dbl_gauge_inc(g$__metric, value - cur_value);
|
||||
|
||||
return Telemetry::__dbl_gauge_dec(g$__metric, cur_value - value);
|
||||
}
|
||||
|
||||
function gauge_family_inc(gf: GaugeFamily, label_values: labels_vector, value: double): bool
|
||||
{
|
||||
return gauge_inc(gauge_with(gf, label_values), value);
|
||||
}
|
||||
|
||||
function gauge_family_dec(gf: GaugeFamily, label_values: labels_vector, value: double): bool
|
||||
{
|
||||
return gauge_dec(gauge_with(gf, label_values), value);
|
||||
}
|
||||
|
||||
function gauge_family_set(gf: GaugeFamily, label_values: labels_vector, value: double): bool
|
||||
{
|
||||
return gauge_set(gauge_with(gf, label_values), value);
|
||||
}
|
||||
|
||||
function register_histogram_family(opts: MetricOpts): HistogramFamily
|
||||
{
|
||||
local f = Telemetry::__dbl_histogram_family(
|
||||
opts$prefix,
|
||||
opts$name,
|
||||
opts$labels,
|
||||
opts$bounds,
|
||||
opts$help_text,
|
||||
opts$unit,
|
||||
opts?$is_total ? opts$is_total : F
|
||||
);
|
||||
return HistogramFamily($__family=f, $__labels=opts$labels);
|
||||
}
|
||||
|
||||
# Fallback Histogram when there are issues with the labels.
|
||||
global error_histogram_hf = register_histogram_family([
|
||||
$prefix="zeek",
|
||||
$name="telemetry_histogram_usage_error",
|
||||
$unit="1",
|
||||
$help_text="This histogram is returned when label usage for histograms is wrong. Check reporter.log if non-zero.",
|
||||
$bounds=vector(1.0)
|
||||
]);
|
||||
|
||||
function histogram_with(hf: HistogramFamily, label_values: labels_vector): Histogram
|
||||
{
|
||||
if ( |hf$__labels| != |label_values| )
|
||||
{
|
||||
Reporter::error(fmt("Invalid label values expected %s, have %s", |hf$__labels|, |label_values|));
|
||||
return histogram_with(error_histogram_hf);
|
||||
}
|
||||
|
||||
local labels = make_labels(hf$__labels, label_values);
|
||||
local m = Telemetry::__dbl_histogram_metric_get_or_add(hf$__family, labels);
|
||||
return Histogram($__metric=m);
|
||||
}
|
||||
|
||||
function histogram_observe(h: Histogram, measurement: double): bool
|
||||
{
|
||||
return Telemetry::__dbl_histogram_observe(h$__metric, measurement);
|
||||
}
|
||||
|
||||
function histogram_family_observe(hf: HistogramFamily, label_values: labels_vector, measurement: double): bool
|
||||
{
|
||||
return histogram_observe(histogram_with(hf, label_values), measurement);
|
||||
}
|
||||
|
||||
function collect_metrics(prefix: string, name: string): vector of Metric
|
||||
{
|
||||
return Telemetry::__collect_metrics(prefix, name);
|
||||
}
|
||||
|
||||
function collect_histogram_metrics(prefix: string, name: string): vector of HistogramMetric
|
||||
{
|
||||
return Telemetry::__collect_histogram_metrics(prefix, name);
|
||||
}
|
||||
|
||||
event run_sync_hook()
|
||||
{
|
||||
hook Telemetry::sync();
|
||||
schedule sync_interval { run_sync_hook() };
|
||||
}
|
||||
|
||||
event zeek_init()
|
||||
{
|
||||
schedule sync_interval { run_sync_hook() };
|
||||
}
|
||||
|
||||
# Expose the Zeek version as Prometheus style info metric
|
||||
global version_gauge_family = Telemetry::register_gauge_family([
|
||||
$prefix="zeek",
|
||||
$name="version_info",
|
||||
$unit="1",
|
||||
$help_text="The Zeek version",
|
||||
$labels=vector("version_number", "major", "minor", "patch", "commit",
|
||||
"beta", "debug","version_string")
|
||||
]);
|
||||
|
||||
event zeek_init()
|
||||
{
|
||||
local v = Version::info;
|
||||
local labels = vector(cat(v$version_number),
|
||||
cat(v$major), cat(v$minor), cat (v$patch),
|
||||
cat(v$commit),
|
||||
v$beta ? "true" : "false",
|
||||
v$debug ? "true" : "false",
|
||||
v$version_string);
|
||||
|
||||
Telemetry::gauge_family_set(version_gauge_family, labels, 1.0);
|
||||
}
|
||||
|
|
@ -43,6 +43,7 @@
|
|||
@load base/frameworks/tunnels
|
||||
@load base/frameworks/openflow
|
||||
@load base/frameworks/netcontrol
|
||||
@load base/frameworks/telemetry
|
||||
|
||||
@load base/protocols/conn
|
||||
@load base/protocols/dce-rpc
|
||||
|
|
|
|||
199
scripts/policy/frameworks/telemetry/log.zeek
Normal file
199
scripts/policy/frameworks/telemetry/log.zeek
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
##! Implementation of a telemetry.log and telemetry_histogram.log file
|
||||
##! using metrics accessible via the Telemetry module.
|
||||
|
||||
@load base/frameworks/telemetry
|
||||
|
||||
module Telemetry;
|
||||
|
||||
export {
|
||||
redef enum Log::ID += { LOG, LOG_HISTOGRAM };
|
||||
|
||||
## How often metrics are reported.
|
||||
option log_interval = 60sec;
|
||||
|
||||
## Only metrics with prefixes in this set will be included in the
|
||||
## `telemetry.log` and `telemetry_histogram.log` files by default.
|
||||
## Setting this option to an empty set includes all prefixes.
|
||||
##
|
||||
## For more fine-grained customization, setting this option to an
|
||||
## empty set and implementing the :zeek:see:`Telemetry::log_policy`
|
||||
## and :zeek:see:`Telemetry::log_policy_histogram` hooks to filter
|
||||
## individual records is recommended.
|
||||
option log_prefixes: set[string] = {"process", "zeek"};
|
||||
|
||||
## Record type used for logging counter and gauge metrics.
|
||||
type Info: record {
|
||||
## Timestamp of reporting.
|
||||
ts: time &log;
|
||||
|
||||
## Peer that generated this log.
|
||||
peer: string &log;
|
||||
|
||||
## Contains the value "counter" or "gauge" depending on
|
||||
## the underlying metric type.
|
||||
metric_type: string &log;
|
||||
|
||||
## The prefix (namespace) of the metric.
|
||||
prefix: string &log;
|
||||
|
||||
## The name of the metric.
|
||||
name: string &log;
|
||||
|
||||
## The unit of this metric, or unset if unit-less.
|
||||
unit: string &log &optional;
|
||||
|
||||
## The names of the individual labels.
|
||||
labels: vector of string &log;
|
||||
|
||||
## The values of the labels as listed in ``labels``.
|
||||
label_values: vector of string &log;
|
||||
|
||||
## The value of this metric.
|
||||
value: double &log;
|
||||
};
|
||||
|
||||
## Record type used for logging histogram metrics.
|
||||
type HistogramInfo: record {
|
||||
## Timestamp of reporting.
|
||||
ts: time &log;
|
||||
|
||||
## Peer that generated this log.
|
||||
peer: string &log;
|
||||
|
||||
## The prefix (namespace) of the metric.
|
||||
prefix: string &log;
|
||||
|
||||
## The name of the metric.
|
||||
name: string &log;
|
||||
|
||||
## The unit of this metric, or unset if unit-less.
|
||||
unit: string &log &optional;
|
||||
|
||||
## The names of the individual labels.
|
||||
labels: vector of string &log;
|
||||
|
||||
## The values of the labels as listed in ``labels``.
|
||||
label_values: vector of string &log;
|
||||
|
||||
## The bounds of the individual buckets
|
||||
bounds: vector of double &log;
|
||||
|
||||
## The number of observations within each individual bucket.
|
||||
values: vector of double &log;
|
||||
|
||||
## The sum over all observations
|
||||
sum: double &log;
|
||||
|
||||
## The total number of observations.
|
||||
observations: double &log;
|
||||
};
|
||||
|
||||
## A default logging policy hook for the stream.
|
||||
global log_policy: Log::PolicyHook;
|
||||
|
||||
## A default logging policy hook for the histogram stream.
|
||||
global log_policy_histogram: Log::PolicyHook;
|
||||
|
||||
## Event triggered for every record in the stream.
|
||||
global log_telemetry: event(rec: Info);
|
||||
|
||||
## Event triggered for every record in the histogram stream.
|
||||
global log_telemetry_histogram: event(rec: HistogramInfo);
|
||||
}
|
||||
|
||||
function do_log()
|
||||
{
|
||||
local ts = network_time();
|
||||
local metrics = Telemetry::collect_metrics();
|
||||
|
||||
for ( i in metrics )
|
||||
{
|
||||
local m = metrics[i];
|
||||
|
||||
# Histograms don't have single values, skip over them.
|
||||
if ( m$opts$metric_type == DOUBLE_HISTOGRAM || m$opts$metric_type == INT_HISTOGRAM )
|
||||
next;
|
||||
|
||||
if ( |log_prefixes| > 0 && m$opts$prefix !in log_prefixes )
|
||||
next;
|
||||
|
||||
# Render the metric_type as a short string. Unknown
|
||||
# shouldn't really happen, but lets have a fallback.
|
||||
local metric_type = "unknown";
|
||||
switch ( m$opts$metric_type ) {
|
||||
case DOUBLE_COUNTER, INT_COUNTER:
|
||||
metric_type = "counter";
|
||||
break;
|
||||
case DOUBLE_GAUGE, INT_GAUGE:
|
||||
metric_type = "gauge";
|
||||
break;
|
||||
}
|
||||
|
||||
local rec = Info($ts=ts,
|
||||
$peer=peer_description,
|
||||
$metric_type=metric_type,
|
||||
$prefix=m$opts$prefix,
|
||||
$name=m$opts$name,
|
||||
$labels=m$opts$labels,
|
||||
$label_values=m$labels,
|
||||
$value=m$value);
|
||||
|
||||
if ( m$opts$unit != "1" )
|
||||
rec$unit = m$opts$unit;
|
||||
|
||||
Log::write(LOG, rec);
|
||||
}
|
||||
|
||||
# Logging of histograms.
|
||||
ts = network_time();
|
||||
local histogram_metrics = Telemetry::collect_histogram_metrics();
|
||||
for ( i in histogram_metrics )
|
||||
{
|
||||
local hm = histogram_metrics[i];
|
||||
|
||||
if ( |log_prefixes| > 0 && hm$opts$prefix !in log_prefixes )
|
||||
next;
|
||||
|
||||
local hrec = HistogramInfo($ts=ts,
|
||||
$peer=peer_description,
|
||||
$prefix=hm$opts$prefix,
|
||||
$name=hm$opts$name,
|
||||
$labels=hm$opts$labels,
|
||||
$label_values=hm$labels,
|
||||
$bounds=hm$opts$bounds,
|
||||
$values=hm$values,
|
||||
$sum=hm$sum,
|
||||
$observations=hm$observations);
|
||||
|
||||
if ( hm$opts$unit != "1" )
|
||||
hrec$unit = hm$opts$unit;
|
||||
|
||||
Log::write(LOG_HISTOGRAM, hrec);
|
||||
}
|
||||
}
|
||||
|
||||
event Telemetry::log()
|
||||
{
|
||||
# We explicitly log once during zeek_done(), so short-circuit
|
||||
# here when we're already in the process of shutting down.
|
||||
if ( zeek_is_terminating() )
|
||||
return;
|
||||
|
||||
do_log();
|
||||
schedule log_interval { Telemetry::log() };
|
||||
}
|
||||
|
||||
event zeek_init() &priority=5
|
||||
{
|
||||
Log::create_stream(LOG, [$columns=Info, $ev=log_telemetry, $path="telemetry", $policy=log_policy]);
|
||||
Log::create_stream(LOG_HISTOGRAM, [$columns=HistogramInfo, $ev=log_telemetry_histogram, $path="telemetry_histogram", $policy=log_policy_histogram]);
|
||||
|
||||
schedule log_interval { Telemetry::log() };
|
||||
}
|
||||
|
||||
# Log late during zeek_done() once more. Any metric updates
|
||||
# afterwards won't be visible in the log.
|
||||
event zeek_done() &priority=-1000
|
||||
{
|
||||
do_log();
|
||||
}
|
||||
|
|
@ -89,6 +89,10 @@ redef digest_salt = "Please change this value.";
|
|||
# Extend email alerting to include hostnames
|
||||
@load policy/frameworks/notice/extend-email/hostnames
|
||||
|
||||
# Enable logging of telemetry data into telemetry.log and
|
||||
# telemetry_histogram.log.
|
||||
@load frameworks/telemetry/log
|
||||
|
||||
# Uncomment the following line to enable detection of the heartbleed attack. Enabling
|
||||
# this might impact performance a bit.
|
||||
# @load policy/protocols/ssl/heartbleed
|
||||
|
|
|
|||
|
|
@ -71,6 +71,7 @@
|
|||
@load frameworks/software/version-changes.zeek
|
||||
@load frameworks/software/vulnerable.zeek
|
||||
@load frameworks/software/windows-version-detection.zeek
|
||||
@load frameworks/telemetry/log.zeek
|
||||
@load integration/barnyard2/__load__.zeek
|
||||
@load integration/barnyard2/main.zeek
|
||||
@load integration/barnyard2/types.zeek
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue