Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 06:38:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Move telmetry label names out of opts records, into main metric records

Browse source
This commit is contained in:
Tim Wojtulewicz 2024-06-02 19:46:50 -07:00
parent b1578d4ded
commit 433c257886
10 changed files with 114 additions and 150 deletions
Download patch file Download diff file Expand all files Collapse all files

35
scripts/base/frameworks/telemetry/main.zeek
View file

@ -36,7 +36,8 @@ export {
};
## Register a counter family.
global register_counter_family: function(opts: MetricOpts): CounterFamily;
global register_counter_family: function(opts: MetricOpts,
label_names: labels_vector &default=vector()): CounterFamily;
## Get a :zeek:see:`Telemetry::Counter` instance given family and label values.
global counter_with: function(cf: CounterFamily,
@ -119,7 +120,8 @@ export {
};
## Register a gauge family.
global register_gauge_family: function(opts: MetricOpts): GaugeFamily;
global register_gauge_family: function(opts: MetricOpts,
label_names: labels_vector &default=vector()): GaugeFamily;
## Get a :zeek:see:`Telemetry::Gauge` instance given family and label values.
@ -215,7 +217,8 @@ export {
};
## Register a histogram family.
global register_histogram_family: function(opts: MetricOpts): HistogramFamily;
global register_histogram_family: function(opts: MetricOpts,
label_names: labels_vector &default=vector()): HistogramFamily;
## Get a :zeek:see:`Telemetry::Histogram` instance given family and label values.
global histogram_with: function(hf: HistogramFamily,
@ -290,16 +293,16 @@ function make_labels(keys: vector of string, values: labels_vector): table[strin
return labels;
}
function register_counter_family(opts: MetricOpts): CounterFamily
function register_counter_family(opts: MetricOpts, label_names: labels_vector): CounterFamily
{
local f = Telemetry::__counter_family(
opts$prefix,
opts$name,
opts$labels,
label_names,
opts$help_text,
opts$unit
);
return CounterFamily($__family=f, $__labels=opts$labels);
return CounterFamily($__family=f, $__labels=label_names);
}
# Fallback Counter returned when there are issues with the labels.
@ -349,16 +352,16 @@ function counter_family_set(cf: CounterFamily, label_values: labels_vector, valu
return counter_set(counter_with(cf, label_values), value);
}
function register_gauge_family(opts: MetricOpts): GaugeFamily
function register_gauge_family(opts: MetricOpts, label_names: labels_vector): GaugeFamily
{
local f = Telemetry::__gauge_family(
opts$prefix,
opts$name,
opts$labels,
label_names,
opts$help_text,
opts$unit
);
return GaugeFamily($__family=f, $__labels=opts$labels);
return GaugeFamily($__family=f, $__labels=label_names);
}
# Fallback Gauge returned when there are issues with the label usage.
@ -417,17 +420,17 @@ function gauge_family_set(gf: GaugeFamily, label_values: labels_vector, value: d
return gauge_set(gauge_with(gf, label_values), value);
}
function register_histogram_family(opts: MetricOpts): HistogramFamily
function register_histogram_family(opts: MetricOpts, label_names: labels_vector): HistogramFamily
{
local f = Telemetry::__histogram_family(
opts$prefix,
opts$name,
opts$labels,
label_names,
opts$bounds,
opts$help_text,
opts$unit
);
return HistogramFamily($__family=f, $__labels=opts$labels);
return HistogramFamily($__family=f, $__labels=label_names);
}
# Fallback Histogram when there are issues with the labels.
@ -483,10 +486,10 @@ global version_gauge_family = Telemetry::register_gauge_family([
$prefix="zeek",
$name="version_info",
$unit="",
$help_text="The Zeek version",
$labels=vector("version_number", "major", "minor", "patch", "commit",
"beta", "debug","version_string")
]);
$help_text="The Zeek version"],
vector("version_number", "major", "minor", "patch", "commit",
"beta", "debug","version_string")
);
event zeek_init()
{

30
scripts/base/init-bare.zeek
View file

@ -5802,14 +5802,6 @@ export {
## Documentation for this metric.
help_text: string &optional;
## The label names (also called dimensions) of the metric. When
## instantiating or working with concrete metrics, corresponding
## label values have to be provided. Examples of a label might
## be the protocol a general observation applies to, the
## directionality in a traffic flow, or protocol-specific
## context like a particular message type.
labels: vector of string &default=vector();
## Whether the metric represents something that is accumulating.
## Defaults to ``T`` for counters and ``F`` for gauges and
## histograms.
@ -5832,8 +5824,16 @@ export {
## A :zeek:see:`Telemetry::MetricOpts` record describing this metric.
opts: MetricOpts;
## The label names (also called dimensions) of the metric. When
## instantiating or working with concrete metrics, corresponding
## label values have to be provided. Examples of a label might
## be the protocol a general observation applies to, the
## directionality in a traffic flow, or protocol-specific
## context like a particular message type.
label_names: vector of string &default=vector();
## The label values associated with this metric, if any.
labels: vector of string;
label_values: vector of string &optional;
## The value of gauge or counter cast to a double
## independent of the underlying data type.
@ -5847,8 +5847,16 @@ export {
## A :zeek:see:`Telemetry::MetricOpts` record describing this histogram.
opts: MetricOpts;
## The label values associated with this histogram, if any.
labels: vector of string;
## The label names (also called dimensions) of the metric. When
## instantiating or working with concrete metrics, corresponding
## label values have to be provided. Examples of a label might
## be the protocol a general observation applies to, the
## directionality in a traffic flow, or protocol-specific
## context like a particular message type.
label_names: vector of string &default=vector();
## The label values associated with this metric, if any.
label_values: vector of string &optional;
## Individual counters for each of the buckets as
## described by the *bounds* field in *opts*;

8
scripts/policy/frameworks/telemetry/log.zeek
View file

@ -132,8 +132,8 @@ function do_log()
$peer=peer_description,
$metric_type=metric_type,
$name=m$opts$name,
$labels=m$opts$labels,
$label_values=m$labels,
$labels=m$label_names,
$label_values=m$label_values,
$value=m$value);
Log::write(LOG, rec);
@ -162,8 +162,8 @@ function do_log()
local hrec = HistogramInfo($ts=ts,
$peer=peer_description,
$name=hm$opts$name,
$labels=hm$opts$labels,
$label_values=hm$labels,
$labels=hm$label_names,
$label_values=hm$label_values,
$bounds=hm$opts$bounds,
$values=hm$values,
$sum=hm$sum,

121
src/telemetry/Manager.cc
View file

@ -134,7 +134,6 @@ RecordValPtr Manager::GetMetricOptsRecord(const prometheus::MetricFamily& metric
static auto name_idx = metric_opts_type->FieldOffset("name");
static auto help_text_idx = metric_opts_type->FieldOffset("help_text");
static auto unit_idx = metric_opts_type->FieldOffset("unit");
static auto labels_idx = metric_opts_type->FieldOffset("labels");
static auto is_total_idx = metric_opts_type->FieldOffset("is_total");
static auto metric_type_idx = metric_opts_type->FieldOffset("metric_type");
@ -156,55 +155,15 @@ RecordValPtr Manager::GetMetricOptsRecord(const prometheus::MetricFamily& metric
// Assume that a metric ending with _total is always a summed metric so we can set that.
record_val->Assign(is_total_idx, val_mgr->Bool(util::ends_with(metric_family.name, "_total")));
auto label_names_vec = make_intrusive<zeek::VectorVal>(string_vec_type);
// Check if this is a Zeek-internal metric. We keep a little more information about a metric
// for these than we do for ones that were inserted into prom-cpp directly.
if ( auto it = families.find(metric_family.name); it != families.end() ) {
record_val->Assign(metric_type_idx,
zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(it->second->MetricType()));
for ( const auto& lbl : it->second->LabelNames() )
label_names_vec->Append(make_intrusive<StringVal>(lbl));
}
else {
// prom-cpp stores everything internally as doubles
if ( metric_family.type == prometheus::MetricType::Counter )
record_val->Assign(metric_type_idx, zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(
BifEnum::Telemetry::MetricType::COUNTER));
if ( metric_family.type == prometheus::MetricType::Gauge )
record_val->Assign(metric_type_idx, zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(
BifEnum::Telemetry::MetricType::GAUGE));
if ( metric_family.type == prometheus::MetricType::Histogram )
record_val->Assign(metric_type_idx, zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(
BifEnum::Telemetry::MetricType::HISTOGRAM));
// prometheus-cpp doesn't store label names anywhere other than in each
// instrument. this is valid because label names can be different
// between instruments within a single family for prometheus. we don't
// follow that model in Zeek, so use the names from the first instrument
// but validate that they're the same in the rest and warn if not.
if ( ! metric_family.metric.empty() ) {
std::unordered_set<std::string> names;
for ( const auto& lbl : metric_family.metric[0].label ) {
label_names_vec->Append(make_intrusive<StringVal>(lbl.name));
names.insert(lbl.name);
}
if ( metric_family.metric.size() > 1 ) {
for ( size_t i = 1; i < metric_family.metric.size(); ++i ) {
for ( const auto& lbl : metric_family.metric[i].label ) {
if ( names.count(lbl.name) == 0 )
reporter->Warning(
"Telemetry labels must be the same across all instruments for metric family %s\n",
metric_family.name.c_str());
}
}
}
}
}
record_val->Assign(labels_idx, label_names_vec);
if ( metric_family.type == prometheus::MetricType::Counter )
record_val->Assign(metric_type_idx, zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(
BifEnum::Telemetry::MetricType::COUNTER));
if ( metric_family.type == prometheus::MetricType::Gauge )
record_val->Assign(metric_type_idx, zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(
BifEnum::Telemetry::MetricType::GAUGE));
if ( metric_family.type == prometheus::MetricType::Histogram )
record_val->Assign(metric_type_idx, zeek::BifType::Enum::Telemetry::MetricType->GetEnumVal(
BifEnum::Telemetry::MetricType::HISTOGRAM));
opts_records.insert({metric_family.name, record_val});
@ -244,8 +203,8 @@ static bool comparer(const std::optional<ZVal>& a, const std::optional<ZVal>& b,
auto a_r = a->ToVal(type)->AsRecordVal();
auto b_r = b->ToVal(type)->AsRecordVal();
auto a_labels = a_r->GetField<VectorVal>("labels");
auto b_labels = b_r->GetField<VectorVal>("labels");
auto a_labels = a_r->GetField<VectorVal>("label_values");
auto b_labels = b_r->GetField<VectorVal>("label_values");
return compare_string_vectors(a_labels, b_labels);
}
@ -259,39 +218,14 @@ static bool compare_histograms(const std::optional<ZVal>& a, const std::optional
return comparer(a, b, metric_record_type);
}
static VectorValPtr build_label_values_vector(const std::vector<prometheus::ClientMetric::Label>& prom_labels,
const VectorValPtr& record_label_names) {
static auto string_vec_type = zeek::id::find_type<zeek::VectorType>("string_vec");
auto label_values_vec = make_intrusive<VectorVal>(string_vec_type);
// This feels really bad, since it's an O(m*n) search to bulld the vector,
// but prometheus-cpp returns us a vector of labels and so we just have to
// search through it.
int i = 0;
for ( const auto& name : record_label_names->RawVec() ) {
auto n = name->AsString()->ToStdStringView();
auto it = std::find_if(prom_labels.begin(), prom_labels.end(),
[n](const prometheus::ClientMetric::Label& l) { return l.name == n; });
if ( it != prom_labels.end() )
label_values_vec->Assign(i, make_intrusive<StringVal>(it->value));
// See the comment in GetMetricOptsRecord about how labels from non-Zeek
// metrics within the same family can have different labels from each
// other. In this case we might leave some fields null in the output.
++i;
}
return label_values_vec;
}
ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view name_pattern) {
static auto metrics_vector_type = zeek::id::find_type<VectorType>("Telemetry::MetricVector");
static auto string_vec_type = zeek::id::find_type<zeek::VectorType>("string_vec");
static auto metric_record_type = zeek::id::find_type<zeek::RecordType>("Telemetry::Metric");
static auto opts_idx = metric_record_type->FieldOffset("opts");
static auto labels_idx = metric_record_type->FieldOffset("labels");
static auto value_idx = metric_record_type->FieldOffset("value");
static auto label_names_idx = metric_record_type->FieldOffset("label_names");
static auto label_values_idx = metric_record_type->FieldOffset("label_values");
static auto metric_opts_type = zeek::id::find_type<zeek::RecordType>("Telemetry::MetricOpts");
static auto metric_type_idx = metric_opts_type->FieldOffset("metric_type");
@ -313,11 +247,9 @@ ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view
continue;
RecordValPtr opts_record = GetMetricOptsRecord(fam);
const auto& label_names = opts_record->GetField<VectorVal>("labels");
for ( const auto& inst : fam.metric ) {
auto r = make_intrusive<zeek::RecordVal>(metric_record_type);
r->Assign(labels_idx, build_label_values_vector(inst.label, label_names));
r->Assign(opts_idx, opts_record);
if ( fam.type == prometheus::MetricType::Counter )
@ -325,6 +257,17 @@ ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view
else if ( fam.type == prometheus::MetricType::Gauge )
r->Assign(value_idx, zeek::make_intrusive<DoubleVal>(inst.gauge.value));
auto label_names_vec = make_intrusive<zeek::VectorVal>(string_vec_type);
auto label_values_vec = make_intrusive<zeek::VectorVal>(string_vec_type);
for ( const auto& lbl : inst.label ) {
label_names_vec->Append(make_intrusive<StringVal>(lbl.name));
label_values_vec->Append(make_intrusive<StringVal>(lbl.value));
}
r->Assign(label_names_idx, label_names_vec);
r->Assign(label_values_idx, label_values_vec);
ret_val->Append(r);
}
}
@ -350,8 +293,9 @@ ValPtr Manager::CollectHistogramMetrics(std::string_view prefix_pattern, std::st
static auto string_vec_type = zeek::id::find_type<zeek::VectorType>("string_vec");
static auto double_vec_type = zeek::id::find_type<zeek::VectorType>("double_vec");
static auto histogram_metric_type = zeek::id::find_type<zeek::RecordType>("Telemetry::HistogramMetric");
static auto labels_idx = histogram_metric_type->FieldOffset("labels");
static auto values_idx = histogram_metric_type->FieldOffset("values");
static auto label_names_idx = histogram_metric_type->FieldOffset("label_names");
static auto label_values_idx = histogram_metric_type->FieldOffset("label_values");
static auto observations_idx = histogram_metric_type->FieldOffset("observations");
static auto sum_idx = histogram_metric_type->FieldOffset("sum");
@ -380,13 +324,22 @@ ValPtr Manager::CollectHistogramMetrics(std::string_view prefix_pattern, std::st
continue;
RecordValPtr opts_record = GetMetricOptsRecord(fam);
const auto& label_names = opts_record->GetField<VectorVal>("labels");
for ( const auto& inst : fam.metric ) {
auto r = make_intrusive<zeek::RecordVal>(histogram_metric_type);
r->Assign(labels_idx, build_label_values_vector(inst.label, label_names));
r->Assign(opts_idx, opts_record);
auto label_names_vec = make_intrusive<zeek::VectorVal>(string_vec_type);
auto label_values_vec = make_intrusive<zeek::VectorVal>(string_vec_type);
for ( const auto& lbl : inst.label ) {
label_names_vec->Append(make_intrusive<StringVal>(lbl.name));
label_values_vec->Append(make_intrusive<StringVal>(lbl.value));
}
r->Assign(label_names_idx, label_names_vec);
r->Assign(label_values_idx, label_values_vec);
auto double_values_vec = make_intrusive<zeek::VectorVal>(double_vec_type);
std::vector<double> boundaries;
uint64_t last = 0.0;

6
testing/btest/Baseline/scripts.base.frameworks.logging.telemetry/telemetry.log
View file

@ -10,7 +10,7 @@
XXXXXXXXXX.XXXXXX zeek counter zeek_log_stream_writes_total module,stream Conn,Conn::LOG 34.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_stream_writes_total module,stream DNS,DNS::LOG 34.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_stream_writes_total module,stream HTTP,HTTP::LOG 14.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_writer_writes_total writer,module,stream,filter-name,path Log::WRITER_ASCII,Conn,Conn::LOG,-,conn 30.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_writer_writes_total writer,module,stream,filter-name,path Log::WRITER_ASCII,DNS,DNS::LOG,-,dns 23.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_writer_writes_total writer,module,stream,filter-name,path Log::WRITER_ASCII,HTTP,HTTP::LOG,-,http 10.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_writer_writes_total filter_name,module,path,stream,writer default,Conn,conn,Conn::LOG,Log::WRITER_ASCII 30.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_writer_writes_total filter_name,module,path,stream,writer default,DNS,dns,DNS::LOG,Log::WRITER_ASCII 23.0
XXXXXXXXXX.XXXXXX zeek counter zeek_log_writer_writes_total filter_name,module,path,stream,writer default,HTTP,http,HTTP::LOG,Log::WRITER_ASCII 10.0
#close XXXX-XX-XX-XX-XX-XX

34
testing/btest/scripts/base/frameworks/telemetry/basic.zeek
View file

@ -14,42 +14,42 @@ global btest_a_cf = Telemetry::register_counter_family([
$prefix="btest",
$name="a_test",
$unit="",
$help_text="A btest metric",
$labels=vector("x", "y")
]);
$help_text="A btest metric"],
vector("x", "y")
);
global btest_b_cf = Telemetry::register_counter_family([
$prefix="btest",
$name="b_test",
$unit="",
$help_text="Another btest metric",
$labels=vector("x", "y")
]);
$help_text="Another btest metric"],
vector("x", "y")
);
global btest_c_cf = Telemetry::register_counter_family([
$prefix="btest",
$name="c_test",
$unit="",
$help_text="The last btest metric",
$labels=vector("x", "y")
]);
$help_text="The last btest metric"],
vector("x", "y")
);
global system_sensor_temp_gf = Telemetry::register_gauge_family([
$prefix="system",
$name="sensor_temperature",
$unit="celsius",
$help_text="Temperatures reported by sensors in the system",
$labels=vector("name")
]);
$help_text="Temperatures reported by sensors in the system"],
vector("name")
);
global btest_sample_histogram_hf = Telemetry::register_histogram_family([
$prefix="btest",
$name="sample_histogram",
$unit="",
$help_text="A sample histogram that is not returned by Telemetry::collect_metrics",
$bounds=vector(1.0, 2.0, 3.0, 4.0, 5.0),
$labels=vector("dim")
]);
$bounds=vector(1.0, 2.0, 3.0, 4.0, 5.0)],
vector("dim")
);
function print_metrics(what: string, metrics: vector of Telemetry::Metric)
{
@ -57,7 +57,7 @@ function print_metrics(what: string, metrics: vector of Telemetry::Metric)
for (i in metrics)
{
local m = metrics[i];
print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$labels, m$labels, m$value;
print m$opts$metric_type, m$opts$prefix, m$opts$name, m$label_names, m$label_values, m$value;
}
}
@ -67,7 +67,7 @@ function print_histogram_metrics(what: string, metrics: vector of Telemetry::His
for (i in metrics)
{
local m = metrics[i];
print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$bounds, m$opts$labels, m$labels, m$values, m$sum, m$observations;
print m$opts$metric_type, m$opts$prefix, m$opts$name, m$opts$bounds, m$label_names, m$label_values, m$values, m$sum, m$observations;
}
}

10
testing/btest/scripts/base/frameworks/telemetry/conn-duration-histogram.zeek
View file

@ -18,11 +18,11 @@ global connection_duration_hf = Telemetry::register_histogram_family([
global realistic_connection_duration_hf = Telemetry::register_histogram_family([
$prefix="zeek",
$name="realistic_connection_duration",
$labels=vector("proto"),
$unit="seconds",
$help_text="Monitored connection durations by protocol",
$bounds=vector(0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300, 900.0, 1800.0)
]);
$bounds=vector(0.1, 1.0, 10.0, 30.0, 60.0, 120.0, 300, 900.0, 1800.0)],
vector("proto"),
);
global connection_duration_h = Telemetry::histogram_with(connection_duration_hf);
@ -42,8 +42,8 @@ event zeek_done() &priority=-100
{
local hm = histogram_metrics[i];
print hm$opts$metric_type, hm$opts$prefix, hm$opts$name;
print hm$opts$labels;
print hm$labels;
print hm$label_names;
print hm$label_values;
print hm$opts$bounds;
print hm$values;
print hm$observations, hm$sum;

4
testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek
View file

@ -16,7 +16,7 @@ event zeek_done() &priority=-100
local ms = Telemetry::collect_metrics("zeek", "event_handler_invocations");
for ( _, m in ms )
{
if ( /zeek_.*|connection_.*/ in cat(m$labels))
print m$opts$prefix, m$opts$name, m$labels, m$value;
if ( /zeek_.*|connection_.*/ in cat(m$label_values))
print m$opts$prefix, m$opts$name, m$label_values, m$value;
}
}

12
testing/btest/scripts/base/utils/json.test
View file

@ -140,9 +140,9 @@ event zeek_init()
$prefix="btest",
$name="btest_testing_gauge",
$unit="",
$help_text="Btest testing",
$labels=vector("dim_1"),
]);
$help_text="Btest testing"],
vector("dim_1"),
);
local gauge = Telemetry::gauge_with(gauge_family, vector("dim_1_value"));
print to_json(gauge);
print to_json(gauge_family);
@ -151,9 +151,9 @@ event zeek_init()
$prefix="btest",
$name="btest_testing_counter",
$unit="",
$help_text="Btest testing",
$labels=vector("dim_1"),
]);
$help_text="Btest testing"],
vector("dim_1"),
);
local counter = Telemetry::counter_with(counter_family, vector("dim_1_value"));
print to_json(counter);
print to_json(counter_family);

4
testing/btest/scripts/policy/frameworks/telemetry/log-prefixes.zeek
View file

@ -12,9 +12,9 @@ global connections_by_proto_cf = Telemetry::register_counter_family([
$prefix="btest",
$name="connections",
$unit="",
$help_text="Total number of monitored connections",
$help_text="Total number of monitored connections"],
$labels=vector("proto")
]);
);
global connection_duration_hf = Telemetry::register_histogram_family([
$prefix="btest",