diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index d77493032c..769bc372f2 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -5947,6 +5947,9 @@ const io_poll_interval_default = 100 &redef; ## .. zeek:see:: io_poll_interval_default const io_poll_interval_live = 10 &redef; +## Whether Zeek is being run under test. This can be used to alter functionality +## while testing, but should be used sparingly. +const running_under_test: bool = F &redef; global done_with_network = F; event net_done(t: time) diff --git a/src/telemetry/Manager.cc b/src/telemetry/Manager.cc index cfe5ee56ff..dbc747e54f 100644 --- a/src/telemetry/Manager.cc +++ b/src/telemetry/Manager.cc @@ -195,8 +195,56 @@ RecordValPtr Manager::GetMetricOptsRecord(const prometheus::MetricFamily& metric return record_val; } +static bool compare_string_vectors(const VectorValPtr& a, const VectorValPtr& b) { + if ( a->Size() < b->Size() ) + return true; + if ( a->Size() > b->Size() ) + return false; + + auto a_v = a->RawVec(); + auto b_v = b->RawVec(); + + auto b_it = b_v.begin(); + for ( auto a_it = a_v.begin(); a_it != a_v.end(); ++a_it, ++b_it ) { + if ( ! a_it->has_value() ) + return false; + if ( ! b_it->has_value() ) + return true; + + if ( (*a_it)->AsString()->ToStdStringView() < (*b_it)->AsString()->ToStdStringView() ) + return true; + } + + return false; +} + +static bool comparer(const std::optional& a, const std::optional& b, const RecordTypePtr& type) { + if ( ! a ) + return false; + + if ( ! b ) + return true; + + auto a_r = a->ToVal(type)->AsRecordVal(); + auto b_r = b->ToVal(type)->AsRecordVal(); + + auto a_labels = a_r->GetField("labels"); + auto b_labels = b_r->GetField("labels"); + return compare_string_vectors(a_labels, b_labels); +} + +static bool compare_metrics(const std::optional& a, const std::optional& b) { + static auto metric_record_type = zeek::id::find_type("Telemetry::Metric"); + return comparer(a, b, metric_record_type); +} + +static bool compare_histograms(const std::optional& a, const std::optional& b) { + static auto metric_record_type = zeek::id::find_type("Telemetry::HistogramMetric"); + return comparer(a, b, metric_record_type); +} + ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view name_pattern) { - static auto metrics_vector_type = zeek::id::find_type("any_vec"); + static auto metrics_vector_type = zeek::id::find_type("Telemetry::MetricVector"); static auto string_vec_type = zeek::id::find_type("string_vec"); static auto metric_record_type = zeek::id::find_type("Telemetry::Metric"); static auto opts_idx = metric_record_type->FieldOffset("opts"); @@ -257,11 +305,24 @@ ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view } } + // If running under test, there are issues with the non-deterministic + // ordering of the metrics coming out of prometheus-cpp, which uses + // std::hash on the label values to sort them. Check for that case and sort + // the results to some fixed order so that the tests have consistent + // results. + if ( ret_val->Size() > 0 ) { + static auto running_under_test = id::find_val("running_under_test")->AsBool(); + if ( running_under_test ) { + auto& vec = ret_val->RawVec(); + std::sort(vec.begin(), vec.end(), compare_histograms); + } + } + return ret_val; } ValPtr Manager::CollectHistogramMetrics(std::string_view prefix_pattern, std::string_view name_pattern) { - static auto metrics_vector_type = zeek::id::find_type("any_vec"); + static auto metrics_vector_type = zeek::id::find_type("Telemetry::HistogramMetricVector"); static auto string_vec_type = zeek::id::find_type("string_vec"); static auto double_vec_type = zeek::id::find_type("double_vec"); static auto count_vec_type = zeek::id::find_type("index_vec"); @@ -363,6 +424,19 @@ ValPtr Manager::CollectHistogramMetrics(std::string_view prefix_pattern, std::st } } + // If running under btest, there are issues with the non-deterministic + // ordering of the metrics coming out of prometheus-cpp, which uses + // std::hash on the label values to sort them. Check for that case and sort + // the results to some fixed order so that the tests have consistent + // results. + if ( ret_val->Size() > 0 ) { + static auto running_under_test = id::find_val("running_under_test")->AsBool(); + if ( running_under_test ) { + auto& vec = ret_val->RawVec(); + std::sort(vec.begin(), vec.end(), compare_histograms); + } + } + return ret_val; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.telemetry.event-handler-invocations/out b/testing/btest/Baseline/scripts.base.frameworks.telemetry.event-handler-invocations/out index e3207ca1d6..1e5c13eb8e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.telemetry.event-handler-invocations/out +++ b/testing/btest/Baseline/scripts.base.frameworks.telemetry.event-handler-invocations/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -zeek, zeek_event_handler_invocations_total, [zeek_done], 1 zeek, zeek_event_handler_invocations_total, [connection_state_remove], 500 +zeek, zeek_event_handler_invocations_total, [zeek_done], 1 zeek, zeek_event_handler_invocations_total, [zeek_init], 1 diff --git a/testing/btest/scripts/base/frameworks/telemetry/basic.zeek b/testing/btest/scripts/base/frameworks/telemetry/basic.zeek index ee9344f0a6..96dd0f1669 100644 --- a/testing/btest/scripts/base/frameworks/telemetry/basic.zeek +++ b/testing/btest/scripts/base/frameworks/telemetry/basic.zeek @@ -8,6 +8,8 @@ @load base/frameworks/telemetry +redef running_under_test = T; + global btest_a_cf = Telemetry::register_counter_family([ $prefix="btest", $name="a_test", diff --git a/testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek b/testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek index e39becd2d4..3a013be7e3 100644 --- a/testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek +++ b/testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek @@ -9,6 +9,7 @@ @load base/frameworks/telemetry +redef running_under_test = T; event zeek_done() &priority=-100 {