Fix some determinism issues with btests

scripts/base/init-bare.zeek

@@ -5947,6 +5947,9 @@ const io_poll_interval_default = 100 &redef;
 ## .. zeek:see:: io_poll_interval_default
 const io_poll_interval_live = 10 &redef;
 
+## Whether Zeek is being run under test. This can be used to alter functionality
+## while testing, but should be used sparingly.
+const running_under_test: bool = F &redef;
 
 global done_with_network = F;
 event net_done(t: time)

src/telemetry/Manager.cc

@@ -195,8 +195,56 @@ RecordValPtr Manager::GetMetricOptsRecord(const prometheus::MetricFamily& metric
     return record_val;
 }
 
+static bool compare_string_vectors(const VectorValPtr& a, const VectorValPtr& b) {
+    if ( a->Size() < b->Size() )
+        return true;
+    if ( a->Size() > b->Size() )
+        return false;
+
+    auto a_v = a->RawVec();
+    auto b_v = b->RawVec();
+
+    auto b_it = b_v.begin();
+    for ( auto a_it = a_v.begin(); a_it != a_v.end(); ++a_it, ++b_it ) {
+        if ( ! a_it->has_value() )
+            return false;
+        if ( ! b_it->has_value() )
+            return true;
+
+        if ( (*a_it)->AsString()->ToStdStringView() < (*b_it)->AsString()->ToStdStringView() )
+            return true;
+    }
+
+    return false;
+}
+
+static bool comparer(const std::optional<ZVal>& a, const std::optional<ZVal>& b, const RecordTypePtr& type) {
+    if ( ! a )
+        return false;
+
+    if ( ! b )
+        return true;
+
+    auto a_r = a->ToVal(type)->AsRecordVal();
+    auto b_r = b->ToVal(type)->AsRecordVal();
+
+    auto a_labels = a_r->GetField<VectorVal>("labels");
+    auto b_labels = b_r->GetField<VectorVal>("labels");
+    return compare_string_vectors(a_labels, b_labels);
+}
+
+static bool compare_metrics(const std::optional<ZVal>& a, const std::optional<ZVal>& b) {
+    static auto metric_record_type = zeek::id::find_type<zeek::RecordType>("Telemetry::Metric");
+    return comparer(a, b, metric_record_type);
+}
+
+static bool compare_histograms(const std::optional<ZVal>& a, const std::optional<ZVal>& b) {
+    static auto metric_record_type = zeek::id::find_type<zeek::RecordType>("Telemetry::HistogramMetric");
+    return comparer(a, b, metric_record_type);
+}
+
 ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view name_pattern) {
-    static auto metrics_vector_type = zeek::id::find_type<VectorType>("any_vec");
+    static auto metrics_vector_type = zeek::id::find_type<VectorType>("Telemetry::MetricVector");
     static auto string_vec_type = zeek::id::find_type<zeek::VectorType>("string_vec");
     static auto metric_record_type = zeek::id::find_type<zeek::RecordType>("Telemetry::Metric");
     static auto opts_idx = metric_record_type->FieldOffset("opts");
@@ -257,11 +305,24 @@ ValPtr Manager::CollectMetrics(std::string_view prefix_pattern, std::string_view
         }
     }
 
+    // If running under test, there are issues with the non-deterministic
+    // ordering of the metrics coming out of prometheus-cpp, which uses
+    // std::hash on the label values to sort them. Check for that case and sort
+    // the results to some fixed order so that the tests have consistent
+    // results.
+    if ( ret_val->Size() > 0 ) {
+        static auto running_under_test = id::find_val("running_under_test")->AsBool();
+        if ( running_under_test ) {
+            auto& vec = ret_val->RawVec();
+            std::sort(vec.begin(), vec.end(), compare_histograms);
+        }
+    }
+
     return ret_val;
 }
 
 ValPtr Manager::CollectHistogramMetrics(std::string_view prefix_pattern, std::string_view name_pattern) {
-    static auto metrics_vector_type = zeek::id::find_type<VectorType>("any_vec");
+    static auto metrics_vector_type = zeek::id::find_type<VectorType>("Telemetry::HistogramMetricVector");
     static auto string_vec_type = zeek::id::find_type<zeek::VectorType>("string_vec");
     static auto double_vec_type = zeek::id::find_type<zeek::VectorType>("double_vec");
     static auto count_vec_type = zeek::id::find_type<zeek::VectorType>("index_vec");
@@ -363,6 +424,19 @@ ValPtr Manager::CollectHistogramMetrics(std::string_view prefix_pattern, std::st
         }
     }
 
+    // If running under btest, there are issues with the non-deterministic
+    // ordering of the metrics coming out of prometheus-cpp, which uses
+    // std::hash on the label values to sort them. Check for that case and sort
+    // the results to some fixed order so that the tests have consistent
+    // results.
+    if ( ret_val->Size() > 0 ) {
+        static auto running_under_test = id::find_val("running_under_test")->AsBool();
+        if ( running_under_test ) {
+            auto& vec = ret_val->RawVec();
+            std::sort(vec.begin(), vec.end(), compare_histograms);
+        }
+    }
+
     return ret_val;
 }
 

testing/btest/Baseline/scripts.base.frameworks.telemetry.event-handler-invocations/out

@@ -1,4 +1,4 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-zeek, zeek_event_handler_invocations_total, [zeek_done], 1
 zeek, zeek_event_handler_invocations_total, [connection_state_remove], 500
+zeek, zeek_event_handler_invocations_total, [zeek_done], 1
 zeek, zeek_event_handler_invocations_total, [zeek_init], 1

testing/btest/scripts/base/frameworks/telemetry/basic.zeek

@@ -8,6 +8,8 @@
 
 @load base/frameworks/telemetry
 
+redef running_under_test = T;
+
 global btest_a_cf = Telemetry::register_counter_family([
 	$prefix="btest",
 	$name="a_test",

testing/btest/scripts/base/frameworks/telemetry/event-handler-invocations.zeek

@@ -9,6 +9,7 @@
 
 @load base/frameworks/telemetry
 
+redef running_under_test = T;
 
 event zeek_done() &priority=-100
 	{