Fix the scripts.policy.frameworks.telemetry.prometheus btest to use the service discovery endpoint

ci/alpine/Dockerfile

@@ -16,6 +16,7 @@ RUN apk add --no-cache \
   musl-fts-dev \
   g++ \
   git \
+  jq \
   libpcap-dev \
   linux-headers \
   make \

ci/centos-stream-8/Dockerfile

@@ -16,6 +16,7 @@ RUN dnf -y install \
     gcc \
     gcc-c++ \
     git \
+    jq \
     libpcap-devel \
     make \
     openssl \

ci/centos-stream-9/Dockerfile

@@ -27,6 +27,7 @@ RUN dnf -y --nobest install \
     gcc \
     gcc-c++ \
     git \
+    jq \
     libpcap-devel \
     make \
     openssl \

ci/debian-11/Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update && apt-get -y install \
     g++ \
     gcc \
     git \
+    jq \
     libkrb5-dev \
     libnode-dev \
     libpcap-dev \

ci/debian-12/Dockerfile

@@ -16,6 +16,7 @@ RUN apt-get update && apt-get -y install \
     g++ \
     gcc \
     git \
+    jq \
     libkrb5-dev \
     libnode-dev \
     libpcap-dev \

ci/fedora-38/Dockerfile

@@ -14,6 +14,7 @@ RUN dnf -y install \
     gcc \
     gcc-c++ \
     git \
+    jq \
     libpcap-devel \
     make \
     nodejs-devel \

ci/fedora-39/Dockerfile

@@ -14,6 +14,7 @@ RUN dnf -y install \
     gcc \
     gcc-c++ \
     git \
+    jq \
     libpcap-devel \
     make \
     nodejs-devel \

ci/freebsd/prepare.sh

@@ -6,7 +6,7 @@ set -e
 set -x
 
 env ASSUME_ALWAYS_YES=YES pkg bootstrap
-pkg install -y bash git cmake swig bison python3 base64 flex ccache
+pkg install -y bash git cmake swig bison python3 base64 flex ccache jq
 pkg upgrade -y curl
 pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")')
 pkg install -y $pyver-sqlite3

ci/opensuse-leap-15.5/Dockerfile

@@ -16,6 +16,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5
     gcc12-c++ \
     git \
     gzip \
+    jq \
     libopenssl-devel \
     libpcap-devel \
     make \

ci/opensuse-leap-15.6/Dockerfile

@@ -16,6 +16,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.6
     gcc12-c++ \
     git \
     gzip \
+    jq \
     libopenssl-devel \
     libpcap-devel \
     make \

ci/opensuse-tumbleweed/Dockerfile

@@ -22,6 +22,7 @@ RUN zypper refresh \
     gcc-c++ \
     git \
     gzip \
+    jq \
     libopenssl-devel \
     libpcap-devel \
     make \

ci/ubuntu-20.04/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get update && apt-get -y install \
     g++ \
     gcc \
     git \
+    jq \
     lcov \
     libkrb5-dev \
     libmaxminddb-dev \

ci/ubuntu-22.04/Dockerfile

@@ -17,6 +17,7 @@ RUN apt-get update && apt-get -y install \
     g++ \
     gcc \
     git \
+    jq \
     lcov \
     libkrb5-dev \
     libmaxminddb-dev \

ci/ubuntu-24.04/Dockerfile

@@ -19,6 +19,7 @@ RUN apt-get update && apt-get -y install \
     g++ \
     gcc \
     git \
+    jq \
     lcov \
     libkrb5-dev \
     libmaxminddb-dev \

testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1..stdout

@@ -1,2 +0,0 @@
-### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-manager-1, Telemetry::metrics_port from cluster config, 1028/tcp

testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1.services.out

@@ -1,5 +1,5 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+endpoint="manager-1"
+endpoint="logger-1"
 endpoint="proxy-1"
-
 endpoint="worker-1"
-

testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek

@@ -7,8 +7,10 @@
 # @TEST-PORT: BROKER_PORT2
 # @TEST-PORT: BROKER_PORT3
 # @TEST-PORT: BROKER_PORT4
-# @TEST-PORT: BROKER_PORT4
+# @TEST-PORT: METRICS_PORT1
-# @TEST-PORT: BROKER_TEST_METRICS_PORT
+# @TEST-PORT: METRICS_PORT2
+# @TEST-PORT: METRICS_PORT3
+# @TEST-PORT: METRICS_PORT4
 #
 # @TEST-REQUIRES: which curl
 # @TEST-EXEC: zeek --parse-only %INPUT
@@ -16,21 +18,20 @@
 # @TEST-EXEC: btest-bg-run logger-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=logger-1 zeek -b %INPUT
 # @TEST-EXEC: btest-bg-run proxy-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=proxy-1 zeek -b %INPUT
 # @TEST-EXEC: btest-bg-run worker-1  ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT
-# @TEST-EXEC: btest-bg-wait 10
+# @TEST-EXEC: btest-bg-wait 30
-# @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff manager-1/services.out
 
 @TEST-START-FILE cluster-layout.zeek
 redef Cluster::nodes = {
-	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1")), $metrics_port=1028/tcp],
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1")), $metrics_port=to_port(getenv("METRICS_PORT1"))],
-	["logger-1"] = [$node_type=Cluster::LOGGER,   $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT2")), $manager="manager-1", $metrics_port=1029/tcp],
+	["logger-1"] = [$node_type=Cluster::LOGGER,   $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT2")), $manager="manager-1", $metrics_port=to_port(getenv("METRICS_PORT2"))],
-	["proxy-1"] = [$node_type=Cluster::PROXY,   $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT3")), $manager="manager-1", $metrics_port=1030/tcp],
+	["proxy-1"] = [$node_type=Cluster::PROXY,   $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT3")), $manager="manager-1", $metrics_port=to_port(getenv("METRICS_PORT3"))],
-	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT4")), $manager="manager-1", $metrics_port=1031/tcp],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT4")), $manager="manager-1", $metrics_port=to_port(getenv("METRICS_PORT4"))],
 };
 @TEST-END-FILE
 
 @TEST-START-FILE request-services.sh
-#!/bin/sh
+#! /usr/bin/env bash
 
 # This script makes repeat curl requests to find all of the metrics data from the
 # hosts listed in the services output from the manager, and outputs it all into a
@@ -39,27 +40,39 @@ redef Cluster::nodes = {
 services_url=$1
 output_file=$2
 
-for host in $(curl -s -m 5 ${services_url} | jq '.[0].targets.[]'); do
+services_data=$(curl -s -m 5 ${services_url})
-	host=$(echo ${host} | sed 's/"//g')
+
-	metrics=$(curl -s -m 5 http://${host}/metrics)
+for host in $(echo ${services_data} | jq -r '.[0].targets[]' | sort); do
-	version_info=$(echo ${metrics} | grep -Eo "zeek_version_info\{[^}]+\}" | grep -o 'endpoint=\"[^"]*\"')
+	metrics=$(curl -m 5 --trace trace-${host}.out http://${host}/metrics)
+	if [ $? -eq 0 ] ; then
+		version_info=$(echo ${metrics} | grep -Eo "zeek_version_info\{[^}]+\}" | grep -o 'endpoint="[^"]*"')
 		echo ${version_info} >> ${output_file};
+	else
+		echo "Failed to request data from ${host}" >> ${output_file}
+	fi
 done
 @TEST-END-FILE
 
 @load policy/frameworks/cluster/experimental
 @load policy/frameworks/telemetry/prometheus
 @load base/frameworks/telemetry
-@load base/utils/active-http
+
+# So the cluster nodes don't terminate right away.
+#redef exit_only_after_terminate=T;
 
 @if ( Cluster::node == "manager-1" )
 
-# Query the Prometheus endpoint using ActiveHTTP for testing, oh my.
+@load base/utils/exec
+
+# Query the Prometheus endpoint using curl for testing, oh my.
 event run_test()
 	{
 	local services_url = fmt("http://localhost:%s/services.json", port_to_count(Telemetry::metrics_port));
-	local result = system(fmt("sh ../request-services.sh %s %s", services_url, "services.out"));
+	local req_cmd = fmt("sh ../request-services.sh %s %s", services_url, "services.out");
-	if ( result != 0 )
+
+	when [req_cmd] ( local result = Exec::run([$cmd=req_cmd]) )
+		{
+		if ( result$exit_code != 0 )
 			{
 			# This is bad.
 			print "ERROR: Failed to request service information";
@@ -68,6 +81,13 @@ event run_test()
 
 		terminate();
 		}
+	timeout 10sec
+		{
+		# This is bad.
+		print "ERROR: Timed out requesting service information";
+		exit(1);
+		}
+	}
 
 # Use a dynamic metrics port for testing to avoid colliding on 9911/tcp
 # when running tests in parallel.