diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile index 0c6c2153ca..30676c3c4d 100644 --- a/ci/alpine/Dockerfile +++ b/ci/alpine/Dockerfile @@ -16,6 +16,7 @@ RUN apk add --no-cache \ musl-fts-dev \ g++ \ git \ + jq \ libpcap-dev \ linux-headers \ make \ diff --git a/ci/centos-stream-8/Dockerfile b/ci/centos-stream-8/Dockerfile index 5d05b01927..d8e7322c11 100644 --- a/ci/centos-stream-8/Dockerfile +++ b/ci/centos-stream-8/Dockerfile @@ -16,6 +16,7 @@ RUN dnf -y install \ gcc \ gcc-c++ \ git \ + jq \ libpcap-devel \ make \ openssl \ diff --git a/ci/centos-stream-9/Dockerfile b/ci/centos-stream-9/Dockerfile index f2db5a21ca..49414d69c6 100644 --- a/ci/centos-stream-9/Dockerfile +++ b/ci/centos-stream-9/Dockerfile @@ -27,6 +27,7 @@ RUN dnf -y --nobest install \ gcc \ gcc-c++ \ git \ + jq \ libpcap-devel \ make \ openssl \ diff --git a/ci/debian-11/Dockerfile b/ci/debian-11/Dockerfile index 9b73b5d512..ea206a18e2 100644 --- a/ci/debian-11/Dockerfile +++ b/ci/debian-11/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && apt-get -y install \ g++ \ gcc \ git \ + jq \ libkrb5-dev \ libnode-dev \ libpcap-dev \ diff --git a/ci/debian-12/Dockerfile b/ci/debian-12/Dockerfile index db2e1bf2b9..f989f4e28a 100644 --- a/ci/debian-12/Dockerfile +++ b/ci/debian-12/Dockerfile @@ -16,6 +16,7 @@ RUN apt-get update && apt-get -y install \ g++ \ gcc \ git \ + jq \ libkrb5-dev \ libnode-dev \ libpcap-dev \ diff --git a/ci/fedora-38/Dockerfile b/ci/fedora-38/Dockerfile index 326a8398bb..5ed4573ac6 100644 --- a/ci/fedora-38/Dockerfile +++ b/ci/fedora-38/Dockerfile @@ -14,6 +14,7 @@ RUN dnf -y install \ gcc \ gcc-c++ \ git \ + jq \ libpcap-devel \ make \ nodejs-devel \ diff --git a/ci/fedora-39/Dockerfile b/ci/fedora-39/Dockerfile index 3b9177415a..4daaa89308 100644 --- a/ci/fedora-39/Dockerfile +++ b/ci/fedora-39/Dockerfile @@ -14,6 +14,7 @@ RUN dnf -y install \ gcc \ gcc-c++ \ git \ + jq \ libpcap-devel \ make \ nodejs-devel \ diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index 6c0bfc2896..629ed3c513 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -6,7 +6,7 @@ set -e set -x env ASSUME_ALWAYS_YES=YES pkg bootstrap -pkg install -y bash git cmake swig bison python3 base64 flex ccache +pkg install -y bash git cmake swig bison python3 base64 flex ccache jq pkg upgrade -y curl pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")') pkg install -y $pyver-sqlite3 diff --git a/ci/opensuse-leap-15.5/Dockerfile b/ci/opensuse-leap-15.5/Dockerfile index 605412a885..be927da6c0 100644 --- a/ci/opensuse-leap-15.5/Dockerfile +++ b/ci/opensuse-leap-15.5/Dockerfile @@ -16,6 +16,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5 gcc12-c++ \ git \ gzip \ + jq \ libopenssl-devel \ libpcap-devel \ make \ diff --git a/ci/opensuse-leap-15.6/Dockerfile b/ci/opensuse-leap-15.6/Dockerfile index af753098de..a2b94c91c8 100644 --- a/ci/opensuse-leap-15.6/Dockerfile +++ b/ci/opensuse-leap-15.6/Dockerfile @@ -16,6 +16,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.6 gcc12-c++ \ git \ gzip \ + jq \ libopenssl-devel \ libpcap-devel \ make \ diff --git a/ci/opensuse-tumbleweed/Dockerfile b/ci/opensuse-tumbleweed/Dockerfile index f701475504..911f7af96b 100644 --- a/ci/opensuse-tumbleweed/Dockerfile +++ b/ci/opensuse-tumbleweed/Dockerfile @@ -22,6 +22,7 @@ RUN zypper refresh \ gcc-c++ \ git \ gzip \ + jq \ libopenssl-devel \ libpcap-devel \ make \ diff --git a/ci/ubuntu-20.04/Dockerfile b/ci/ubuntu-20.04/Dockerfile index 38a68aa196..6af6030de3 100644 --- a/ci/ubuntu-20.04/Dockerfile +++ b/ci/ubuntu-20.04/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update && apt-get -y install \ g++ \ gcc \ git \ + jq \ lcov \ libkrb5-dev \ libmaxminddb-dev \ diff --git a/ci/ubuntu-22.04/Dockerfile b/ci/ubuntu-22.04/Dockerfile index fd3659c00b..12c5290aab 100644 --- a/ci/ubuntu-22.04/Dockerfile +++ b/ci/ubuntu-22.04/Dockerfile @@ -17,6 +17,7 @@ RUN apt-get update && apt-get -y install \ g++ \ gcc \ git \ + jq \ lcov \ libkrb5-dev \ libmaxminddb-dev \ diff --git a/ci/ubuntu-24.04/Dockerfile b/ci/ubuntu-24.04/Dockerfile index 5aa2daa00e..ab7ad2cf25 100644 --- a/ci/ubuntu-24.04/Dockerfile +++ b/ci/ubuntu-24.04/Dockerfile @@ -19,6 +19,7 @@ RUN apt-get update && apt-get -y install \ g++ \ gcc \ git \ + jq \ lcov \ libkrb5-dev \ libmaxminddb-dev \ diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1..stdout b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1..stdout deleted file mode 100644 index 4fdec13f30..0000000000 --- a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1..stdout +++ /dev/null @@ -1,2 +0,0 @@ -### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -manager-1, Telemetry::metrics_port from cluster config, 1028/tcp diff --git a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1.services.out b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1.services.out index 29931892ad..ab13f24568 100644 --- a/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1.services.out +++ b/testing/btest/Baseline/scripts.policy.frameworks.telemetry.prometheus/manager-1.services.out @@ -1,5 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +endpoint="manager-1" +endpoint="logger-1" endpoint="proxy-1" - endpoint="worker-1" - diff --git a/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek b/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek index 605593a7ab..ff7155b267 100644 --- a/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek +++ b/testing/btest/scripts/policy/frameworks/telemetry/prometheus.zeek @@ -7,8 +7,10 @@ # @TEST-PORT: BROKER_PORT2 # @TEST-PORT: BROKER_PORT3 # @TEST-PORT: BROKER_PORT4 -# @TEST-PORT: BROKER_PORT4 -# @TEST-PORT: BROKER_TEST_METRICS_PORT +# @TEST-PORT: METRICS_PORT1 +# @TEST-PORT: METRICS_PORT2 +# @TEST-PORT: METRICS_PORT3 +# @TEST-PORT: METRICS_PORT4 # # @TEST-REQUIRES: which curl # @TEST-EXEC: zeek --parse-only %INPUT @@ -16,21 +18,20 @@ # @TEST-EXEC: btest-bg-run logger-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=logger-1 zeek -b %INPUT # @TEST-EXEC: btest-bg-run proxy-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=proxy-1 zeek -b %INPUT # @TEST-EXEC: btest-bg-run worker-1 ZEEKPATH=$ZEEKPATH:.. CLUSTER_NODE=worker-1 zeek -b %INPUT -# @TEST-EXEC: btest-bg-wait 10 -# @TEST-EXEC: btest-diff manager-1/.stdout +# @TEST-EXEC: btest-bg-wait 30 # @TEST-EXEC: btest-diff manager-1/services.out @TEST-START-FILE cluster-layout.zeek redef Cluster::nodes = { - ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1")), $metrics_port=1028/tcp], - ["logger-1"] = [$node_type=Cluster::LOGGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT2")), $manager="manager-1", $metrics_port=1029/tcp], - ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT3")), $manager="manager-1", $metrics_port=1030/tcp], - ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT4")), $manager="manager-1", $metrics_port=1031/tcp], + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT1")), $metrics_port=to_port(getenv("METRICS_PORT1"))], + ["logger-1"] = [$node_type=Cluster::LOGGER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT2")), $manager="manager-1", $metrics_port=to_port(getenv("METRICS_PORT2"))], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT3")), $manager="manager-1", $metrics_port=to_port(getenv("METRICS_PORT3"))], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=to_port(getenv("BROKER_PORT4")), $manager="manager-1", $metrics_port=to_port(getenv("METRICS_PORT4"))], }; @TEST-END-FILE @TEST-START-FILE request-services.sh -#!/bin/sh +#! /usr/bin/env bash # This script makes repeat curl requests to find all of the metrics data from the # hosts listed in the services output from the manager, and outputs it all into a @@ -39,34 +40,53 @@ redef Cluster::nodes = { services_url=$1 output_file=$2 -for host in $(curl -s -m 5 ${services_url} | jq '.[0].targets.[]'); do - host=$(echo ${host} | sed 's/"//g') - metrics=$(curl -s -m 5 http://${host}/metrics) - version_info=$(echo ${metrics} | grep -Eo "zeek_version_info\{[^}]+\}" | grep -o 'endpoint=\"[^"]*\"') - echo ${version_info} >> ${output_file}; +services_data=$(curl -s -m 5 ${services_url}) + +for host in $(echo ${services_data} | jq -r '.[0].targets[]' | sort); do + metrics=$(curl -m 5 --trace trace-${host}.out http://${host}/metrics) + if [ $? -eq 0 ] ; then + version_info=$(echo ${metrics} | grep -Eo "zeek_version_info\{[^}]+\}" | grep -o 'endpoint="[^"]*"') + echo ${version_info} >> ${output_file}; + else + echo "Failed to request data from ${host}" >> ${output_file} + fi done @TEST-END-FILE @load policy/frameworks/cluster/experimental @load policy/frameworks/telemetry/prometheus @load base/frameworks/telemetry -@load base/utils/active-http + +# So the cluster nodes don't terminate right away. +#redef exit_only_after_terminate=T; @if ( Cluster::node == "manager-1" ) -# Query the Prometheus endpoint using ActiveHTTP for testing, oh my. +@load base/utils/exec + +# Query the Prometheus endpoint using curl for testing, oh my. event run_test() { local services_url = fmt("http://localhost:%s/services.json", port_to_count(Telemetry::metrics_port)); - local result = system(fmt("sh ../request-services.sh %s %s", services_url, "services.out")); - if ( result != 0 ) + local req_cmd = fmt("sh ../request-services.sh %s %s", services_url, "services.out"); + + when [req_cmd] ( local result = Exec::run([$cmd=req_cmd]) ) + { + if ( result$exit_code != 0 ) + { + # This is bad. + print "ERROR: Failed to request service information"; + exit(1); + } + + terminate(); + } + timeout 10sec { # This is bad. - print "ERROR: Failed to request service information"; + print "ERROR: Timed out requesting service information"; exit(1); } - - terminate(); } # Use a dynamic metrics port for testing to avoid colliding on 9911/tcp