From 11ecc7d7ebfe951d7e67f40260636d1353043c3b Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Mon, 20 Apr 2020 17:16:04 -0700
Subject: [PATCH 1/2] Add new CI task for running benchmarks on a remote host

- Adds centos-8 docker instance to the CI configuration
- Adds new benchmark.sh script for passing a build artifact to a remote host to run benchmarks
---
 .cirrus.yml            | 24 +++++++++++++++++++++-
 ci/benchmark.sh        | 45 ++++++++++++++++++++++++++++++++++++++++++
 ci/build.sh            | 10 ++++++++--
 ci/centos-8/Dockerfile | 41 ++++++++++++++++++++++++++++++++++++++
 4 files changed, 117 insertions(+), 3 deletions(-)
 create mode 100755 ci/benchmark.sh
 create mode 100644 ci/centos-8/Dockerfile

diff --git a/.cirrus.yml b/.cirrus.yml
index 813835e56a..e3c89fd755 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -3,7 +3,7 @@ btest_jobs: &BTEST_JOBS 8
 btest_retries: &BTEST_RETRIES 2
 memory: &MEMORY 6GB
 
-config: &CONFIG --build-type=release --enable-cpp-tests  --disable-broker-tests
+config: &CONFIG --build-type=release --enable-cpp-tests  --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 memcheck_config: &MEMCHECK_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address --enable-fuzzers
 
 resources_template: &RESOURCES_TEMPLATE
@@ -53,6 +53,16 @@ env:
   # the zeek-testing-private repository.
   ZEEK_TESTING_PRIVATE_SSH_KEY: ENCRYPTED[6631d7bf11e6553c531222953fb6de4d4a48a86a5dbc21a97604d5ca1791845718c985d9086f125cead6908e1b5f2b23]
 
+  # This is the key used to create HMAC auth keys for the benchmark script. This
+  # was generated by creating a new key using openssl, and then running sha256
+  # on it.
+  ZEEK_BENCHMARK_HMAC_KEY: ENCRYPTED[412224bbea9652030da976537f4d96c79ee79a0ba5a2f93b6c32953e1be0362defdf5fa07b3dc54ae61f9a52be30eac7]
+
+  # This is the https endpoint host and port used for benchmarking. It's kept
+  # encrypted as a security measure to avoid leaking the host's information.
+  ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c]
+  ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4]
+
 # Linux EOL timelines: https://linuxlifecycle.com/
 # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
 
@@ -70,6 +80,18 @@ centos7_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+centos8_task:
+  container:
+    # CentOS 8 EOL: May 31, 2029
+    dockerfile: ci/centos-8/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  env:
+    ZEEK_CI_CREATE_ARTIFACT: 1
+  << : *CI_TEMPLATE
+  upload_binary_artifacts:
+    path: build.tgz
+  benchmark_script: ./ci/benchmark.sh
+
 debian9_task:
   container:
     # Debian 9 EOL: June 2022
diff --git a/ci/benchmark.sh b/ci/benchmark.sh
new file mode 100755
index 0000000000..61584bdc07
--- /dev/null
+++ b/ci/benchmark.sh
@@ -0,0 +1,45 @@
+#! /usr/bin/env bash
+
+ZEEK_BENCHMARK_ENDPOINT="/zeek"
+
+# Setting this causes any command failures to immediately cause the script to fail.
+set -e
+
+# Don't do this for any branch that isn't from the main zeek repo.
+# TODO: is it possible to do this from cirrus.yml instead of here?
+if [ "${CIRRUS_REPO_OWNER}" != "zeek" ]; then
+    echo "Benchmarks are skipped for repositories outside of the main Zeek project"
+    exit 0
+fi
+
+BUILD_URL="https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/${CIRRUS_TASK_NAME}/upload_binary/build.tgz"
+
+# Generate an md5 hash of the build file. We can do this here because the path to the
+# file still exists from the prior scripts.
+BUILD_HASH=$(md5sum build.tgz | awk '{print $1}')
+
+# Generate an HMAC digest for the path plus a timestamp to send as an authentication
+# header. Openssl outputs a hex string here so there's no need to base64 encode it.
+# TODO: would it make sense to add the build hash as part of the hmac key here just
+# for more uniqueness?
+TIMESTAMP=$(date +'%s')
+HMAC_DIGEST=$(echo "${ZEEK_BENCHMARK_ENDPOINT}-${TIMESTAMP}" | openssl dgst -sha256 -hmac ${ZEEK_BENCHMARK_HMAC_KEY} | awk '{print $2}')
+
+TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_ENDPOINT}"
+
+# Turn this back off because we want to be able to capture the output from curl if
+# it fails.
+set +e
+
+# Make a request to the benchmark host.
+RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}")
+STATUS=$?
+
+# If we got a bad status back from the host, we want to make sure to mask the host
+# and port from the output.
+if [ $STATUS -ne 0 ]; then
+    RESULTS=$(echo "${RESULTS}" | sed "s/${ZEEK_BENCHMARK_HOST}/<secret>/g" | sed "s/:${ZEEK_BENCHMARK_PORT}/:<secret>/g")
+fi
+
+echo "$RESULTS"
+exit $STATUS
diff --git a/ci/build.sh b/ci/build.sh
index df0aa66933..78616f1f2e 100755
--- a/ci/build.sh
+++ b/ci/build.sh
@@ -3,5 +3,11 @@
 set -e
 set -x
 
-./configure ${ZEEK_CI_CONFIGURE_FLAGS}
-make -j ${ZEEK_CI_CPUS}
+if [ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]; then
+    ./configure ${ZEEK_CI_CONFIGURE_FLAGS}
+    make -j ${ZEEK_CI_CPUS}
+else
+    ./configure ${ZEEK_CI_CONFIGURE_FLAGS} --prefix=${CIRRUS_WORKING_DIR}/install
+    make -j ${ZEEK_CI_CPUS} install
+    tar -czf build.tgz ${CIRRUS_WORKING_DIR}/install
+fi
diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile
new file mode 100644
index 0000000000..a2be5dae3b
--- /dev/null
+++ b/ci/centos-8/Dockerfile
@@ -0,0 +1,41 @@
+FROM centos:8
+
+RUN dnf -y install epel-release dnf-plugins-core \
+  && dnf clean all && rm -rf /var/cache/dnf
+
+RUN dnf config-manager --set-enabled PowerTools
+
+RUN dnf -y update && dnf -y install \
+    git \
+    cmake3 \
+    make \
+    gcc \
+    gcc-c++ \
+    flex \
+    bison \
+    swig \
+    openssl \
+    openssl-devel \
+    libpcap-devel \
+    python3 \
+    python3-devel \
+    python3-pip \
+    zlib-devel \
+    libsqlite3x-devel \
+    findutils \
+    which \
+  && dnf clean all && rm -rf /var/cache/dnf
+
+# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+# this is a simple workaround until we drop Python 2 support and explicitly
+# use `python3` for all invocations (e.g. in shebangs).
+RUN ln -sf /usr/bin/python3 /usr/local/bin/python
+RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
+
+RUN pip install junit2html
+
+RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env
+
+ENV BASH_ENV="/usr/bin/zeek-ci-env" \
+    ENV="/usr/bin/zeek-ci-env" \
+    PROMPT_COMMAND=". /usr/bin/zeek-ci-env"

From 70d2f63b3f42edd6e1f8a8c570f7e8eec95d43bf Mon Sep 17 00:00:00 2001
From: Tim Wojtulewicz <tim@corelight.com>
Date: Tue, 9 Jun 2020 16:39:18 -0700
Subject: [PATCH 2/2] Review cleanup

- Use sha256 for build file hash
- Use build file hash as part of the data for the HMAC digest
- Remove a few unnecessary lines from the centos8 dockerfile
- Pass timestamp in UTC
---
 ci/benchmark.sh        | 8 +++-----
 ci/centos-8/Dockerfile | 6 ------
 2 files changed, 3 insertions(+), 11 deletions(-)

diff --git a/ci/benchmark.sh b/ci/benchmark.sh
index 61584bdc07..2719d45b76 100755
--- a/ci/benchmark.sh
+++ b/ci/benchmark.sh
@@ -16,14 +16,12 @@ BUILD_URL="https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/${CIRR
 
 # Generate an md5 hash of the build file. We can do this here because the path to the
 # file still exists from the prior scripts.
-BUILD_HASH=$(md5sum build.tgz | awk '{print $1}')
+BUILD_HASH=$(sha256sum build.tgz | awk '{print $1}')
 
 # Generate an HMAC digest for the path plus a timestamp to send as an authentication
 # header. Openssl outputs a hex string here so there's no need to base64 encode it.
-# TODO: would it make sense to add the build hash as part of the hmac key here just
-# for more uniqueness?
-TIMESTAMP=$(date +'%s')
-HMAC_DIGEST=$(echo "${ZEEK_BENCHMARK_ENDPOINT}-${TIMESTAMP}" | openssl dgst -sha256 -hmac ${ZEEK_BENCHMARK_HMAC_KEY} | awk '{print $2}')
+TIMESTAMP=$(date -u +'%s')
+HMAC_DIGEST=$(echo "${ZEEK_BENCHMARK_ENDPOINT}-${TIMESTAMP}-${BUILD_HASH}" | openssl dgst -sha256 -hmac ${ZEEK_BENCHMARK_HMAC_KEY} | awk '{print $2}')
 
 TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_ENDPOINT}"
 
diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile
index a2be5dae3b..9a9df30a1e 100644
--- a/ci/centos-8/Dockerfile
+++ b/ci/centos-8/Dockerfile
@@ -33,9 +33,3 @@ RUN ln -sf /usr/bin/python3 /usr/local/bin/python
 RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
 
 RUN pip install junit2html
-
-RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env
-
-ENV BASH_ENV="/usr/bin/zeek-ci-env" \
-    ENV="/usr/bin/zeek-ci-env" \
-    PROMPT_COMMAND=". /usr/bin/zeek-ci-env"