Merge remote-tracking branch 'origin/topic/timw/ci-benchmark'

* origin/topic/timw/ci-benchmark:
  Review cleanup
  Add new CI task for running benchmarks on a remote host

.cirrus.yml

@@ -3,7 +3,7 @@ btest_jobs: &BTEST_JOBS 4
 btest_retries: &BTEST_RETRIES 2
 memory: &MEMORY 4GB
 
-config: &CONFIG --build-type=release --enable-cpp-tests  --disable-broker-tests
+config: &CONFIG --build-type=release --enable-cpp-tests  --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 memcheck_config: &MEMCHECK_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address --enable-fuzzers
 
 resources_template: &RESOURCES_TEMPLATE
@@ -57,6 +57,16 @@ env:
   # the zeek-testing-private repository.
   ZEEK_TESTING_PRIVATE_SSH_KEY: ENCRYPTED[!dbdba93df9c166f926480cebff52dab303589257b3b3ee53aa392021aff2881ed9aafefef26aa9a1b71a49d663d1361c!]
 
+  # This is the key used to create HMAC auth keys for the benchmark script. This
+  # was generated by creating a new key using openssl, and then running sha256
+  # on it.
+  ZEEK_BENCHMARK_HMAC_KEY: ENCRYPTED[412224bbea9652030da976537f4d96c79ee79a0ba5a2f93b6c32953e1be0362defdf5fa07b3dc54ae61f9a52be30eac7]
+
+  # This is the https endpoint host and port used for benchmarking. It's kept
+  # encrypted as a security measure to avoid leaking the host's information.
+  ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c]
+  ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4]
+
 # Linux EOL timelines: https://linuxlifecycle.com/
 # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle
 
@@ -74,6 +84,18 @@ centos7_task:
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
 
+centos8_task:
+  container:
+    # CentOS 8 EOL: May 31, 2029
+    dockerfile: ci/centos-8/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  env:
+    ZEEK_CI_CREATE_ARTIFACT: 1
+  << : *CI_TEMPLATE
+  upload_binary_artifacts:
+    path: build.tgz
+  benchmark_script: ./ci/benchmark.sh
+
 debian9_task:
   container:
     # Debian 9 EOL: June 2022

CHANGES

@@ -1,4 +1,8 @@
 
+3.2.0-dev.787 | 2020-06-15 10:27:48 -0700
+
+  * Add new CI task for running benchmarks on a remote host (Tim Wojtulewicz, Corelight)
+
 3.2.0-dev.783 | 2020-06-11 23:21:41 -0700
 
   * Compare pcap_next_ex() result to PCAP_ERROR/PCAP_ERROR_BREAK (Jon Siwek, Corelight)

VERSION

@@ -1 +1 @@
-3.2.0-dev.783
+3.2.0-dev.787

ci/benchmark.sh

@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+
+ZEEK_BENCHMARK_ENDPOINT="/zeek"
+
+# Setting this causes any command failures to immediately cause the script to fail.
+set -e
+
+# Don't do this for any branch that isn't from the main zeek repo.
+# TODO: is it possible to do this from cirrus.yml instead of here?
+if [ "${CIRRUS_REPO_OWNER}" != "zeek" ]; then
+    echo "Benchmarks are skipped for repositories outside of the main Zeek project"
+    exit 0
+fi
+
+BUILD_URL="https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/${CIRRUS_TASK_NAME}/upload_binary/build.tgz"
+
+# Generate an md5 hash of the build file. We can do this here because the path to the
+# file still exists from the prior scripts.
+BUILD_HASH=$(sha256sum build.tgz | awk '{print $1}')
+
+# Generate an HMAC digest for the path plus a timestamp to send as an authentication
+# header. Openssl outputs a hex string here so there's no need to base64 encode it.
+TIMESTAMP=$(date -u +'%s')
+HMAC_DIGEST=$(echo "${ZEEK_BENCHMARK_ENDPOINT}-${TIMESTAMP}-${BUILD_HASH}" | openssl dgst -sha256 -hmac ${ZEEK_BENCHMARK_HMAC_KEY} | awk '{print $2}')
+
+TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_ENDPOINT}"
+
+# Turn this back off because we want to be able to capture the output from curl if
+# it fails.
+set +e
+
+# Make a request to the benchmark host.
+RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}")
+STATUS=$?
+
+# If we got a bad status back from the host, we want to make sure to mask the host
+# and port from the output.
+if [ $STATUS -ne 0 ]; then
+    RESULTS=$(echo "${RESULTS}" | sed "s/${ZEEK_BENCHMARK_HOST}/<secret>/g" | sed "s/:${ZEEK_BENCHMARK_PORT}/:<secret>/g")
+fi
+
+echo "$RESULTS"
+exit $STATUS

ci/build.sh

@@ -3,6 +3,13 @@
 set -e
 set -x
 
-./configure ${ZEEK_CI_CONFIGURE_FLAGS}
+if [ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]; then
-cd build
+    ./configure ${ZEEK_CI_CONFIGURE_FLAGS}
-make -j ${ZEEK_CI_CPUS}
+    cd build
+    make -j ${ZEEK_CI_CPUS}
+else
+    ./configure ${ZEEK_CI_CONFIGURE_FLAGS} --prefix=${CIRRUS_WORKING_DIR}/install
+    cd build
+    make -j ${ZEEK_CI_CPUS} install
+    tar -czf build.tgz ${CIRRUS_WORKING_DIR}/install
+fi

ci/centos-8/Dockerfile

@@ -0,0 +1,35 @@
+FROM centos:8
+
+RUN dnf -y install epel-release dnf-plugins-core \
+  && dnf clean all && rm -rf /var/cache/dnf
+
+RUN dnf config-manager --set-enabled PowerTools
+
+RUN dnf -y update && dnf -y install \
+    git \
+    cmake3 \
+    make \
+    gcc \
+    gcc-c++ \
+    flex \
+    bison \
+    swig \
+    openssl \
+    openssl-devel \
+    libpcap-devel \
+    python3 \
+    python3-devel \
+    python3-pip \
+    zlib-devel \
+    libsqlite3x-devel \
+    findutils \
+    which \
+  && dnf clean all && rm -rf /var/cache/dnf
+
+# Many distros adhere to PEP 394's recommendation for `python` = `python2` so
+# this is a simple workaround until we drop Python 2 support and explicitly
+# use `python3` for all invocations (e.g. in shebangs).
+RUN ln -sf /usr/bin/python3 /usr/local/bin/python
+RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip
+
+RUN pip install junit2html