diff --git a/.cirrus.yml b/.cirrus.yml index 813835e56a..e3c89fd755 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,7 +3,7 @@ btest_jobs: &BTEST_JOBS 8 btest_retries: &BTEST_RETRIES 2 memory: &MEMORY 6GB -config: &CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests +config: &CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install memcheck_config: &MEMCHECK_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address --enable-fuzzers resources_template: &RESOURCES_TEMPLATE @@ -53,6 +53,16 @@ env: # the zeek-testing-private repository. ZEEK_TESTING_PRIVATE_SSH_KEY: ENCRYPTED[6631d7bf11e6553c531222953fb6de4d4a48a86a5dbc21a97604d5ca1791845718c985d9086f125cead6908e1b5f2b23] + # This is the key used to create HMAC auth keys for the benchmark script. This + # was generated by creating a new key using openssl, and then running sha256 + # on it. + ZEEK_BENCHMARK_HMAC_KEY: ENCRYPTED[412224bbea9652030da976537f4d96c79ee79a0ba5a2f93b6c32953e1be0362defdf5fa07b3dc54ae61f9a52be30eac7] + + # This is the https endpoint host and port used for benchmarking. It's kept + # encrypted as a security measure to avoid leaking the host's information. + ZEEK_BENCHMARK_HOST: ENCRYPTED[62ecdc93e839800d754d09d9a9070e9cb9b209e7d7dd2472ba38648f786ff272d0e0ea71233d0910025f2c6f3771259c] + ZEEK_BENCHMARK_PORT: ENCRYPTED[fb34ae2d51bac798fc01da052f3772154e17bbe2c1c5615509e82935248e748053fda399a0caf909632b6272cebff9f4] + # Linux EOL timelines: https://linuxlifecycle.com/ # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle @@ -70,6 +80,18 @@ centos7_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE +centos8_task: + container: + # CentOS 8 EOL: May 31, 2029 + dockerfile: ci/centos-8/Dockerfile + << : *RESOURCES_TEMPLATE + env: + ZEEK_CI_CREATE_ARTIFACT: 1 + << : *CI_TEMPLATE + upload_binary_artifacts: + path: build.tgz + benchmark_script: ./ci/benchmark.sh + debian9_task: container: # Debian 9 EOL: June 2022 diff --git a/ci/benchmark.sh b/ci/benchmark.sh new file mode 100755 index 0000000000..61584bdc07 --- /dev/null +++ b/ci/benchmark.sh @@ -0,0 +1,45 @@ +#! /usr/bin/env bash + +ZEEK_BENCHMARK_ENDPOINT="/zeek" + +# Setting this causes any command failures to immediately cause the script to fail. +set -e + +# Don't do this for any branch that isn't from the main zeek repo. +# TODO: is it possible to do this from cirrus.yml instead of here? +if [ "${CIRRUS_REPO_OWNER}" != "zeek" ]; then + echo "Benchmarks are skipped for repositories outside of the main Zeek project" + exit 0 +fi + +BUILD_URL="https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}/${CIRRUS_TASK_NAME}/upload_binary/build.tgz" + +# Generate an md5 hash of the build file. We can do this here because the path to the +# file still exists from the prior scripts. +BUILD_HASH=$(md5sum build.tgz | awk '{print $1}') + +# Generate an HMAC digest for the path plus a timestamp to send as an authentication +# header. Openssl outputs a hex string here so there's no need to base64 encode it. +# TODO: would it make sense to add the build hash as part of the hmac key here just +# for more uniqueness? +TIMESTAMP=$(date +'%s') +HMAC_DIGEST=$(echo "${ZEEK_BENCHMARK_ENDPOINT}-${TIMESTAMP}" | openssl dgst -sha256 -hmac ${ZEEK_BENCHMARK_HMAC_KEY} | awk '{print $2}') + +TARGET="https://${ZEEK_BENCHMARK_HOST}:${ZEEK_BENCHMARK_PORT}${ZEEK_BENCHMARK_ENDPOINT}" + +# Turn this back off because we want to be able to capture the output from curl if +# it fails. +set +e + +# Make a request to the benchmark host. +RESULTS=$(curl -sS --stderr - --fail --insecure -X POST -H "Zeek-HMAC: ${HMAC_DIGEST}" -H "Zeek-HMAC-Timestamp: ${TIMESTAMP}" "${TARGET}?branch=${CIRRUS_BRANCH}&build=${BUILD_URL}&build_hash=${BUILD_HASH}") +STATUS=$? + +# If we got a bad status back from the host, we want to make sure to mask the host +# and port from the output. +if [ $STATUS -ne 0 ]; then + RESULTS=$(echo "${RESULTS}" | sed "s/${ZEEK_BENCHMARK_HOST}//g" | sed "s/:${ZEEK_BENCHMARK_PORT}/:/g") +fi + +echo "$RESULTS" +exit $STATUS diff --git a/ci/build.sh b/ci/build.sh index df0aa66933..78616f1f2e 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -3,5 +3,11 @@ set -e set -x -./configure ${ZEEK_CI_CONFIGURE_FLAGS} -make -j ${ZEEK_CI_CPUS} +if [ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]; then + ./configure ${ZEEK_CI_CONFIGURE_FLAGS} + make -j ${ZEEK_CI_CPUS} +else + ./configure ${ZEEK_CI_CONFIGURE_FLAGS} --prefix=${CIRRUS_WORKING_DIR}/install + make -j ${ZEEK_CI_CPUS} install + tar -czf build.tgz ${CIRRUS_WORKING_DIR}/install +fi diff --git a/ci/centos-8/Dockerfile b/ci/centos-8/Dockerfile new file mode 100644 index 0000000000..a2be5dae3b --- /dev/null +++ b/ci/centos-8/Dockerfile @@ -0,0 +1,41 @@ +FROM centos:8 + +RUN dnf -y install epel-release dnf-plugins-core \ + && dnf clean all && rm -rf /var/cache/dnf + +RUN dnf config-manager --set-enabled PowerTools + +RUN dnf -y update && dnf -y install \ + git \ + cmake3 \ + make \ + gcc \ + gcc-c++ \ + flex \ + bison \ + swig \ + openssl \ + openssl-devel \ + libpcap-devel \ + python3 \ + python3-devel \ + python3-pip \ + zlib-devel \ + libsqlite3x-devel \ + findutils \ + which \ + && dnf clean all && rm -rf /var/cache/dnf + +# Many distros adhere to PEP 394's recommendation for `python` = `python2` so +# this is a simple workaround until we drop Python 2 support and explicitly +# use `python3` for all invocations (e.g. in shebangs). +RUN ln -sf /usr/bin/python3 /usr/local/bin/python +RUN ln -sf /usr/bin/pip3 /usr/local/bin/pip + +RUN pip install junit2html + +RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env + +ENV BASH_ENV="/usr/bin/zeek-ci-env" \ + ENV="/usr/bin/zeek-ci-env" \ + PROMPT_COMMAND=". /usr/bin/zeek-ci-env"