diff --git a/.cirrus.yml b/.cirrus.yml index 3f339dbde1..929fc75f2f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -11,11 +11,11 @@ btest_retries: &BTEST_RETRIES 2 memory: &MEMORY 16GB config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install --ccache +no_spicy_config: &NO_SPICY_CONFIG --build-type=release --disable-broker-tests --disable-spicy --prefix=$CIRRUS_WORKING_DIR/install --ccache static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install --ccache asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy --ccache ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy --ccache tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy --ccache -openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install --ccache resources_template: &RESOURCES_TEMPLATE cpu: *CPUS @@ -44,13 +44,6 @@ freebsd_environment: &FREEBSD_ENVIRONMENT ZEEK_CI_CPUS: 8 ZEEK_CI_BTEST_JOBS: 8 -sanitizers_resource_template: &SANITIZERS_RESOURCE_TEMPLATE - cpu: 4 - # Sanitizers uses a lot more memory than a typical config. - memory: 12GB - # For greediness, see https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4 - greedy: true - builds_only_if_template: &BUILDS_ONLY_IF_TEMPLATE # Rules for skipping builds: # - Do not run builds for anything that's cron triggered @@ -159,25 +152,22 @@ env: # a solution for the mtime pruning above. ZEEK_CCACHE_EPOCH: 2 - # Cache Spicy JIT results. - HILTI_CXX_COMPILER_LAUNCHER: ccache - # Linux EOL timelines: https://linuxlifecycle.com/ # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle +fedora39_task: + container: + # Fedora 39 EOL: Around Nov 2024 + dockerfile: ci/fedora-39/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + fedora38_task: container: # Fedora 38 EOL: Around May 2024 dockerfile: ci/fedora-38/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE - -fedora37_task: - container: - # Fedora 37 EOL: Around Dec 2024 - dockerfile: ci/fedora-37/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR centosstream9_task: @@ -202,14 +192,35 @@ centos7_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR + env: + ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG debian12_task: container: - # Debian 12 (bookworm) EOL: (not yet released) + # Debian 12 (bookworm) EOL: TBD + dockerfile: ci/debian-12/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + +arm_debian12_task: + arm_container: + # Debian 12 (bookworm) EOL: TBD + dockerfile: ci/debian-12/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG + +debian12_static_task: + container: + # Just use a recent/common distro to run a static compile test. + # Debian 12 (bookworm) EOL: TBD dockerfile: ci/debian-12/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR + env: + ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG debian11_task: container: @@ -217,24 +228,7 @@ debian11_task: dockerfile: ci/debian-11/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE - -arm_debian11_task: - arm_container: - # Debian 11 EOL: June 2026 - dockerfile: ci/debian-11/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE - -debian11_static_task: - container: - # Just use a recent/common distro to run a static compile test. - # Debian 11 EOL: June 2026 - dockerfile: ci/debian-11/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR - env: - ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG debian10_task: container: @@ -243,10 +237,10 @@ debian10_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE -opensuse_leap_15_4_task: +opensuse_leap_15_5_task: container: - # Opensuse Leap 15.4 EOL: ~Nov 2023 - dockerfile: ci/opensuse-leap-15.4/Dockerfile + # Opensuse Leap 15.5 EOL: ~Dec 2024 + dockerfile: ci/opensuse-leap-15.5/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE @@ -259,6 +253,13 @@ opensuse_tumbleweed_task: << : *CI_TEMPLATE # << : *SKIP_TASK_ON_PR +ubuntu23_task: + container: + # Ubuntu 23.10 EOL: July 2024 + dockerfile: ci/ubuntu-23.10/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + ubuntu22_task: container: # Ubuntu 22.04 EOL: April 2027 @@ -298,6 +299,13 @@ alpine_task: # Apple doesn't publish official long-term support timelines. # We aim to support both the current and previous macOS release. +macos_sonoma_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-sonoma-base:latest + prepare_script: ./ci/macos/prepare.sh + << : *CI_TEMPLATE + << : *MACOS_ENVIRONMENT + macos_ventura_task: macos_instance: image: ghcr.io/cirruslabs/macos-ventura-base:latest @@ -305,21 +313,11 @@ macos_ventura_task: << : *CI_TEMPLATE << : *MACOS_ENVIRONMENT -macos_monterey_task: - macos_instance: - image: ghcr.io/cirruslabs/macos-monterey-base:latest - prepare_script: ./ci/macos/prepare.sh - << : *CI_TEMPLATE - << : *MACOS_ENVIRONMENT - << : *SKIP_TASK_ON_PR - # FreeBSD EOL timelines: https://www.freebsd.org/security/#sup freebsd14_task: freebsd_instance: - # We don't support FreeBSD 14 yet, this is a purely informative task - image_family: freebsd-14-0-snap - allow_failures: true - skip_notification: true + # FreeBSD 14 EOL: Nov 30 2028 + image_family: freebsd-14-0 << : *FREEBSD_RESOURCES_TEMPLATE prepare_script: ./ci/freebsd/prepare.sh @@ -341,7 +339,7 @@ asan_sanitizer_task: container: # Just uses a recent/common distro to run memory error/leak checks. dockerfile: ci/ubuntu-20.04/Dockerfile - << : *SANITIZERS_RESOURCE_TEMPLATE + << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE test_fuzzers_script: ./ci/test-fuzzers.sh @@ -349,14 +347,13 @@ asan_sanitizer_task: env: CXXFLAGS: -DZEEK_DICT_DEBUG ZEEK_CI_CONFIGURE_FLAGS: *ASAN_SANITIZER_CONFIG - ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1 ASAN_OPTIONS: detect_leaks=1 ubsan_sanitizer_task: container: # Just uses a recent/common distro to run undefined behavior checks. dockerfile: ci/ubuntu-20.04/Dockerfile - << : *SANITIZERS_RESOURCE_TEMPLATE + << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR @@ -364,7 +361,6 @@ ubsan_sanitizer_task: env: CXXFLAGS: -DZEEK_DICT_DEBUG ZEEK_CI_CONFIGURE_FLAGS: *UBSAN_SANITIZER_CONFIG - ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1 ZEEK_TAILORED_UB_CHECKS: 1 UBSAN_OPTIONS: print_stacktrace=1 @@ -372,7 +368,7 @@ ubsan_sanitizer_task: # container: # # Just uses a recent/common distro to run memory error/leak checks. # dockerfile: ci/ubuntu-22.04/Dockerfile -# << : *SANITIZERS_RESOURCE_TEMPLATE +# << : *RESOURCES_TEMPLATE # << : *CI_TEMPLATE # << : *SKIP_TASK_ON_PR @@ -432,6 +428,7 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE CIRRUS_LOG_TIMESTAMP: true BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH} + BUILDKIT_PROGRESS: plain always: ccache_cache: @@ -630,7 +627,7 @@ cluster_testing_docker_builder: test_script: # Invoke btest directly here. This mirrors ci/test.sh, ensures we don't # accidentally build a Docker image, and enables console-level output: - - cd testing/external/zeek-testing-cluster && ../../../auxil/btest/btest -d -b -j ${ZEEK_CI_BTEST_JOBS} + - cd testing/external/zeek-testing-cluster && ../../../auxil/btest/btest -A -d -b -j ${ZEEK_CI_BTEST_JOBS} on_failure: upload_cluster_testing_artifacts: path: "testing/external/zeek-testing-cluster/.tmp/**" diff --git a/CHANGES b/CHANGES index 82ebcf543e..313a38009c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,134 @@ +6.0.2-42 | 2024-01-18 16:25:03 -0700 + + * CI: Remove unused openssl30_config (Tim Wojtulewicz, Corelight) + + (cherry picked from commit 652ba502aab843574402ec53aa0a6561b21253f3) + + * ci: Remove ZEEK_CI_DISABLE_SCRIPT_PROFILING logic (Arne Welzel, Corelight) + + To my knowledge this doesn't make a huge difference anymore and locally + I'm only using ASAN builds. It's not actually too slow. + + (cherry picked from commit 344369f169b872b8dcb569f089b02e026fe7cfc2) + + * CI: Move Debian variations from 11 to 12 (Christian Kreibich, Corelight) + + This switches the ARM and static builds from 11 to 12. x86 and ARM now build by + default, and 11 only with fullci. 10 remains as-is, because we like to torture + ourselves. + + (cherry picked from commit 4f6c25bb91b935666b6748b95c2b6ee55c1491ad) + + * CI: Add Ubuntu 23.10 (Christian Kreibich, Corelight) + + (cherry picked from commit 866d4b4127ccfd8904f8867b468babd330dd6d52) + + * CI: Drop openSUSE Leap 15.4, about to EOL. (Christian Kreibich, Corelight) + + (cherry picked from commit ddd56b9ad7171a9f8ee3de03bc362223f2fad0fc) + + * CI: FreeBSD 14 is out now, 12 is about to EOL. (Christian Kreibich, Corelight) + + (cherry picked from commit 18b8e6d6bceb4492d842d9bacbd3ad9dfbdd7702) + + * CI: distro EOL comment tweaks (Christian Kreibich, Corelight) + + (cherry picked from commit eb1c4bf019f6a39f1d76de717eee1b34cdc50378) + + * CI: drop Fedora 37, add Fedora 39 (Christian Kreibich, Corelight) + + This makes 39 the ony to run at all times, and 38 one to run only with fullci. + + (cherry picked from commit ee8cc77050e39aea18f2da144dc8108110bb42dc) + + * CI: Install missing packages on opensuse (Tim Wojtulewicz, Corelight) + + (cherry picked from commit a3ad1a24f11770d5b8c39af1671beb5d2b4c65b2) + + * CI: Use other base64 options on macOS Sonoma too (Tim Wojtulewicz, Corelight) + + (cherry picked from commit b1e63ffd268cdb2cd28098a0a8578a3038be8fd4) + + * CI: Add macOS Sonoma build, remove macOS Monterey build (Tim Wojtulewicz, Corelight) + + (cherry picked from commit 35ca1e88c7f1398c37bcbc4e20700a2aff2c4169) + + * Fix tests so they work both with GNU and BSD tools (Benjamin Bannier, Corelight) + + The GNU and BSD versions of `touch` and `truncate` allow slightly + different arguments, change the tests so they work in both versions. + + (cherry picked from commit caaffd0324ac0557ba4c5ce0290d07a4d013a735) + + * Install libmaxminddb in macOS CI (Benjamin Bannier, Corelight) + + (cherry picked from commit dfba2d6df707840cabf822d70354f9e2e635b863) + + * CI: Pass -A flag to btest for cluster-testing builds (Tim Wojtulewicz, Corelight) + + (cherry picked from commit ac7685c679cdb6d61768ac20449175eaaaacc34c) + + * CI: Add more logging during docker builds (Tim Wojtulewicz, Corelight) + + (cherry picked from commit 3d1380305beaf2e48e22444a285e1f64e4d120d0) + + * CI: disable Spicy on CentOS 7 (Christian Kreibich, Corelight) + + Based on 1774a25f0079076851232a81247a320ddc31b705 in master, which mainly + touched QUIC configuration that does not apply here just yet, hence not a + cherry-pick. + + * CI: Disable spicy on arm_debian11 and opensuse_leap_15_4 builds (Tim Wojtulewicz, Corelight) + + These two builds are struggling to complete due to OOM issues. Reducing the + number of CPUs didn't help, so this temporarily disables Spicy from + building on those hosts to reduce the memory usage. + + (cherry picked from commit bbf7f34cdc532da4a8fcdc72b8b12afa4f0fdd9c) + + * CI: Remove sanitizers resource template (Tim Wojtulewicz, Corelight) + + This template was added when the sanitizer builds needed more memory than all + of the other builds. We've since increased the amount of memory that the other + builds use beyond what's requested in this template, so it doesn't make sense + to keep it around. + + (cherry picked from commit 4c6c89d8636b249041bdaabbaaaa7b83154295b6) + + * Enable ccache-backed Spicy JIT for users of developer scripts. (Benjamin Bannier, Corelight) + + For CI and more general BTest execution we already enabled ccache + previously, but developers using a Spicy build out of the build + directory still needed to manually set up caching. + + With this patch we enable it for such cases as well via + `zeek-path-dev.*sh`. + + (cherry picked from commit e624c7500e442ed3b4595435019c46c69f57eac8) + + * Add OpenSUSE Leap 15.5 to ci (Johanna Amann, Corelight) + + This also adds 15.4 to the list of distributions that are skipped by + default - let me know if this is not a good idea. I assume that by now + most people will be running 15.5 (15.4 will be EOL in ~3 months). + + (cherry picked from commit 4ad2a6ab9a015595e643b572ae27756f038c77dc) + + * Run clang-format on the branch to fix some issues from patch merges (Tim Wojtulewicz, Corelight) + + * Enable darwin builds for zeek-security repo (Tim Wojtulewicz, Corelight) + + (cherry picked from commit c4edd8410e7bd863e5ff528dcc321758103f678d) + + * OCSP: Open-code unknown revoke reason strings (Arne Welzel, Corelight) + + OpenSSL 3.2.0 knows about more reasons. Add some backwards compatibility. + + Reference: https://github.com/openssl/openssl/commit/1c8a7f5091e2c5aebc043be86bcbedc6947e1c6f + (cherry picked from commit 02d00a19849d15f472b32a98a8fee27b20f2cb14) + + * Revert update to scripts.base.protocols.ssl.ocsp-revoked btest baseline (Tim Wojtulewicz) + 6.0.2-15 | 2024-01-12 13:33:19 -0700 * Bump Spicy to latest LTS release. (Benjamin Bannier, Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index c095e51b07..f62a78b461 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -525,7 +525,16 @@ file( "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n" "setenv ZEEK_SPICY_LIBRARY_PATH \"${cmake_source_dir}/scripts/spicy\"\n") +if (CMAKE_CXX_COMPILER_LAUNCHER) + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh + "export HILTI_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}\n") + + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh + "setenv HILTI_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}\n") +endif () + file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) + execute_process( COMMAND grep "^constexpr int PLUGIN_API_VERSION" INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/plugin/Plugin.h diff --git a/VERSION b/VERSION index d1eda9709d..3250bf71e6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.2-15 +6.0.2-42 diff --git a/ci/fedora-37/Dockerfile b/ci/fedora-39/Dockerfile similarity index 92% rename from ci/fedora-37/Dockerfile rename to ci/fedora-39/Dockerfile index 43335ff770..3b9177415a 100644 --- a/ci/fedora-37/Dockerfile +++ b/ci/fedora-39/Dockerfile @@ -1,8 +1,8 @@ -FROM fedora:37 +FROM fedora:39 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230816 +ENV DOCKERFILE_VERSION 20231208 RUN dnf -y install \ bison \ diff --git a/ci/init-external-repos.sh b/ci/init-external-repos.sh index ce2d3f4a6d..7f27f582be 100755 --- a/ci/init-external-repos.sh +++ b/ci/init-external-repos.sh @@ -51,8 +51,8 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ "${CIRRUS_REPO_OWNER}" == "zeek" ]] && [[ ! -d banner "Trying to clone zeek-testing-private git repo" echo "${ZEEK_TESTING_PRIVATE_SSH_KEY}" >cirrus_key.b64 - if [ "${CIRRUS_TASK_NAME}" == "macos_ventura" ]; then - # The base64 command provided with macOS Ventura requires an argument + if [ "${CIRRUS_TASK_NAME}" == "macos_ventura" -o "${CIRRUS_TASK_NAME}" == "macos_sonoma" ]; then + # The base64 command provided with macOS Ventura/Sonoma requires an argument # to pass the input filename base64 -d -i cirrus_key.b64 >cirrus_key else diff --git a/ci/macos/prepare.sh b/ci/macos/prepare.sh index 6d3909a90d..52c4472272 100755 --- a/ci/macos/prepare.sh +++ b/ci/macos/prepare.sh @@ -7,7 +7,7 @@ set -x brew update brew upgrade cmake -brew install openssl@3 swig bison flex ccache +brew install openssl@3 swig bison flex ccache libmaxminddb python3 -m pip install --user websockets # Brew doesn't create the /opt/homebrew/opt/openssl symlink if you install diff --git a/ci/opensuse-leap-15.4/Dockerfile b/ci/opensuse-leap-15.5/Dockerfile similarity index 51% rename from ci/opensuse-leap-15.4/Dockerfile rename to ci/opensuse-leap-15.5/Dockerfile index 6adc631fbf..605412a885 100644 --- a/ci/opensuse-leap-15.4/Dockerfile +++ b/ci/opensuse-leap-15.5/Dockerfile @@ -1,10 +1,10 @@ -FROM opensuse/leap:15.4 +FROM opensuse/leap:15.5 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230816 +ENV DOCKERFILE_VERSION 20230905 -RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.4:Update/standard/openSUSE:Leap:15.4:Update.repo \ +RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5:Update/standard/openSUSE:Leap:15.5:Update.repo \ && zypper refresh \ && zypper in -y \ bison \ @@ -12,23 +12,28 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.4 cmake \ curl \ flex \ - gcc10 \ - gcc10-c++ \ + gcc12 \ + gcc12-c++ \ git \ gzip \ libopenssl-devel \ libpcap-devel \ make \ - python3 \ - python3-devel \ - python3-pip \ + openssh \ + python311 \ + python311-devel \ + python311-pip \ swig \ tar \ which \ zlib-devel \ && rm -rf /var/cache/zypp +RUN update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.11 100 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 100 +RUN update-alternatives --install /usr/bin/python3-config python3-config /usr/bin/python3.11-config 100 + RUN pip3 install websockets junit2html -RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-10 100 -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 100 +RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 diff --git a/ci/opensuse-tumbleweed/Dockerfile b/ci/opensuse-tumbleweed/Dockerfile index 18996c0008..6318847801 100644 --- a/ci/opensuse-tumbleweed/Dockerfile +++ b/ci/opensuse-tumbleweed/Dockerfile @@ -25,9 +25,11 @@ RUN zypper refresh \ libopenssl-devel \ libpcap-devel \ make \ + openssh \ python3 \ python3-devel \ python3-pip \ + python3-websockets \ swig \ tar \ util-linux \ @@ -35,4 +37,4 @@ RUN zypper refresh \ zlib-devel \ && rm -rf /var/cache/zypp -RUN pip3 install --break-system-packages websockets junit2html +RUN pip3 install --break-system-packages junit2html diff --git a/ci/test.sh b/ci/test.sh index b643293911..310212ab00 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -59,13 +59,6 @@ function run_btests { pushd testing/btest - # Commenting out this line in btest.cfg causes the script profiling/coverage - # to be disabled. We do this for the sanitizer build right now because of a - # fairly significant performance bug when running tests. - if [ "${ZEEK_CI_DISABLE_SCRIPT_PROFILING}" = "1" ]; then - sed -i 's/^ZEEK_PROFILER_FILE/#ZEEK_PROFILER_FILE/g' btest.cfg - fi - ${BTEST} -z ${ZEEK_CI_BTEST_RETRIES} -d -A -x btest-results.xml -j ${ZEEK_CI_BTEST_JOBS} || result=1 make coverage prep_artifacts @@ -74,15 +67,6 @@ function run_btests { } function run_external_btests { - # Commenting out this line in btest.cfg causes the script profiling/coverage - # to be disabled. We do this for the sanitizer build right now because of a - # fairly significant performance bug when running tests. - if [ "${ZEEK_CI_DISABLE_SCRIPT_PROFILING}" = "1" ]; then - pushd testing/external - sed -i 's/^ZEEK_PROFILER_FILE/#ZEEK_PROFILER_FILE/g' subdir-btest.cfg - popd - fi - local zeek_testing_pid="" local zeek_testing_pid_private="" pushd testing/external/zeek-testing diff --git a/ci/ubuntu-23.10/Dockerfile b/ci/ubuntu-23.10/Dockerfile new file mode 100644 index 0000000000..de278f6f87 --- /dev/null +++ b/ci/ubuntu-23.10/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:23.10 + +ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" + +# A version field to invalidate Cirrus's build cache when needed, as suggested in +# https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 +ENV DOCKERFILE_VERSION 20230828 + +RUN apt-get update && apt-get -y install \ + bc \ + bison \ + bsdmainutils \ + ccache \ + cmake \ + curl \ + flex \ + g++ \ + gcc \ + git \ + libkrb5-dev \ + libmaxminddb-dev \ + libpcap-dev \ + libssl-dev \ + make \ + python3 \ + python3-dev \ + python3-pip \ + python3-websockets \ + ruby \ + sqlite3 \ + swig \ + unzip \ + wget \ + zlib1g-dev \ + && apt autoclean \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages junit2html diff --git a/docker/builder.Dockerfile b/docker/builder.Dockerfile index 0c1685ac2e..662f3ef3e7 100644 --- a/docker/builder.Dockerfile +++ b/docker/builder.Dockerfile @@ -3,6 +3,11 @@ # Layer to build Zeek. FROM debian:bookworm-slim +# Make the shell split commands in the log so we can determine reasons for +# failures more easily. +SHELL ["/bin/sh", "-x", "-c"] + +# Allow apt to retry 3 times before failing. RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries # Configure system for build. diff --git a/docker/final.Dockerfile b/docker/final.Dockerfile index 97e4d80599..eeab916a93 100644 --- a/docker/final.Dockerfile +++ b/docker/final.Dockerfile @@ -3,6 +3,11 @@ # Final layer containing all artifacts. FROM debian:bookworm-slim +# Make the shell split commands in the log so we can determine reasons for +# failures more easily. +SHELL ["/bin/sh", "-x", "-c"] + +# Allow apt to retry 3 times before failing. RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries RUN apt-get -q update \ diff --git a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log index 5ec6a3e3ec..e511eebbdc 100644 --- a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log +++ b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log @@ -3,15 +3,15 @@ ts level message location 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83 +1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 98 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84 +1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 99 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83 +1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 98 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84 +1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 99 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 diff --git a/testing/btest/core/mmdb/reopen.zeek b/testing/btest/core/mmdb/reopen.zeek index d9c120c8dc..265bb7b6c9 100644 --- a/testing/btest/core/mmdb/reopen.zeek +++ b/testing/btest/core/mmdb/reopen.zeek @@ -14,17 +14,23 @@ redef mmdb_dir = "./mmdb"; global pkt = 0; +function timestamp(n: count): string + { + return fmt("2020-01-01T00:%s:00", n); + } + event new_packet(c: connection, p: pkt_hdr) { ++pkt; - # Set MMDB's modification time to current network time. + + # Increment MMDB's modification time. local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); - if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), asn_fn), "") ) + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), asn_fn), "") ) exit(1); - if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), city_fn), "") ) + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), city_fn), "") ) exit(1); print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); diff --git a/testing/btest/core/mmdb/temporary-error.zeek b/testing/btest/core/mmdb/temporary-error.zeek index 0a5102535c..74b7ac4a16 100644 --- a/testing/btest/core/mmdb/temporary-error.zeek +++ b/testing/btest/core/mmdb/temporary-error.zeek @@ -15,16 +15,31 @@ redef mmdb_dir = "./mmdb"; global pkt = 0; +global asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); +global city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); + +global asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb"); +global city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb"); + +function timestamp(n: count): string + { + return fmt("2020-01-01T00:%s:00", n); + } + +event zeek_init() + { + # Set the initial modification time for the MMDBs. + for ( db in vector(asn_fn, city_fn, asn_fn_backup, city_fn_backup) ) + { + if ( ! piped_exec(fmt("test -f %s && touch -d %s %s", db, timestamp(pkt), db), "") ) + exit(1); + } + } + event new_packet(c: connection, p: pkt_hdr) { ++pkt; - local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); - local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); - - local asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb"); - local city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb"); - if ( pkt == 1 ) { print "start"; @@ -32,10 +47,10 @@ event new_packet(c: connection, p: pkt_hdr) if ( pkt == 2 ) { print "corrupting db"; - if ( ! piped_exec(fmt("truncate --size=8 %s", asn_fn), "") ) + if ( ! piped_exec(fmt("truncate -s 8 %s", asn_fn), "") ) exit(1); - if ( ! piped_exec(fmt("truncate --size=8 %s", city_fn), "") ) + if ( ! piped_exec(fmt("truncate -s 8 %s", city_fn), "") ) exit(1); } else if ( pkt == 4 ) @@ -73,11 +88,11 @@ event new_packet(c: connection, p: pkt_hdr) exit(1); } - # Set MMDB's modification time to current network time for predictability. - if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", asn_fn, network_time(), asn_fn), "") ) + # Increment MMDB's modification time. + if ( ! piped_exec(fmt("test -f %s && touch -d %s %s", asn_fn, timestamp(pkt), asn_fn), "") ) exit(1); - if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", city_fn, network_time(), city_fn), "") ) + if ( ! piped_exec(fmt("test -f %s && touch -d %s %s", city_fn, timestamp(pkt), city_fn), "") ) exit(1); print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1);