From 0388090f08a66cb2d0afdec1c036a934c97471ed Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 5 Sep 2023 15:23:00 +0100 Subject: [PATCH 01/20] Add OpenSUSE Leap 15.5 to ci This also adds 15.4 to the list of distributions that are skipped by default - let me know if this is not a good idea. I assume that by now most people will be running 15.5 (15.4 will be EOL in ~3 months). (cherry picked from commit 4ad2a6ab9a015595e643b572ae27756f038c77dc) --- .cirrus.yml | 8 +++++++ ci/opensuse-leap-15.5/Dockerfile | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 ci/opensuse-leap-15.5/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml index 3f339dbde1..8c9adee203 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -249,6 +249,14 @@ opensuse_leap_15_4_task: dockerfile: ci/opensuse-leap-15.4/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE + << : *SKIP_TASK_ON_PR + +opensuse_leap_15_5_task: + container: + # Opensuse Leap 15.5 EOL: ~Dec 2024 + dockerfile: ci/opensuse-leap-15.5/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE opensuse_tumbleweed_task: container: diff --git a/ci/opensuse-leap-15.5/Dockerfile b/ci/opensuse-leap-15.5/Dockerfile new file mode 100644 index 0000000000..3fee52ef48 --- /dev/null +++ b/ci/opensuse-leap-15.5/Dockerfile @@ -0,0 +1,38 @@ +FROM opensuse/leap:15.5 + +# A version field to invalidate Cirrus's build cache when needed, as suggested in +# https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 +ENV DOCKERFILE_VERSION 20230905 + +RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5:Update/standard/openSUSE:Leap:15.5:Update.repo \ + && zypper refresh \ + && zypper in -y \ + bison \ + ccache \ + cmake \ + curl \ + flex \ + gcc12 \ + gcc12-c++ \ + git \ + gzip \ + libopenssl-devel \ + libpcap-devel \ + make \ + python311 \ + python311-devel \ + python311-pip \ + swig \ + tar \ + which \ + zlib-devel \ + && rm -rf /var/cache/zypp + +RUN update-alternatives --install /usr/bin/pip3 pip3 /usr/bin/pip3.11 100 +RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 100 +RUN update-alternatives --install /usr/bin/python3-config python3-config /usr/bin/python3.11-config 100 + +RUN pip3 install websockets junit2html + +RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-12 100 +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-12 100 From 0122d9d934d3412197d67c4c9dc9212ea56b873d Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Mon, 2 Oct 2023 13:46:15 +0200 Subject: [PATCH 02/20] Enable ccache-backed Spicy JIT for users of developer scripts. For CI and more general BTest execution we already enabled ccache previously, but developers using a Spicy build out of the build directory still needed to manually set up caching. With this patch we enable it for such cases as well via `zeek-path-dev.*sh`. (cherry picked from commit e624c7500e442ed3b4595435019c46c69f57eac8) --- .cirrus.yml | 3 --- CMakeLists.txt | 9 +++++++++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 8c9adee203..ef7ee015d6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -159,9 +159,6 @@ env: # a solution for the mtime pruning above. ZEEK_CCACHE_EPOCH: 2 - # Cache Spicy JIT results. - HILTI_CXX_COMPILER_LAUNCHER: ccache - # Linux EOL timelines: https://linuxlifecycle.com/ # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle diff --git a/CMakeLists.txt b/CMakeLists.txt index c095e51b07..f62a78b461 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -525,7 +525,16 @@ file( "setenv HILTI_CXX_INCLUDE_DIRS \"`${cmake_binary_dir}/hilti-cxx-include-dirs`\"\n" "setenv ZEEK_SPICY_LIBRARY_PATH \"${cmake_source_dir}/scripts/spicy\"\n") +if (CMAKE_CXX_COMPILER_LAUNCHER) + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh + "export HILTI_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}\n") + + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh + "setenv HILTI_CXX_COMPILER_LAUNCHER=${CMAKE_CXX_COMPILER_LAUNCHER}\n") +endif () + file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) + execute_process( COMMAND grep "^constexpr int PLUGIN_API_VERSION" INPUT_FILE ${CMAKE_CURRENT_SOURCE_DIR}/src/plugin/Plugin.h From a6096cf75135bc609beb754ed8372dbaa244b721 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 2 Oct 2023 14:13:49 -0700 Subject: [PATCH 03/20] CI: Remove sanitizers resource template This template was added when the sanitizer builds needed more memory than all of the other builds. We've since increased the amount of memory that the other builds use beyond what's requested in this template, so it doesn't make sense to keep it around. (cherry picked from commit 4c6c89d8636b249041bdaabbaaaa7b83154295b6) --- .cirrus.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index ef7ee015d6..d4d9516980 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -44,13 +44,6 @@ freebsd_environment: &FREEBSD_ENVIRONMENT ZEEK_CI_CPUS: 8 ZEEK_CI_BTEST_JOBS: 8 -sanitizers_resource_template: &SANITIZERS_RESOURCE_TEMPLATE - cpu: 4 - # Sanitizers uses a lot more memory than a typical config. - memory: 12GB - # For greediness, see https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4 - greedy: true - builds_only_if_template: &BUILDS_ONLY_IF_TEMPLATE # Rules for skipping builds: # - Do not run builds for anything that's cron triggered @@ -346,7 +339,7 @@ asan_sanitizer_task: container: # Just uses a recent/common distro to run memory error/leak checks. dockerfile: ci/ubuntu-20.04/Dockerfile - << : *SANITIZERS_RESOURCE_TEMPLATE + << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE test_fuzzers_script: ./ci/test-fuzzers.sh @@ -361,7 +354,7 @@ ubsan_sanitizer_task: container: # Just uses a recent/common distro to run undefined behavior checks. dockerfile: ci/ubuntu-20.04/Dockerfile - << : *SANITIZERS_RESOURCE_TEMPLATE + << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR @@ -377,7 +370,7 @@ ubsan_sanitizer_task: # container: # # Just uses a recent/common distro to run memory error/leak checks. # dockerfile: ci/ubuntu-22.04/Dockerfile -# << : *SANITIZERS_RESOURCE_TEMPLATE +# << : *RESOURCES_TEMPLATE # << : *CI_TEMPLATE # << : *SKIP_TASK_ON_PR From 23742165f342c17abe26c4a7184c8687c256473d Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 2 Oct 2023 14:20:51 -0700 Subject: [PATCH 04/20] CI: Disable spicy on arm_debian11 and opensuse_leap_15_4 builds These two builds are struggling to complete due to OOM issues. Reducing the number of CPUs didn't help, so this temporarily disables Spicy from building on those hosts to reduce the memory usage. (cherry picked from commit bbf7f34cdc532da4a8fcdc72b8b12afa4f0fdd9c) --- .cirrus.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index d4d9516980..4ab55c5ab3 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -11,6 +11,7 @@ btest_retries: &BTEST_RETRIES 2 memory: &MEMORY 16GB config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install --ccache +no_spicy_config: &NO_SPICY_CONFIG --build-type=release --disable-broker-tests --disable-spicy --prefix=$CIRRUS_WORKING_DIR/install --ccache static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install --ccache asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy --ccache ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy --ccache @@ -214,6 +215,8 @@ arm_debian11_task: dockerfile: ci/debian-11/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG debian11_static_task: container: @@ -240,6 +243,8 @@ opensuse_leap_15_4_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR + env: + ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG opensuse_leap_15_5_task: container: From 63a771103a399da5401d68fc0d1501e8f25a2cc1 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 17 Jan 2024 12:25:21 -0800 Subject: [PATCH 05/20] CI: disable Spicy on CentOS 7 Based on 1774a25f0079076851232a81247a320ddc31b705 in master, which mainly touched QUIC configuration that does not apply here just yet, hence not a cherry-pick. --- .cirrus.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 4ab55c5ab3..739d4feca6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -193,6 +193,8 @@ centos7_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR + env: + ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG debian12_task: container: From 80085ba018e685476476f03ea9d8325011c66769 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Thu, 12 Oct 2023 15:00:52 -0700 Subject: [PATCH 06/20] CI: Add more logging during docker builds (cherry picked from commit 3d1380305beaf2e48e22444a285e1f64e4d120d0) --- .cirrus.yml | 1 + docker/builder.Dockerfile | 5 +++++ docker/final.Dockerfile | 5 +++++ 3 files changed, 11 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 739d4feca6..0b334b7955 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -437,6 +437,7 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE CIRRUS_LOG_TIMESTAMP: true BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH} + BUILDKIT_PROGRESS: plain always: ccache_cache: diff --git a/docker/builder.Dockerfile b/docker/builder.Dockerfile index 0c1685ac2e..662f3ef3e7 100644 --- a/docker/builder.Dockerfile +++ b/docker/builder.Dockerfile @@ -3,6 +3,11 @@ # Layer to build Zeek. FROM debian:bookworm-slim +# Make the shell split commands in the log so we can determine reasons for +# failures more easily. +SHELL ["/bin/sh", "-x", "-c"] + +# Allow apt to retry 3 times before failing. RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries # Configure system for build. diff --git a/docker/final.Dockerfile b/docker/final.Dockerfile index 97e4d80599..eeab916a93 100644 --- a/docker/final.Dockerfile +++ b/docker/final.Dockerfile @@ -3,6 +3,11 @@ # Final layer containing all artifacts. FROM debian:bookworm-slim +# Make the shell split commands in the log so we can determine reasons for +# failures more easily. +SHELL ["/bin/sh", "-x", "-c"] + +# Allow apt to retry 3 times before failing. RUN echo 'Acquire::Retries "3";' > /etc/apt/apt.conf.d/80-retries RUN apt-get -q update \ From fe7a21874d0f74c5edf547824b7a59f2cec76711 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Tue, 24 Oct 2023 15:42:21 -0700 Subject: [PATCH 07/20] CI: Pass -A flag to btest for cluster-testing builds (cherry picked from commit ac7685c679cdb6d61768ac20449175eaaaacc34c) --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 0b334b7955..4ef493ef79 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -636,7 +636,7 @@ cluster_testing_docker_builder: test_script: # Invoke btest directly here. This mirrors ci/test.sh, ensures we don't # accidentally build a Docker image, and enables console-level output: - - cd testing/external/zeek-testing-cluster && ../../../auxil/btest/btest -d -b -j ${ZEEK_CI_BTEST_JOBS} + - cd testing/external/zeek-testing-cluster && ../../../auxil/btest/btest -A -d -b -j ${ZEEK_CI_BTEST_JOBS} on_failure: upload_cluster_testing_artifacts: path: "testing/external/zeek-testing-cluster/.tmp/**" From 279372b6f9bbe26ea4d66d379a3153ae781fadaf Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 26 Oct 2023 14:47:59 +0200 Subject: [PATCH 08/20] Install libmaxminddb in macOS CI (cherry picked from commit dfba2d6df707840cabf822d70354f9e2e635b863) --- ci/macos/prepare.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/macos/prepare.sh b/ci/macos/prepare.sh index 6d3909a90d..52c4472272 100755 --- a/ci/macos/prepare.sh +++ b/ci/macos/prepare.sh @@ -7,7 +7,7 @@ set -x brew update brew upgrade cmake -brew install openssl@3 swig bison flex ccache +brew install openssl@3 swig bison flex ccache libmaxminddb python3 -m pip install --user websockets # Brew doesn't create the /opt/homebrew/opt/openssl symlink if you install From b169f56098c2a8b07f9d77832959b65b812ed18a Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Thu, 26 Oct 2023 15:13:02 +0200 Subject: [PATCH 09/20] Fix tests so they work both with GNU and BSD tools The GNU and BSD versions of `touch` and `truncate` allow slightly different arguments, change the tests so they work in both versions. (cherry picked from commit caaffd0324ac0557ba4c5ce0290d07a4d013a735) --- .../core.mmdb.temporary-error/reporter.log | 8 ++-- testing/btest/core/mmdb/reopen.zeek | 12 ++++-- testing/btest/core/mmdb/temporary-error.zeek | 37 +++++++++++++------ 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log index 5ec6a3e3ec..e511eebbdc 100644 --- a/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log +++ b/testing/btest/Baseline/core.mmdb.temporary-error/reporter.log @@ -3,15 +3,15 @@ ts level message location 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-ASN.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83 +1299470395.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 98 1299470395.000000 Reporter::INFO Modification time change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 1299470395.000000 Reporter::INFO Failed to open MaxMind DB: .<...>/GeoLite2-City.mmdb [The MaxMind DB file contains invalid metadata] , line 1 -1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84 +1299470395.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 99 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 83 +1299473995.000000 Reporter::ERROR Failed to open GeoIP ASN database (lookup_autonomous_system(128.3.0.1)) <...>/temporary-error.zeek, line 98 1299473995.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 -1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 84 +1299473995.000000 Reporter::ERROR Failed to open GeoIP location database (lookup_location(128.3.0.1)) <...>/temporary-error.zeek, line 99 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Closing stale MaxMind DB [.<...>/GeoLite2-ASN.mmdb] , line 1 1299477595.000000 Reporter::INFO Inode change detected for MaxMind DB [.<...>/GeoLite2-City.mmdb] , line 1 diff --git a/testing/btest/core/mmdb/reopen.zeek b/testing/btest/core/mmdb/reopen.zeek index d9c120c8dc..265bb7b6c9 100644 --- a/testing/btest/core/mmdb/reopen.zeek +++ b/testing/btest/core/mmdb/reopen.zeek @@ -14,17 +14,23 @@ redef mmdb_dir = "./mmdb"; global pkt = 0; +function timestamp(n: count): string + { + return fmt("2020-01-01T00:%s:00", n); + } + event new_packet(c: connection, p: pkt_hdr) { ++pkt; - # Set MMDB's modification time to current network time. + + # Increment MMDB's modification time. local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); - if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), asn_fn), "") ) + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), asn_fn), "") ) exit(1); - if ( ! piped_exec(fmt("touch -d @%s %s", network_time(), city_fn), "") ) + if ( ! piped_exec(fmt("touch -d %s %s", timestamp(pkt), city_fn), "") ) exit(1); print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); diff --git a/testing/btest/core/mmdb/temporary-error.zeek b/testing/btest/core/mmdb/temporary-error.zeek index 0a5102535c..74b7ac4a16 100644 --- a/testing/btest/core/mmdb/temporary-error.zeek +++ b/testing/btest/core/mmdb/temporary-error.zeek @@ -15,16 +15,31 @@ redef mmdb_dir = "./mmdb"; global pkt = 0; +global asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); +global city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); + +global asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb"); +global city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb"); + +function timestamp(n: count): string + { + return fmt("2020-01-01T00:%s:00", n); + } + +event zeek_init() + { + # Set the initial modification time for the MMDBs. + for ( db in vector(asn_fn, city_fn, asn_fn_backup, city_fn_backup) ) + { + if ( ! piped_exec(fmt("test -f %s && touch -d %s %s", db, timestamp(pkt), db), "") ) + exit(1); + } + } + event new_packet(c: connection, p: pkt_hdr) { ++pkt; - local asn_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-ASN.mmdb"); - local city_fn = safe_shell_quote(mmdb_dir + "/GeoLite2-City.mmdb"); - - local asn_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-ASN.mmdb"); - local city_fn_backup = safe_shell_quote(mmdb_dir + "-backup/GeoLite2-City.mmdb"); - if ( pkt == 1 ) { print "start"; @@ -32,10 +47,10 @@ event new_packet(c: connection, p: pkt_hdr) if ( pkt == 2 ) { print "corrupting db"; - if ( ! piped_exec(fmt("truncate --size=8 %s", asn_fn), "") ) + if ( ! piped_exec(fmt("truncate -s 8 %s", asn_fn), "") ) exit(1); - if ( ! piped_exec(fmt("truncate --size=8 %s", city_fn), "") ) + if ( ! piped_exec(fmt("truncate -s 8 %s", city_fn), "") ) exit(1); } else if ( pkt == 4 ) @@ -73,11 +88,11 @@ event new_packet(c: connection, p: pkt_hdr) exit(1); } - # Set MMDB's modification time to current network time for predictability. - if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", asn_fn, network_time(), asn_fn), "") ) + # Increment MMDB's modification time. + if ( ! piped_exec(fmt("test -f %s && touch -d %s %s", asn_fn, timestamp(pkt), asn_fn), "") ) exit(1); - if ( ! piped_exec(fmt("test -f %s && touch -d @%s %s", city_fn, network_time(), city_fn), "") ) + if ( ! piped_exec(fmt("test -f %s && touch -d %s %s", city_fn, timestamp(pkt), city_fn), "") ) exit(1); print network_time(), pkt, 128.3.0.1, "asn", lookup_autonomous_system(128.3.0.1); From 147fa3b1843de43154135ad03af52d7a57f72745 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 6 Nov 2023 11:09:44 -0700 Subject: [PATCH 10/20] CI: Add macOS Sonoma build, remove macOS Monterey build (cherry picked from commit 35ca1e88c7f1398c37bcbc4e20700a2aff2c4169) --- .cirrus.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 4ef493ef79..b508499af6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -303,6 +303,13 @@ alpine_task: # Apple doesn't publish official long-term support timelines. # We aim to support both the current and previous macOS release. +macos_sonoma_task: + macos_instance: + image: ghcr.io/cirruslabs/macos-sonoma-base:latest + prepare_script: ./ci/macos/prepare.sh + << : *CI_TEMPLATE + << : *MACOS_ENVIRONMENT + macos_ventura_task: macos_instance: image: ghcr.io/cirruslabs/macos-ventura-base:latest @@ -310,14 +317,6 @@ macos_ventura_task: << : *CI_TEMPLATE << : *MACOS_ENVIRONMENT -macos_monterey_task: - macos_instance: - image: ghcr.io/cirruslabs/macos-monterey-base:latest - prepare_script: ./ci/macos/prepare.sh - << : *CI_TEMPLATE - << : *MACOS_ENVIRONMENT - << : *SKIP_TASK_ON_PR - # FreeBSD EOL timelines: https://www.freebsd.org/security/#sup freebsd14_task: freebsd_instance: From 15e5d4aee59fc3cdf45a733c6020f006461488f2 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Mon, 6 Nov 2023 11:29:44 -0700 Subject: [PATCH 11/20] CI: Use other base64 options on macOS Sonoma too (cherry picked from commit b1e63ffd268cdb2cd28098a0a8578a3038be8fd4) --- ci/init-external-repos.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/init-external-repos.sh b/ci/init-external-repos.sh index ce2d3f4a6d..7f27f582be 100755 --- a/ci/init-external-repos.sh +++ b/ci/init-external-repos.sh @@ -51,8 +51,8 @@ if [[ -n "${CIRRUS_CI}" ]] && [[ "${CIRRUS_REPO_OWNER}" == "zeek" ]] && [[ ! -d banner "Trying to clone zeek-testing-private git repo" echo "${ZEEK_TESTING_PRIVATE_SSH_KEY}" >cirrus_key.b64 - if [ "${CIRRUS_TASK_NAME}" == "macos_ventura" ]; then - # The base64 command provided with macOS Ventura requires an argument + if [ "${CIRRUS_TASK_NAME}" == "macos_ventura" -o "${CIRRUS_TASK_NAME}" == "macos_sonoma" ]; then + # The base64 command provided with macOS Ventura/Sonoma requires an argument # to pass the input filename base64 -d -i cirrus_key.b64 >cirrus_key else From 3cf68e38681a23bcae37120fab7bc36bed85f26f Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 10 Nov 2023 14:42:01 -0700 Subject: [PATCH 12/20] CI: Install missing packages on opensuse (cherry picked from commit a3ad1a24f11770d5b8c39af1671beb5d2b4c65b2) --- ci/opensuse-leap-15.4/Dockerfile | 7 ++++--- ci/opensuse-leap-15.5/Dockerfile | 1 + ci/opensuse-tumbleweed/Dockerfile | 4 +++- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/ci/opensuse-leap-15.4/Dockerfile b/ci/opensuse-leap-15.4/Dockerfile index 6adc631fbf..4ec1eb50ca 100644 --- a/ci/opensuse-leap-15.4/Dockerfile +++ b/ci/opensuse-leap-15.4/Dockerfile @@ -19,9 +19,10 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.4 libopenssl-devel \ libpcap-devel \ make \ - python3 \ - python3-devel \ - python3-pip \ + openssh \ + python39 \ + python39-devel \ + python39-pip \ swig \ tar \ which \ diff --git a/ci/opensuse-leap-15.5/Dockerfile b/ci/opensuse-leap-15.5/Dockerfile index 3fee52ef48..605412a885 100644 --- a/ci/opensuse-leap-15.5/Dockerfile +++ b/ci/opensuse-leap-15.5/Dockerfile @@ -19,6 +19,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.5 libopenssl-devel \ libpcap-devel \ make \ + openssh \ python311 \ python311-devel \ python311-pip \ diff --git a/ci/opensuse-tumbleweed/Dockerfile b/ci/opensuse-tumbleweed/Dockerfile index 18996c0008..6318847801 100644 --- a/ci/opensuse-tumbleweed/Dockerfile +++ b/ci/opensuse-tumbleweed/Dockerfile @@ -25,9 +25,11 @@ RUN zypper refresh \ libopenssl-devel \ libpcap-devel \ make \ + openssh \ python3 \ python3-devel \ python3-pip \ + python3-websockets \ swig \ tar \ util-linux \ @@ -35,4 +37,4 @@ RUN zypper refresh \ zlib-devel \ && rm -rf /var/cache/zypp -RUN pip3 install --break-system-packages websockets junit2html +RUN pip3 install --break-system-packages junit2html From a979fa419d3513bd9222e77b4b91bec03c9021d2 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 8 Dec 2023 18:16:34 -0800 Subject: [PATCH 13/20] CI: drop Fedora 37, add Fedora 39 This makes 39 the ony to run at all times, and 38 one to run only with fullci. (cherry picked from commit ee8cc77050e39aea18f2da144dc8108110bb42dc) --- .cirrus.yml | 14 +++++++------- ci/{fedora-37 => fedora-39}/Dockerfile | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) rename ci/{fedora-37 => fedora-39}/Dockerfile (92%) diff --git a/.cirrus.yml b/.cirrus.yml index b508499af6..7005a6ce43 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -156,19 +156,19 @@ env: # Linux EOL timelines: https://linuxlifecycle.com/ # Fedora (~13 months): https://fedoraproject.org/wiki/Fedora_Release_Life_Cycle +fedora39_task: + container: + # Fedora 39 EOL: Around Nov 2024 + dockerfile: ci/fedora-39/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + fedora38_task: container: # Fedora 38 EOL: Around May 2024 dockerfile: ci/fedora-38/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE - -fedora37_task: - container: - # Fedora 37 EOL: Around Dec 2024 - dockerfile: ci/fedora-37/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR centosstream9_task: diff --git a/ci/fedora-37/Dockerfile b/ci/fedora-39/Dockerfile similarity index 92% rename from ci/fedora-37/Dockerfile rename to ci/fedora-39/Dockerfile index 43335ff770..3b9177415a 100644 --- a/ci/fedora-37/Dockerfile +++ b/ci/fedora-39/Dockerfile @@ -1,8 +1,8 @@ -FROM fedora:37 +FROM fedora:39 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230816 +ENV DOCKERFILE_VERSION 20231208 RUN dnf -y install \ bison \ From ff231637274479c5d60ceff173f34fdfdb5dabb3 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 8 Dec 2023 18:30:32 -0800 Subject: [PATCH 14/20] CI: distro EOL comment tweaks (cherry picked from commit eb1c4bf019f6a39f1d76de717eee1b34cdc50378) --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 7005a6ce43..a9c0032a9c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -198,7 +198,7 @@ centos7_task: debian12_task: container: - # Debian 12 (bookworm) EOL: (not yet released) + # Debian 12 (bookworm) EOL: TBD dockerfile: ci/debian-12/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE @@ -240,7 +240,7 @@ debian10_task: opensuse_leap_15_4_task: container: - # Opensuse Leap 15.4 EOL: ~Nov 2023 + # Opensuse Leap 15.4 EOL: ~Dec 2023 dockerfile: ci/opensuse-leap-15.4/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE From 1d14faa18ff7e262216f21311e049eb11b64b455 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Fri, 8 Dec 2023 18:43:57 -0800 Subject: [PATCH 15/20] CI: FreeBSD 14 is out now, 12 is about to EOL. (cherry picked from commit 18b8e6d6bceb4492d842d9bacbd3ad9dfbdd7702) --- .cirrus.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index a9c0032a9c..1e77139bf0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -320,10 +320,8 @@ macos_ventura_task: # FreeBSD EOL timelines: https://www.freebsd.org/security/#sup freebsd14_task: freebsd_instance: - # We don't support FreeBSD 14 yet, this is a purely informative task - image_family: freebsd-14-0-snap - allow_failures: true - skip_notification: true + # FreeBSD 14 EOL: Nov 30 2028 + image_family: freebsd-14-0 << : *FREEBSD_RESOURCES_TEMPLATE prepare_script: ./ci/freebsd/prepare.sh From c683228e8a9eb7858190756e00238d214114e4eb Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 13 Dec 2023 22:03:25 -0800 Subject: [PATCH 16/20] CI: Drop openSUSE Leap 15.4, about to EOL. (cherry picked from commit ddd56b9ad7171a9f8ee3de03bc362223f2fad0fc) --- .cirrus.yml | 10 --------- ci/opensuse-leap-15.4/Dockerfile | 35 -------------------------------- 2 files changed, 45 deletions(-) delete mode 100644 ci/opensuse-leap-15.4/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml index 1e77139bf0..c740c171cb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -238,16 +238,6 @@ debian10_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE -opensuse_leap_15_4_task: - container: - # Opensuse Leap 15.4 EOL: ~Dec 2023 - dockerfile: ci/opensuse-leap-15.4/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE - << : *SKIP_TASK_ON_PR - env: - ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG - opensuse_leap_15_5_task: container: # Opensuse Leap 15.5 EOL: ~Dec 2024 diff --git a/ci/opensuse-leap-15.4/Dockerfile b/ci/opensuse-leap-15.4/Dockerfile deleted file mode 100644 index 4ec1eb50ca..0000000000 --- a/ci/opensuse-leap-15.4/Dockerfile +++ /dev/null @@ -1,35 +0,0 @@ -FROM opensuse/leap:15.4 - -# A version field to invalidate Cirrus's build cache when needed, as suggested in -# https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230816 - -RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.4:Update/standard/openSUSE:Leap:15.4:Update.repo \ - && zypper refresh \ - && zypper in -y \ - bison \ - ccache \ - cmake \ - curl \ - flex \ - gcc10 \ - gcc10-c++ \ - git \ - gzip \ - libopenssl-devel \ - libpcap-devel \ - make \ - openssh \ - python39 \ - python39-devel \ - python39-pip \ - swig \ - tar \ - which \ - zlib-devel \ - && rm -rf /var/cache/zypp - -RUN pip3 install websockets junit2html - -RUN update-alternatives --install /usr/bin/cc cc /usr/bin/gcc-10 100 -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/g++-10 100 From ee0bc98a9a1c7b3e7dc993ffa068608d03ef659e Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 14 Dec 2023 15:35:13 -0800 Subject: [PATCH 17/20] CI: Add Ubuntu 23.10 (cherry picked from commit 866d4b4127ccfd8904f8867b468babd330dd6d52) --- .cirrus.yml | 7 +++++++ ci/ubuntu-23.10/Dockerfile | 38 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) create mode 100644 ci/ubuntu-23.10/Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml index c740c171cb..e4dd3b04d1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -254,6 +254,13 @@ opensuse_tumbleweed_task: << : *CI_TEMPLATE # << : *SKIP_TASK_ON_PR +ubuntu23_task: + container: + # Ubuntu 23.10 EOL: July 2024 + dockerfile: ci/ubuntu-23.10/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + ubuntu22_task: container: # Ubuntu 22.04 EOL: April 2027 diff --git a/ci/ubuntu-23.10/Dockerfile b/ci/ubuntu-23.10/Dockerfile new file mode 100644 index 0000000000..de278f6f87 --- /dev/null +++ b/ci/ubuntu-23.10/Dockerfile @@ -0,0 +1,38 @@ +FROM ubuntu:23.10 + +ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" + +# A version field to invalidate Cirrus's build cache when needed, as suggested in +# https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 +ENV DOCKERFILE_VERSION 20230828 + +RUN apt-get update && apt-get -y install \ + bc \ + bison \ + bsdmainutils \ + ccache \ + cmake \ + curl \ + flex \ + g++ \ + gcc \ + git \ + libkrb5-dev \ + libmaxminddb-dev \ + libpcap-dev \ + libssl-dev \ + make \ + python3 \ + python3-dev \ + python3-pip \ + python3-websockets \ + ruby \ + sqlite3 \ + swig \ + unzip \ + wget \ + zlib1g-dev \ + && apt autoclean \ + && rm -rf /var/lib/apt/lists/* + +RUN pip3 install --break-system-packages junit2html From e054bec9358569eabcaf440f73fd7e2cc748a44f Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Thu, 14 Dec 2023 15:40:47 -0800 Subject: [PATCH 18/20] CI: Move Debian variations from 11 to 12 This switches the ARM and static builds from 11 to 12. x86 and ARM now build by default, and 11 only with fullci. 10 remains as-is, because we like to torture ourselves. (cherry picked from commit 4f6c25bb91b935666b6748b95c2b6ee55c1491ad) --- .cirrus.yml | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index e4dd3b04d1..2cefe12827 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -202,7 +202,26 @@ debian12_task: dockerfile: ci/debian-12/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE + +arm_debian12_task: + arm_container: + # Debian 12 (bookworm) EOL: TBD + dockerfile: ci/debian-12/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG + +debian12_static_task: + container: + # Just use a recent/common distro to run a static compile test. + # Debian 12 (bookworm) EOL: TBD + dockerfile: ci/debian-12/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR + env: + ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG debian11_task: container: @@ -210,26 +229,7 @@ debian11_task: dockerfile: ci/debian-11/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE - -arm_debian11_task: - arm_container: - # Debian 11 EOL: June 2026 - dockerfile: ci/debian-11/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE - env: - ZEEK_CI_CONFIGURE_FLAGS: *NO_SPICY_CONFIG - -debian11_static_task: - container: - # Just use a recent/common distro to run a static compile test. - # Debian 11 EOL: June 2026 - dockerfile: ci/debian-11/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE << : *SKIP_TASK_ON_PR - env: - ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG debian10_task: container: From d7aae6afa5eaf871abd767cca219768899a01568 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Fri, 15 Dec 2023 15:12:19 +0100 Subject: [PATCH 19/20] ci: Remove ZEEK_CI_DISABLE_SCRIPT_PROFILING logic To my knowledge this doesn't make a huge difference anymore and locally I'm only using ASAN builds. It's not actually too slow. (cherry picked from commit 344369f169b872b8dcb569f089b02e026fe7cfc2) --- .cirrus.yml | 2 -- ci/test.sh | 16 ---------------- 2 files changed, 18 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 2cefe12827..39fe08e39a 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -348,7 +348,6 @@ asan_sanitizer_task: env: CXXFLAGS: -DZEEK_DICT_DEBUG ZEEK_CI_CONFIGURE_FLAGS: *ASAN_SANITIZER_CONFIG - ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1 ASAN_OPTIONS: detect_leaks=1 ubsan_sanitizer_task: @@ -363,7 +362,6 @@ ubsan_sanitizer_task: env: CXXFLAGS: -DZEEK_DICT_DEBUG ZEEK_CI_CONFIGURE_FLAGS: *UBSAN_SANITIZER_CONFIG - ZEEK_CI_DISABLE_SCRIPT_PROFILING: 1 ZEEK_TAILORED_UB_CHECKS: 1 UBSAN_OPTIONS: print_stacktrace=1 diff --git a/ci/test.sh b/ci/test.sh index b643293911..310212ab00 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -59,13 +59,6 @@ function run_btests { pushd testing/btest - # Commenting out this line in btest.cfg causes the script profiling/coverage - # to be disabled. We do this for the sanitizer build right now because of a - # fairly significant performance bug when running tests. - if [ "${ZEEK_CI_DISABLE_SCRIPT_PROFILING}" = "1" ]; then - sed -i 's/^ZEEK_PROFILER_FILE/#ZEEK_PROFILER_FILE/g' btest.cfg - fi - ${BTEST} -z ${ZEEK_CI_BTEST_RETRIES} -d -A -x btest-results.xml -j ${ZEEK_CI_BTEST_JOBS} || result=1 make coverage prep_artifacts @@ -74,15 +67,6 @@ function run_btests { } function run_external_btests { - # Commenting out this line in btest.cfg causes the script profiling/coverage - # to be disabled. We do this for the sanitizer build right now because of a - # fairly significant performance bug when running tests. - if [ "${ZEEK_CI_DISABLE_SCRIPT_PROFILING}" = "1" ]; then - pushd testing/external - sed -i 's/^ZEEK_PROFILER_FILE/#ZEEK_PROFILER_FILE/g' subdir-btest.cfg - popd - fi - local zeek_testing_pid="" local zeek_testing_pid_private="" pushd testing/external/zeek-testing From 29e4b13881b59e5959d03d349fbed0b04a888ae6 Mon Sep 17 00:00:00 2001 From: Tim Wojtulewicz Date: Fri, 5 Jan 2024 10:12:54 -0700 Subject: [PATCH 20/20] CI: Remove unused openssl30_config (cherry picked from commit 652ba502aab843574402ec53aa0a6561b21253f3) --- .cirrus.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index 39fe08e39a..929fc75f2f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -16,7 +16,6 @@ static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enab asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy --ccache ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy --ccache tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy --ccache -openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install --ccache resources_template: &RESOURCES_TEMPLATE cpu: *CPUS