From f9b0681c98de3dd68f05bd1a8f36fcf8a6ca9068 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 18 Jan 2023 15:18:58 +0100 Subject: [PATCH 1/7] github: No more docker workflow --- .github/workflows/docker.yml | 195 ----------------------------------- 1 file changed, 195 deletions(-) delete mode 100644 .github/workflows/docker.yml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 8794f6c1fa..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Check and publish Docker images - -on: - pull_request: - push: - branches: [master] - tags: - - 'v*' - - '!v*-dev' - - 'release' - -defaults: - run: - shell: bash - -env: - IMAGE_NAME: zeek-image.tar - IMAGE_FILE: /tmp/zeek-image.tar - IMAGE_PATH: /tmp - -permissions: - contents: read - -jobs: - docker-build: - runs-on: ubuntu-latest - env: - TEST_TAG: zeek:latest - CONFFLAGS: --generator=Ninja --build-type=Release - steps: - - uses: actions/checkout@v3 - with: - submodules: "recursive" - - # Create and boot a loader. This will e.g., provide caching - # so we avoid rebuilds of the same image after this step. - - uses: docker/setup-buildx-action@v2 - - - name: Build image - uses: docker/build-push-action@v3 - with: - context: ./ - file: docker/Dockerfile - build-args: | - CONFFLAGS=${{ env.CONFFLAGS }} - load: true - tags: ${{ env.TEST_TAG }} - - - name: Run btests - run: make -C docker/btest - - - name: Save image tarball - run: docker save -o ${{ env.IMAGE_FILE }} ${{ env.TEST_TAG }} - - - name: Get version - id: version - run: echo "RELEASE_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT - - - name: Compute target tag - id: target - env: - RELEASE_VERSION: ${{ steps.version.outputs.RELEASE_VERSION }} - run: | - # Translate the GitHub reference into a tag name. - # - # - `release` tag maps to `zeek:latest` - # - `v*` tag (excluding `v*-dev` tags) maps to `zeek:RELEASE_VERSION` - # - `master` branch maps to `zeek-dev:latest` - # - # Any other refs are not published below. - if [ "${GITHUB_REF}" = "refs/tags/release" ]; then - echo "tag=zeek:latest" >> $GITHUB_OUTPUT - elif [ "${GITHUB_REF}" = "refs/heads/master" ]; then - echo "tag=zeek-dev:latest" >> $GITHUB_OUTPUT - elif [[ "${GITHUB_REF}" = refs/tags/v* ]] && [[ "${GITHUB_REF}" != refs/tags/v*-dev ]]; then - echo "tag=zeek:${RELEASE_VERSION}" >> $GITHUB_OUTPUT - fi - - - name: Login to ECR - # Don't publish on forks. Also note that secrets for the login are not - # available for pull requests, so trigger on pushes only. - if: github.repository == 'zeek/zeek' && github.event_name == 'push' - uses: docker/login-action@v1 - with: - registry: public.ecr.aws - username: ${{ secrets.AWS_ACCESS_KEY_ID }} - password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Login to DockerHub - uses: docker/login-action@v2 - # Don't publish on forks. Also note that secrets for the login are not - # available for pull requests, so trigger on pushes only. - if: github.repository == 'zeek/zeek' && github.event_name == 'push' - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Push image - # Only publish if we did compute a tag. - if: github.repository == 'zeek/zeek' && github.event_name == 'push' && steps.target.outputs.tag != '' - uses: docker/build-push-action@v3 - with: - context: ./ - file: docker/Dockerfile - build-args: | - CONFFLAGS=${{ env.CONFFLAGS }} - push: true - tags: | - public.ecr.aws/zeek/${{ steps.target.outputs.tag }} - docker.io/zeekurity/${{ steps.target.outputs.tag }} - docker.io/zeek/${{ steps.target.outputs.tag }} - - - name: Preserve image artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ env.IMAGE_NAME }} - path: ${{ env.IMAGE_FILE }} - retention-days: 1 - - - name: Preserve btest artifacts - uses: actions/upload-artifact@v3 - if: failure() - with: - name: docker-btest - path: docker/btest/.tmp - if-no-files-found: ignore - - cluster-testing: - # We need the Zeek Docker image build job to complete first, since we need - # the resulting image for our docker-compose setup. - needs: docker-build - runs-on: ubuntu-latest - steps: - # Grab the sources so we have access to btest. Could also use pip, but it - # seems appealing to be using the in-tree version of btest. btest is in a - # submodule; we check it out selectively to save time. - - uses: actions/checkout@v3 - - name: Check out btest - run: git submodule update --init ./auxil/btest - - - name: Download Docker image artifact - uses: actions/download-artifact@v3 - with: - name: ${{ env.IMAGE_NAME }} - path: ${{ env.IMAGE_PATH }} - - - name: Load Docker image - run: | - docker load --input ${{ env.IMAGE_FILE }} - docker tag zeek:latest zeektest:latest - - # The testsuite ref to use for this version of Zeek is stored in a file in - # the Zeek source tree. - - name: Get testsuite version - run: | - echo "TESTSUITE_COMMIT=$(cat ./testing/external/commit-hash.zeek-testing-cluster)" >> $GITHUB_ENV - - - name: Retrieve cluster testsuite - uses: actions/checkout@v3 - with: - repository: zeek/zeek-testing-cluster - path: testing/external/zeek-testing-cluster - ref: ${{ ENV.TESTSUITE_COMMIT }} - - - name: Run testsuite - run: make -C testing/external/zeek-testing-cluster - - # upload-artifact balks at certain characters in artifact - # filenames, so substitute them for dots. - - name: Sanitize artifacts - if: failure() - run: | - sudo apt-get -q update && sudo apt-get install -q -y rename - find testing/external/zeek-testing-cluster/.tmp -depth -execdir rename 's/[":<>|*?\r\n]/./g' "{}" \; - - - name: Preserve btest artifacts - uses: actions/upload-artifact@v3 - if: failure() - with: - name: cluster-btest - path: testing/external/zeek-testing-cluster/.tmp - if-no-files-found: ignore - - # To save storage space, truncate the Docker image artifact - # when this run was successful. - - name: Truncate Docker image - run: | - truncate -s0 ${{ env.IMAGE_FILE }} - - - name: Store truncated image artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ env.IMAGE_NAME }} - path: ${{ env.IMAGE_FILE }} - retention-days: 1 From 5510b2496a51fe5cb2d6f7c76fd7c63eb2f9bb1d Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 18 Jan 2023 14:54:05 +0100 Subject: [PATCH 2/7] cirrus: Add docker_builder tasks to build and push images --- .cirrus.yml | 92 +++++++++++++++++++++++++++++ ci/container-images-tag-and-push.sh | 38 ++++++++++++ 2 files changed, 130 insertions(+) create mode 100755 ci/container-images-tag-and-push.sh diff --git a/.cirrus.yml b/.cirrus.yml index 07d7a4653e..e3b4ce3fea 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -382,3 +382,95 @@ windows_task: CTEST_OUTPUT_ON_FAILURE: 1 << : *BRANCH_WHITELIST << : *SKIP_TASK_ON_PR + + +# Container images +# +# Use two separate tasks to build images for amd64 and arm64. +# Use use a third docker_builder task to collect the produced images +# (through CIRRUS_HTTP_CACHE) and push them to the registry as +# zeek/zeek:v1.2.3- or zeek/zeek-dev:latest- tags. Once +# pushed, create a manifest for zeek/zeek:v1.2.3 or zeek/zeek-dev:latest +# that includes the just pushed architecture specific images. +# +# We've previously tried using docker buildx with QEMU using GitHub +# actions. The emulated arm64 build on the amd64 VMs they provide took +# more than 6 hours and timed out. Using separate builders on Cirrus allows +# us build natively and much faster at the expense of the docker manifest +# wrangling (and not being able to use the nice GitHub actions). +docker_build_template: &DOCKER_BUILD_TEMPLATE + set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV + build_script: + - git submodule update --init --recursive + - set -x; cd docker && docker build -f Dockerfile --tag ${IMAGE_TAG} .. + - set -x; docker save $IMAGE_TAG | zstd > image.zst + - curl -v -X POST --data-binary @image.zst http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-${CIRRUS_ARCH} + +arm64_container_image_docker_builder: + env: + CIRRUS_ARCH: arm64 + << : *DOCKER_BUILD_TEMPLATE + +amd64_container_image_docker_builder: + << : *DOCKER_BUILD_TEMPLATE + +container_image_manifest_docker_builder: + cpu: 1 + # Push master builds to zeek/zeek-dev, or tagged release branches to zeek/zeek + only_if: > + ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' && + ( $CIRRUS_BRANCH == 'master' || + ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '') + ) + ) + env: + DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!] + DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!] + AWS_ECR_ACCESS_KEY_ID: ENCRYPTED[!eff52f6442e1bc78bce5b15a23546344df41bf519f6201924cb70c7af12db23f442c0e5f2b3687c2d856ceb11fcb8c49!] + AWS_ECR_SECRET_ACCESS_KEY: ENCRYPTED[!748bc302dd196140a5fa8e89c9efd148882dc846d4e723787d2de152eb136fa98e8dea7e6d2d6779d94f72dd3c088228!] + login_script: | + docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD + AWS_ACCESS_KEY_ID=$AWS_ECR_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_ECR_SECRET_ACCESS_KEY aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS $AWS_ECR_USERNAME --password-stdin public.ecr.aws + set_image_tag_script: | + # If we have a CIRRUS_TAG, use the value in VERSION to push the multiarch + # images, otherwise use latest. Basically we push the arch images as + # zeek/zeek:1.2.3- or + # zeek/zeek-dev:latest- + # and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG} + # for tags, or zeek/zeek-dev:latest for pushes to master. + if [ -n "${CIRRUS_TAG}" ]; then + echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV + echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV + echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV + echo "ARCH_IMAGE_NAME=zeek" >> $CIRRUS_ENV + elif [ "${CIRRUS_BRANCH}" = "master" ]; then + echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV + echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV + echo "ARCH_IMAGE_NAME=zeek-dev" >> $CIRRUS_ENV + echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV + # Hunk for testing and pushing into zeek/zeek-next. Make sure + # to allow the branch in the above only_if attribute of this task. + # elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then + # echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV + # echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV + # echo "ARCH_IMAGE_NAME=zeek-next" >> $CIRRUS_ENV + # echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV + else + echo "Bad tag/branch for container_image_manifest" + env + exit 1 + fi + build_script: + # Fetch and load the images from the previous tasks + - set -x; curl -v -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-arm64 + - set -x; curl -v -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-amd64 + - set -x; zstd -d < ${CIRRUS_BUILD_ID}-image-arm64 | docker load + - set -x; zstd -d < ${CIRRUS_BUILD_ID}-image-amd64 | docker load + + - ./ci/container-images-tag-and-push.sh + + - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh + depends_on: + - arm64_container_image + - amd64_container_image diff --git a/ci/container-images-tag-and-push.sh b/ci/container-images-tag-and-push.sh new file mode 100755 index 0000000000..3eef706c34 --- /dev/null +++ b/ci/container-images-tag-and-push.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# This script expects two local images in the local container registry: +# +# zeek/zeek-multiarch:arm64 +# zeek/zeek-multiarch:amd64 +# +# It retags these according to the environment ARCH_IMAGE_NAME and +# ARCH_IMAGE_TAG as zeek/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-{arm64,amd64}, +# pushes them to the registry, then creates a manifest based on MANIFEST_NAME +# and MANIFEST_TAG environment variables as zeek/${MANIFEST_NAME}:${MANIFEST_TAG} +# including the two tags. +# +# REGISTRY_PREFIX can be used to prefix images with a registry. Needs +# to end with a slash. +set -eux + +REGISTRY_PREFIX=${REGISTRY_PREFIX:-} +ZEEK_IMAGE_REPO=${ZEEK_IMAGE_REPO:-zeek} + +# Check for ending slash in registry prefix +if [ -n "${REGISTRY_PREFIX}" ]; then + if [[ ! "${REGISTRY_PREFIX}" =~ .+/$ ]]; then + echo "Missing slash in: ${REGISTRY_PREFIX}" + exit 1 + fi +fi + +docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 +docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 +docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 +docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 + +docker manifest create ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} \ + ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 \ + ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 + +docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} From 7651c4631481e2d275be3a9e30d5ed0a92377408 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 31 Jan 2023 11:45:17 +0100 Subject: [PATCH 3/7] cirrus: Use ccache for faster container builds --- .cirrus.yml | 40 ++++++++++++++++++++++++++++++++++----- docker/builder.Dockerfile | 29 ++++++++++++++++++++++++++++ docker/final.Dockerfile | 25 ++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 docker/builder.Dockerfile create mode 100644 docker/final.Dockerfile diff --git a/.cirrus.yml b/.cirrus.yml index e3b4ce3fea..d9133d8f11 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -399,12 +399,42 @@ windows_task: # us build natively and much faster at the expense of the docker manifest # wrangling (and not being able to use the nice GitHub actions). docker_build_template: &DOCKER_BUILD_TEMPLATE + cpu: *CPUS + memory: *MEMORY set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV - build_script: - - git submodule update --init --recursive - - set -x; cd docker && docker build -f Dockerfile --tag ${IMAGE_TAG} .. - - set -x; docker save $IMAGE_TAG | zstd > image.zst - - curl -v -X POST --data-binary @image.zst http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-${CIRRUS_ARCH} + + env: + ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release + CIRRUS_LOG_TIMESTAMP: true + IMAGE_CACHE_DIR: /tmp/image-cache + + always: + ccache_cache: + folder: /tmp/ccache + fingerprint_script: echo ccache-$CIRRUS_TASK_NAME-$CIRRUS_OS + reupload_on_changes: true + image_cache: + folder: /tmp/image-cache + fingerprint_script: echo image-cache-$CIRRUS_TASK_NAME-$CIRRUS_OS + reupload_on_changes: true + + sync_submodules_script: git submodule update --recursive --init --recommend-shallow -j $(nproc) + + prepare_builder_script: + - set -x + - mkdir -p ${IMAGE_CACHE_DIR} + - if [ -f ${IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${IMAGE_CACHE_DIR}/builder.zst | docker load; fi + - if [ -f ${IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${IMAGE_CACHE_DIR}/final.zst | docker load; fi + - (cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile .) + - docker save zeek-builder:latest | zstd > ${IMAGE_CACHE_DIR}/builder.zst + build_zeek_script: + - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install" + # The "zeek-build" tag is used within final.Dockerfile using COPY --from=... + - docker commit zeek-builder-container zeek-build + build_final_script: + - (cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile .) + - docker save ${IMAGE_TAG} | zstd > ${IMAGE_CACHE_DIR}/final.zst + - curl -sSf -X POST --data-binary @${IMAGE_CACHE_DIR}/final.zst http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-${CIRRUS_ARCH} arm64_container_image_docker_builder: env: diff --git a/docker/builder.Dockerfile b/docker/builder.Dockerfile new file mode 100644 index 0000000000..aa1becec83 --- /dev/null +++ b/docker/builder.Dockerfile @@ -0,0 +1,29 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +# Layer to build Zeek. +FROM debian:bullseye-slim + +# Configure system for build. +RUN apt-get -q update \ + && apt-get install -q -y --no-install-recommends \ + bind9 \ + bison \ + ccache \ + cmake \ + flex \ + g++ \ + gcc \ + libfl2 \ + libfl-dev \ + libmaxminddb-dev \ + libpcap-dev \ + libssl-dev \ + libz-dev \ + make \ + python3-minimal \ + python3.9-dev \ + swig \ + ninja-build \ + python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/docker/final.Dockerfile b/docker/final.Dockerfile new file mode 100644 index 0000000000..775b1b32b0 --- /dev/null +++ b/docker/final.Dockerfile @@ -0,0 +1,25 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +# Final layer containing all artifacts. +FROM debian:bullseye-slim + +RUN apt-get -q update \ + && apt-get install -q -y --no-install-recommends \ + ca-certificates \ + git \ + libmaxminddb0 \ + libpython3.9 \ + libpcap0.8 \ + libssl1.1 \ + libz1 \ + python3-minimal \ + python3-git \ + python3-semantic-version \ + python3-websocket \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy over Zeek installation from build +COPY --from=zeek-build /usr/local/zeek /usr/local/zeek +ENV PATH "/usr/local/zeek/bin:${PATH}" +ENV PYTHONPATH "/usr/local/zeek/lib/zeek/python:${PYTHONPATH}" From b5dc4c66172afbc8d5698ac940a077d9fdc71d7b Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 31 Jan 2023 11:46:31 +0100 Subject: [PATCH 4/7] cirrus: Polish container_image_manifest_docker_builder --- .cirrus.yml | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index d9133d8f11..2fc1f21559 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -491,15 +491,16 @@ container_image_manifest_docker_builder: env exit 1 fi - build_script: + load_image_script: # Fetch and load the images from the previous tasks - - set -x; curl -v -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-arm64 - - set -x; curl -v -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-amd64 - - set -x; zstd -d < ${CIRRUS_BUILD_ID}-image-arm64 | docker load - - set -x; zstd -d < ${CIRRUS_BUILD_ID}-image-amd64 | docker load - + - set -x; + - curl -sSf -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-arm64 + - curl -sSf -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-amd64 + - zstd -d < ${CIRRUS_BUILD_ID}-image-arm64 | docker load + - zstd -d < ${CIRRUS_BUILD_ID}-image-amd64 | docker load + tag_push_script: + # Tag images and push to Docker Hub and AWS ECR - ./ci/container-images-tag-and-push.sh - - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh depends_on: - arm64_container_image From 5a611d4bdb3ad27a083075a0a5ad35630c119ee7 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 31 Jan 2023 11:47:08 +0100 Subject: [PATCH 5/7] cirrus: Add cluster testing --- .cirrus.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 2fc1f21559..b20e918d60 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -505,3 +505,30 @@ container_image_manifest_docker_builder: depends_on: - arm64_container_image - amd64_container_image + +cluster_testing_docker_builder: + cpu: *CPUS + memory: *MEMORY + only_if: $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' + env: + CIRRUS_LOG_TIMESTAMP: true + # At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to + # see if it should rebuild the Zeek image or not. + GITHUB_ACTION: fake + install_deps_script: + # The cluster tests depend on jq and docker_builder doesn't have that :-( + - apt-get -q update && apt-get install -y --no-install-recommends jq + sync_btest_script: git submodule update --init ./auxil/btest/ + checkout_script: + - set -x; cd testing/external/ && git clone https://github.com/zeek/zeek-testing-cluster.git && make checkout-repos + load_image_script: + - curl -sSf -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-amd64 + - zstd -d < ${CIRRUS_BUILD_ID}-image-amd64 | docker load + - docker tag zeek/zeek-multiarch:amd64 zeektest:latest + test_script: + - cd testing/external/zeek-testing-cluster && make + on_failure: + upload_cluster_testing_artifacts: + path: "testing/external/zeek-testing-cluster/.tmp/**" + depends_on: + - amd64_container_image From 3f7adbe6b8982ca3aa54f5b7fb650045812dd82f Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Tue, 31 Jan 2023 20:20:19 +0100 Subject: [PATCH 6/7] cirrus: Replace curl with _cache --- .cirrus.yml | 56 +++++++++++++++++++++++++++++++++++------------------ 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index b20e918d60..c86998e8ec 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -406,35 +406,41 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE env: ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release CIRRUS_LOG_TIMESTAMP: true - IMAGE_CACHE_DIR: /tmp/image-cache + BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache + ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH} always: ccache_cache: folder: /tmp/ccache fingerprint_script: echo ccache-$CIRRUS_TASK_NAME-$CIRRUS_OS reupload_on_changes: true - image_cache: - folder: /tmp/image-cache - fingerprint_script: echo image-cache-$CIRRUS_TASK_NAME-$CIRRUS_OS + + builder_image_cache: + folder: /tmp/builder-image-cache + fingerprint_script: echo builder-image-cache-$CIRRUS_TASK_NAME-$CIRRUS_OS + reupload_on_changes: true + + zeek_image_cache: + folder: /tmp/zeek-image-cache-${CIRRUS_ARCH} + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-${CIRRUS_ARCH} reupload_on_changes: true sync_submodules_script: git submodule update --recursive --init --recommend-shallow -j $(nproc) prepare_builder_script: - set -x - - mkdir -p ${IMAGE_CACHE_DIR} - - if [ -f ${IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${IMAGE_CACHE_DIR}/builder.zst | docker load; fi - - if [ -f ${IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${IMAGE_CACHE_DIR}/final.zst | docker load; fi + - mkdir -p ${BUILDER_IMAGE_CACHE_DIR} ${ZEEK_IMAGE_CACHE_DIR} + - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/builder.zst | docker load; fi + - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/final.zst | docker load; fi - (cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile .) - - docker save zeek-builder:latest | zstd > ${IMAGE_CACHE_DIR}/builder.zst + - docker save zeek-builder:latest | zstd > ${BUILDER_IMAGE_CACHE_DIR}/builder.zst build_zeek_script: - - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install" + - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -e CCACHE_NOSTATS=1 -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install" # The "zeek-build" tag is used within final.Dockerfile using COPY --from=... - docker commit zeek-builder-container zeek-build build_final_script: - - (cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile .) - - docker save ${IMAGE_TAG} | zstd > ${IMAGE_CACHE_DIR}/final.zst - - curl -sSf -X POST --data-binary @${IMAGE_CACHE_DIR}/final.zst http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-${CIRRUS_ARCH} + - cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile . + - docker save ${IMAGE_TAG} | zstd > ${ZEEK_IMAGE_CACHE_DIR}/final.zst arm64_container_image_docker_builder: env: @@ -442,6 +448,8 @@ arm64_container_image_docker_builder: << : *DOCKER_BUILD_TEMPLATE amd64_container_image_docker_builder: + env: + CIRRUS_ARCH: amd64 << : *DOCKER_BUILD_TEMPLATE container_image_manifest_docker_builder: @@ -491,13 +499,21 @@ container_image_manifest_docker_builder: env exit 1 fi + + # These should've been populated by the previous jobs + zeek_image_arm64_cache: + folder: /tmp/zeek-image-cache-arm64 + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-arm64 + + zeek_image_amd64_cache: + folder: /tmp/zeek-image-cache-amd64 + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64 + load_image_script: - # Fetch and load the images from the previous tasks - set -x; - - curl -sSf -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-arm64 - - curl -sSf -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-amd64 - - zstd -d < ${CIRRUS_BUILD_ID}-image-arm64 | docker load - - zstd -d < ${CIRRUS_BUILD_ID}-image-amd64 | docker load + - ls -lha /tmp/zeek-image-cache-*/ + - zstd -d < /tmp/zeek-image-cache-arm64/final.zst | docker load + - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load tag_push_script: # Tag images and push to Docker Hub and AWS ECR - ./ci/container-images-tag-and-push.sh @@ -521,9 +537,11 @@ cluster_testing_docker_builder: sync_btest_script: git submodule update --init ./auxil/btest/ checkout_script: - set -x; cd testing/external/ && git clone https://github.com/zeek/zeek-testing-cluster.git && make checkout-repos + zeek_image_amd64_cache: + folder: /tmp/zeek-image-cache-amd64 + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64 load_image_script: - - curl -sSf -O http://$CIRRUS_HTTP_CACHE_HOST/${CIRRUS_BUILD_ID}-image-amd64 - - zstd -d < ${CIRRUS_BUILD_ID}-image-amd64 | docker load + - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load - docker tag zeek/zeek-multiarch:amd64 zeektest:latest test_script: - cd testing/external/zeek-testing-cluster && make From 20cc5541e06554ebc7c4945b20d340f2ffc5f5c8 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 1 Feb 2023 08:51:44 +0100 Subject: [PATCH 7/7] cirrus/containers: Do not install btest pcaps --- .cirrus.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index c86998e8ec..9e4d40d3b6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -404,7 +404,7 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV env: - ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release + ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps CIRRUS_LOG_TIMESTAMP: true BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH} @@ -432,7 +432,7 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE - mkdir -p ${BUILDER_IMAGE_CACHE_DIR} ${ZEEK_IMAGE_CACHE_DIR} - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/builder.zst | docker load; fi - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/final.zst | docker load; fi - - (cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile .) + - cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile . - docker save zeek-builder:latest | zstd > ${BUILDER_IMAGE_CACHE_DIR}/builder.zst build_zeek_script: - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -e CCACHE_NOSTATS=1 -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install"