diff --git a/.cirrus.yml b/.cirrus.yml index 07d7a4653e..9e4d40d3b6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -382,3 +382,171 @@ windows_task: CTEST_OUTPUT_ON_FAILURE: 1 << : *BRANCH_WHITELIST << : *SKIP_TASK_ON_PR + + +# Container images +# +# Use two separate tasks to build images for amd64 and arm64. +# Use use a third docker_builder task to collect the produced images +# (through CIRRUS_HTTP_CACHE) and push them to the registry as +# zeek/zeek:v1.2.3- or zeek/zeek-dev:latest- tags. Once +# pushed, create a manifest for zeek/zeek:v1.2.3 or zeek/zeek-dev:latest +# that includes the just pushed architecture specific images. +# +# We've previously tried using docker buildx with QEMU using GitHub +# actions. The emulated arm64 build on the amd64 VMs they provide took +# more than 6 hours and timed out. Using separate builders on Cirrus allows +# us build natively and much faster at the expense of the docker manifest +# wrangling (and not being able to use the nice GitHub actions). +docker_build_template: &DOCKER_BUILD_TEMPLATE + cpu: *CPUS + memory: *MEMORY + set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV + + env: + ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps + CIRRUS_LOG_TIMESTAMP: true + BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache + ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH} + + always: + ccache_cache: + folder: /tmp/ccache + fingerprint_script: echo ccache-$CIRRUS_TASK_NAME-$CIRRUS_OS + reupload_on_changes: true + + builder_image_cache: + folder: /tmp/builder-image-cache + fingerprint_script: echo builder-image-cache-$CIRRUS_TASK_NAME-$CIRRUS_OS + reupload_on_changes: true + + zeek_image_cache: + folder: /tmp/zeek-image-cache-${CIRRUS_ARCH} + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-${CIRRUS_ARCH} + reupload_on_changes: true + + sync_submodules_script: git submodule update --recursive --init --recommend-shallow -j $(nproc) + + prepare_builder_script: + - set -x + - mkdir -p ${BUILDER_IMAGE_CACHE_DIR} ${ZEEK_IMAGE_CACHE_DIR} + - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/builder.zst | docker load; fi + - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/final.zst | docker load; fi + - cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile . + - docker save zeek-builder:latest | zstd > ${BUILDER_IMAGE_CACHE_DIR}/builder.zst + build_zeek_script: + - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -e CCACHE_NOSTATS=1 -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install" + # The "zeek-build" tag is used within final.Dockerfile using COPY --from=... + - docker commit zeek-builder-container zeek-build + build_final_script: + - cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile . + - docker save ${IMAGE_TAG} | zstd > ${ZEEK_IMAGE_CACHE_DIR}/final.zst + +arm64_container_image_docker_builder: + env: + CIRRUS_ARCH: arm64 + << : *DOCKER_BUILD_TEMPLATE + +amd64_container_image_docker_builder: + env: + CIRRUS_ARCH: amd64 + << : *DOCKER_BUILD_TEMPLATE + +container_image_manifest_docker_builder: + cpu: 1 + # Push master builds to zeek/zeek-dev, or tagged release branches to zeek/zeek + only_if: > + ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' && + ( $CIRRUS_BRANCH == 'master' || + ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '') + ) + ) + env: + DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!] + DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!] + AWS_ECR_ACCESS_KEY_ID: ENCRYPTED[!eff52f6442e1bc78bce5b15a23546344df41bf519f6201924cb70c7af12db23f442c0e5f2b3687c2d856ceb11fcb8c49!] + AWS_ECR_SECRET_ACCESS_KEY: ENCRYPTED[!748bc302dd196140a5fa8e89c9efd148882dc846d4e723787d2de152eb136fa98e8dea7e6d2d6779d94f72dd3c088228!] + login_script: | + docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD + AWS_ACCESS_KEY_ID=$AWS_ECR_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_ECR_SECRET_ACCESS_KEY aws ecr-public get-login-password --region us-east-1 | \ + docker login --username AWS $AWS_ECR_USERNAME --password-stdin public.ecr.aws + set_image_tag_script: | + # If we have a CIRRUS_TAG, use the value in VERSION to push the multiarch + # images, otherwise use latest. Basically we push the arch images as + # zeek/zeek:1.2.3- or + # zeek/zeek-dev:latest- + # and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG} + # for tags, or zeek/zeek-dev:latest for pushes to master. + if [ -n "${CIRRUS_TAG}" ]; then + echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV + echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV + echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV + echo "ARCH_IMAGE_NAME=zeek" >> $CIRRUS_ENV + elif [ "${CIRRUS_BRANCH}" = "master" ]; then + echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV + echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV + echo "ARCH_IMAGE_NAME=zeek-dev" >> $CIRRUS_ENV + echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV + # Hunk for testing and pushing into zeek/zeek-next. Make sure + # to allow the branch in the above only_if attribute of this task. + # elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then + # echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV + # echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV + # echo "ARCH_IMAGE_NAME=zeek-next" >> $CIRRUS_ENV + # echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV + else + echo "Bad tag/branch for container_image_manifest" + env + exit 1 + fi + + # These should've been populated by the previous jobs + zeek_image_arm64_cache: + folder: /tmp/zeek-image-cache-arm64 + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-arm64 + + zeek_image_amd64_cache: + folder: /tmp/zeek-image-cache-amd64 + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64 + + load_image_script: + - set -x; + - ls -lha /tmp/zeek-image-cache-*/ + - zstd -d < /tmp/zeek-image-cache-arm64/final.zst | docker load + - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load + tag_push_script: + # Tag images and push to Docker Hub and AWS ECR + - ./ci/container-images-tag-and-push.sh + - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh + depends_on: + - arm64_container_image + - amd64_container_image + +cluster_testing_docker_builder: + cpu: *CPUS + memory: *MEMORY + only_if: $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' + env: + CIRRUS_LOG_TIMESTAMP: true + # At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to + # see if it should rebuild the Zeek image or not. + GITHUB_ACTION: fake + install_deps_script: + # The cluster tests depend on jq and docker_builder doesn't have that :-( + - apt-get -q update && apt-get install -y --no-install-recommends jq + sync_btest_script: git submodule update --init ./auxil/btest/ + checkout_script: + - set -x; cd testing/external/ && git clone https://github.com/zeek/zeek-testing-cluster.git && make checkout-repos + zeek_image_amd64_cache: + folder: /tmp/zeek-image-cache-amd64 + fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64 + load_image_script: + - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load + - docker tag zeek/zeek-multiarch:amd64 zeektest:latest + test_script: + - cd testing/external/zeek-testing-cluster && make + on_failure: + upload_cluster_testing_artifacts: + path: "testing/external/zeek-testing-cluster/.tmp/**" + depends_on: + - amd64_container_image diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml deleted file mode 100644 index 8794f6c1fa..0000000000 --- a/.github/workflows/docker.yml +++ /dev/null @@ -1,195 +0,0 @@ -name: Check and publish Docker images - -on: - pull_request: - push: - branches: [master] - tags: - - 'v*' - - '!v*-dev' - - 'release' - -defaults: - run: - shell: bash - -env: - IMAGE_NAME: zeek-image.tar - IMAGE_FILE: /tmp/zeek-image.tar - IMAGE_PATH: /tmp - -permissions: - contents: read - -jobs: - docker-build: - runs-on: ubuntu-latest - env: - TEST_TAG: zeek:latest - CONFFLAGS: --generator=Ninja --build-type=Release - steps: - - uses: actions/checkout@v3 - with: - submodules: "recursive" - - # Create and boot a loader. This will e.g., provide caching - # so we avoid rebuilds of the same image after this step. - - uses: docker/setup-buildx-action@v2 - - - name: Build image - uses: docker/build-push-action@v3 - with: - context: ./ - file: docker/Dockerfile - build-args: | - CONFFLAGS=${{ env.CONFFLAGS }} - load: true - tags: ${{ env.TEST_TAG }} - - - name: Run btests - run: make -C docker/btest - - - name: Save image tarball - run: docker save -o ${{ env.IMAGE_FILE }} ${{ env.TEST_TAG }} - - - name: Get version - id: version - run: echo "RELEASE_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT - - - name: Compute target tag - id: target - env: - RELEASE_VERSION: ${{ steps.version.outputs.RELEASE_VERSION }} - run: | - # Translate the GitHub reference into a tag name. - # - # - `release` tag maps to `zeek:latest` - # - `v*` tag (excluding `v*-dev` tags) maps to `zeek:RELEASE_VERSION` - # - `master` branch maps to `zeek-dev:latest` - # - # Any other refs are not published below. - if [ "${GITHUB_REF}" = "refs/tags/release" ]; then - echo "tag=zeek:latest" >> $GITHUB_OUTPUT - elif [ "${GITHUB_REF}" = "refs/heads/master" ]; then - echo "tag=zeek-dev:latest" >> $GITHUB_OUTPUT - elif [[ "${GITHUB_REF}" = refs/tags/v* ]] && [[ "${GITHUB_REF}" != refs/tags/v*-dev ]]; then - echo "tag=zeek:${RELEASE_VERSION}" >> $GITHUB_OUTPUT - fi - - - name: Login to ECR - # Don't publish on forks. Also note that secrets for the login are not - # available for pull requests, so trigger on pushes only. - if: github.repository == 'zeek/zeek' && github.event_name == 'push' - uses: docker/login-action@v1 - with: - registry: public.ecr.aws - username: ${{ secrets.AWS_ACCESS_KEY_ID }} - password: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - - - name: Login to DockerHub - uses: docker/login-action@v2 - # Don't publish on forks. Also note that secrets for the login are not - # available for pull requests, so trigger on pushes only. - if: github.repository == 'zeek/zeek' && github.event_name == 'push' - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - - name: Push image - # Only publish if we did compute a tag. - if: github.repository == 'zeek/zeek' && github.event_name == 'push' && steps.target.outputs.tag != '' - uses: docker/build-push-action@v3 - with: - context: ./ - file: docker/Dockerfile - build-args: | - CONFFLAGS=${{ env.CONFFLAGS }} - push: true - tags: | - public.ecr.aws/zeek/${{ steps.target.outputs.tag }} - docker.io/zeekurity/${{ steps.target.outputs.tag }} - docker.io/zeek/${{ steps.target.outputs.tag }} - - - name: Preserve image artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ env.IMAGE_NAME }} - path: ${{ env.IMAGE_FILE }} - retention-days: 1 - - - name: Preserve btest artifacts - uses: actions/upload-artifact@v3 - if: failure() - with: - name: docker-btest - path: docker/btest/.tmp - if-no-files-found: ignore - - cluster-testing: - # We need the Zeek Docker image build job to complete first, since we need - # the resulting image for our docker-compose setup. - needs: docker-build - runs-on: ubuntu-latest - steps: - # Grab the sources so we have access to btest. Could also use pip, but it - # seems appealing to be using the in-tree version of btest. btest is in a - # submodule; we check it out selectively to save time. - - uses: actions/checkout@v3 - - name: Check out btest - run: git submodule update --init ./auxil/btest - - - name: Download Docker image artifact - uses: actions/download-artifact@v3 - with: - name: ${{ env.IMAGE_NAME }} - path: ${{ env.IMAGE_PATH }} - - - name: Load Docker image - run: | - docker load --input ${{ env.IMAGE_FILE }} - docker tag zeek:latest zeektest:latest - - # The testsuite ref to use for this version of Zeek is stored in a file in - # the Zeek source tree. - - name: Get testsuite version - run: | - echo "TESTSUITE_COMMIT=$(cat ./testing/external/commit-hash.zeek-testing-cluster)" >> $GITHUB_ENV - - - name: Retrieve cluster testsuite - uses: actions/checkout@v3 - with: - repository: zeek/zeek-testing-cluster - path: testing/external/zeek-testing-cluster - ref: ${{ ENV.TESTSUITE_COMMIT }} - - - name: Run testsuite - run: make -C testing/external/zeek-testing-cluster - - # upload-artifact balks at certain characters in artifact - # filenames, so substitute them for dots. - - name: Sanitize artifacts - if: failure() - run: | - sudo apt-get -q update && sudo apt-get install -q -y rename - find testing/external/zeek-testing-cluster/.tmp -depth -execdir rename 's/[":<>|*?\r\n]/./g' "{}" \; - - - name: Preserve btest artifacts - uses: actions/upload-artifact@v3 - if: failure() - with: - name: cluster-btest - path: testing/external/zeek-testing-cluster/.tmp - if-no-files-found: ignore - - # To save storage space, truncate the Docker image artifact - # when this run was successful. - - name: Truncate Docker image - run: | - truncate -s0 ${{ env.IMAGE_FILE }} - - - name: Store truncated image artifact - uses: actions/upload-artifact@v3 - with: - name: ${{ env.IMAGE_NAME }} - path: ${{ env.IMAGE_FILE }} - retention-days: 1 diff --git a/CHANGES b/CHANGES index 81d1c3ed71..c3797b86e4 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,19 @@ +5.2.0-dev.589 | 2023-02-01 09:02:00 +0100 + + * cirrus/containers: Do not install btest pcaps (Arne Welzel, Corelight) + + * cirrus: Replace curl with _cache (Arne Welzel, Corelight) + + * cirrus: Add cluster testing (Arne Welzel, Corelight) + + * cirrus: Polish container_image_manifest_docker_builder (Arne Welzel, Corelight) + + * cirrus: Use ccache for faster container builds (Arne Welzel, Corelight) + + * cirrus: Add docker_builder tasks to build and push images (Arne Welzel, Corelight) + + * github: No more docker workflow (Arne Welzel, Corelight) + 5.2.0-dev.579 | 2023-01-31 14:49:29 +0100 * mysql: Recognize when client/server negotiate SSL (Arne Welzel, Corelight) diff --git a/NEWS b/NEWS index a07cfe56c6..08e2a101e1 100644 --- a/NEWS +++ b/NEWS @@ -59,6 +59,12 @@ New Functionality The Zeek team wants to give a huge thank you to the team at Microsoft for all of their effort in completing this port. +- Zeek container images are now being published to zeek/zeek and zeek/zeek-dev + rather than zeekurity/zeek and zeekurity/zeek-dev on Docker Hub (and continue + to be published to public.ecr.aws) Further, container images for amd64 and + arm64 platforms are now available. Main driver for the latter was to allow + usage of the official container images on Apple's M1 systems. + - New ``analyzer_confirmation_info`` and ``analyzer_violation_info`` events with accompanying record types ``AnalyzerConfirmationInfo`` and ``AnalyzerViolationInfo`` have been added. These supersede diff --git a/VERSION b/VERSION index aa16f6dac6..9cc28ae97a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.2.0-dev.579 +5.2.0-dev.589 diff --git a/ci/container-images-tag-and-push.sh b/ci/container-images-tag-and-push.sh new file mode 100755 index 0000000000..3eef706c34 --- /dev/null +++ b/ci/container-images-tag-and-push.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# This script expects two local images in the local container registry: +# +# zeek/zeek-multiarch:arm64 +# zeek/zeek-multiarch:amd64 +# +# It retags these according to the environment ARCH_IMAGE_NAME and +# ARCH_IMAGE_TAG as zeek/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-{arm64,amd64}, +# pushes them to the registry, then creates a manifest based on MANIFEST_NAME +# and MANIFEST_TAG environment variables as zeek/${MANIFEST_NAME}:${MANIFEST_TAG} +# including the two tags. +# +# REGISTRY_PREFIX can be used to prefix images with a registry. Needs +# to end with a slash. +set -eux + +REGISTRY_PREFIX=${REGISTRY_PREFIX:-} +ZEEK_IMAGE_REPO=${ZEEK_IMAGE_REPO:-zeek} + +# Check for ending slash in registry prefix +if [ -n "${REGISTRY_PREFIX}" ]; then + if [[ ! "${REGISTRY_PREFIX}" =~ .+/$ ]]; then + echo "Missing slash in: ${REGISTRY_PREFIX}" + exit 1 + fi +fi + +docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 +docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 +docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 +docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 + +docker manifest create ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} \ + ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 \ + ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 + +docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} diff --git a/docker/builder.Dockerfile b/docker/builder.Dockerfile new file mode 100644 index 0000000000..aa1becec83 --- /dev/null +++ b/docker/builder.Dockerfile @@ -0,0 +1,29 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +# Layer to build Zeek. +FROM debian:bullseye-slim + +# Configure system for build. +RUN apt-get -q update \ + && apt-get install -q -y --no-install-recommends \ + bind9 \ + bison \ + ccache \ + cmake \ + flex \ + g++ \ + gcc \ + libfl2 \ + libfl-dev \ + libmaxminddb-dev \ + libpcap-dev \ + libssl-dev \ + libz-dev \ + make \ + python3-minimal \ + python3.9-dev \ + swig \ + ninja-build \ + python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* diff --git a/docker/final.Dockerfile b/docker/final.Dockerfile new file mode 100644 index 0000000000..775b1b32b0 --- /dev/null +++ b/docker/final.Dockerfile @@ -0,0 +1,25 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +# Final layer containing all artifacts. +FROM debian:bullseye-slim + +RUN apt-get -q update \ + && apt-get install -q -y --no-install-recommends \ + ca-certificates \ + git \ + libmaxminddb0 \ + libpython3.9 \ + libpcap0.8 \ + libssl1.1 \ + libz1 \ + python3-minimal \ + python3-git \ + python3-semantic-version \ + python3-websocket \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy over Zeek installation from build +COPY --from=zeek-build /usr/local/zeek /usr/local/zeek +ENV PATH "/usr/local/zeek/bin:${PATH}" +ENV PYTHONPATH "/usr/local/zeek/lib/zeek/python:${PYTHONPATH}"