Merge remote-tracking branch 'origin/topic/awelzel/2674-arm64-containers-on-cirrus'

* origin/topic/awelzel/2674-arm64-containers-on-cirrus:
  cirrus/containers: Do not install btest pcaps
  cirrus: Replace curl with _cache
  cirrus: Add cluster testing
  cirrus: Polish container_image_manifest_docker_builder
  cirrus: Use ccache for faster container builds
  cirrus: Add docker_builder tasks to build and push images
  github: No more docker workflow

.cirrus.yml

@@ -382,3 +382,171 @@ windows_task:
     CTEST_OUTPUT_ON_FAILURE: 1
   << : *BRANCH_WHITELIST
   << : *SKIP_TASK_ON_PR
+
+
+# Container images
+#
+# Use two separate tasks to build images for amd64 and arm64.
+# Use use a third docker_builder task to collect the produced images
+# (through CIRRUS_HTTP_CACHE) and push them to the registry as
+# zeek/zeek:v1.2.3-<arch> or zeek/zeek-dev:latest-<arch> tags. Once
+# pushed, create a manifest for zeek/zeek:v1.2.3 or zeek/zeek-dev:latest
+# that includes the just pushed architecture specific images.
+#
+# We've previously tried using docker buildx with QEMU using GitHub
+# actions. The emulated arm64 build on the amd64 VMs they provide took
+# more than 6 hours and timed out. Using separate builders on Cirrus allows
+# us build natively and much faster at the expense of the docker manifest
+# wrangling (and not being able to use the nice GitHub actions).
+docker_build_template: &DOCKER_BUILD_TEMPLATE
+  cpu: *CPUS
+  memory: *MEMORY
+  set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV
+
+  env:
+    ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps
+    CIRRUS_LOG_TIMESTAMP: true
+    BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache
+    ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH}
+
+  always:
+    ccache_cache:
+      folder: /tmp/ccache
+      fingerprint_script: echo ccache-$CIRRUS_TASK_NAME-$CIRRUS_OS
+      reupload_on_changes: true
+
+    builder_image_cache:
+      folder: /tmp/builder-image-cache
+      fingerprint_script: echo builder-image-cache-$CIRRUS_TASK_NAME-$CIRRUS_OS
+      reupload_on_changes: true
+
+    zeek_image_cache:
+      folder: /tmp/zeek-image-cache-${CIRRUS_ARCH}
+      fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-${CIRRUS_ARCH}
+      reupload_on_changes: true
+
+  sync_submodules_script: git submodule update --recursive --init --recommend-shallow -j $(nproc)
+
+  prepare_builder_script:
+    - set -x
+    - mkdir -p ${BUILDER_IMAGE_CACHE_DIR} ${ZEEK_IMAGE_CACHE_DIR}
+    - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/builder.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/builder.zst | docker load; fi
+    - if [ -f ${BUILDER_IMAGE_CACHE_DIR}/final.zst ]; then zstd -d < ${BUILDER_IMAGE_CACHE_DIR}/final.zst | docker load; fi
+    - cd docker && docker build --cache-from zeek-builder:latest -t zeek-builder:latest -f builder.Dockerfile .
+    - docker save zeek-builder:latest | zstd > ${BUILDER_IMAGE_CACHE_DIR}/builder.zst
+  build_zeek_script:
+    - docker run --name zeek-builder-container -e CCACHE_DIR=/tmp/ccache -e CCACHE_NOSTATS=1 -v $(pwd):/src/zeek -v/tmp/ccache:/tmp/ccache -w /src/zeek zeek-builder:latest bash -c "./configure $ZEEK_CONFIGURE_FLAGS && ninja -C build install"
+    # The "zeek-build" tag is used within final.Dockerfile using COPY --from=...
+    - docker commit zeek-builder-container zeek-build
+  build_final_script:
+    - cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile .
+    - docker save ${IMAGE_TAG} | zstd > ${ZEEK_IMAGE_CACHE_DIR}/final.zst
+
+arm64_container_image_docker_builder:
+  env:
+    CIRRUS_ARCH: arm64
+  << : *DOCKER_BUILD_TEMPLATE
+
+amd64_container_image_docker_builder:
+  env:
+    CIRRUS_ARCH: amd64
+  << : *DOCKER_BUILD_TEMPLATE
+
+container_image_manifest_docker_builder:
+  cpu: 1
+  # Push master builds to zeek/zeek-dev, or tagged release branches to zeek/zeek
+  only_if: >
+    ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' &&
+      ( $CIRRUS_BRANCH == 'master' ||
+        ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '')
+      )
+    )
+  env:
+    DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!]
+    DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!]
+    AWS_ECR_ACCESS_KEY_ID: ENCRYPTED[!eff52f6442e1bc78bce5b15a23546344df41bf519f6201924cb70c7af12db23f442c0e5f2b3687c2d856ceb11fcb8c49!]
+    AWS_ECR_SECRET_ACCESS_KEY: ENCRYPTED[!748bc302dd196140a5fa8e89c9efd148882dc846d4e723787d2de152eb136fa98e8dea7e6d2d6779d94f72dd3c088228!]
+  login_script: |
+    docker login --username $DOCKER_USERNAME --password $DOCKER_PASSWORD
+    AWS_ACCESS_KEY_ID=$AWS_ECR_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY=$AWS_ECR_SECRET_ACCESS_KEY aws ecr-public get-login-password --region us-east-1 | \
+        docker login --username AWS $AWS_ECR_USERNAME --password-stdin public.ecr.aws
+  set_image_tag_script: |
+    # If we have a CIRRUS_TAG, use the value in VERSION to push the multiarch
+    # images, otherwise use latest. Basically we push the arch images as
+    #    zeek/zeek:1.2.3-<amd64|arm64> or
+    #    zeek/zeek-dev:latest-<amd64|arm64>
+    # and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG}
+    # for tags, or zeek/zeek-dev:latest for pushes to master.
+    if [ -n "${CIRRUS_TAG}" ]; then
+      echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV
+      echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_NAME=zeek"  >> $CIRRUS_ENV
+    elif [ "${CIRRUS_BRANCH}" = "master" ]; then
+      echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV
+      echo "MANIFEST_TAG=latest"  >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_NAME=zeek-dev"  >> $CIRRUS_ENV
+      echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
+    # Hunk for testing and pushing into zeek/zeek-next. Make sure
+    # to allow the branch in the above only_if attribute of this task.
+    # elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then
+    #  echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV
+    #  echo "MANIFEST_TAG=latest"  >> $CIRRUS_ENV
+    #  echo "ARCH_IMAGE_NAME=zeek-next"  >> $CIRRUS_ENV
+    #  echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
+    else
+      echo "Bad tag/branch for container_image_manifest"
+      env
+      exit 1
+    fi
+
+  # These should've been populated by the previous jobs
+  zeek_image_arm64_cache:
+    folder: /tmp/zeek-image-cache-arm64
+    fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-arm64
+
+  zeek_image_amd64_cache:
+    folder: /tmp/zeek-image-cache-amd64
+    fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64
+
+  load_image_script:
+    - set -x;
+    - ls -lha /tmp/zeek-image-cache-*/
+    - zstd -d < /tmp/zeek-image-cache-arm64/final.zst | docker load
+    - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load
+  tag_push_script:
+    # Tag images and push to Docker Hub and AWS ECR
+    - ./ci/container-images-tag-and-push.sh
+    - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh
+  depends_on:
+    - arm64_container_image
+    - amd64_container_image
+
+cluster_testing_docker_builder:
+  cpu: *CPUS
+  memory: *MEMORY
+  only_if: $CIRRUS_REPO_FULL_NAME == 'zeek/zeek'
+  env:
+    CIRRUS_LOG_TIMESTAMP: true
+    # At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to
+    # see if it should rebuild the Zeek image or not.
+    GITHUB_ACTION: fake
+  install_deps_script:
+    # The cluster tests depend on jq and docker_builder doesn't have that :-(
+    - apt-get -q update && apt-get install -y --no-install-recommends jq
+  sync_btest_script: git submodule update --init ./auxil/btest/
+  checkout_script:
+    - set -x; cd testing/external/ && git clone https://github.com/zeek/zeek-testing-cluster.git && make checkout-repos
+  zeek_image_amd64_cache:
+    folder: /tmp/zeek-image-cache-amd64
+    fingerprint_key: zeek-image-cache-${CIRRUS_BUILD_ID}-amd64
+  load_image_script:
+    - zstd -d < /tmp/zeek-image-cache-amd64/final.zst | docker load
+    - docker tag zeek/zeek-multiarch:amd64 zeektest:latest
+  test_script:
+    - cd testing/external/zeek-testing-cluster && make
+  on_failure:
+    upload_cluster_testing_artifacts:
+      path: "testing/external/zeek-testing-cluster/.tmp/**"
+  depends_on:
+    - amd64_container_image

.github/workflows/docker.yml

@@ -1,195 +0,0 @@
-name: Check and publish Docker images
-
-on:
-  pull_request:
-  push:
-    branches: [master]
-    tags:
-      - 'v*'
-      - '!v*-dev'
-      - 'release'
-
-defaults:
-  run:
-    shell: bash
-
-env:
-  IMAGE_NAME: zeek-image.tar
-  IMAGE_FILE: /tmp/zeek-image.tar
-  IMAGE_PATH: /tmp
-
-permissions:
-  contents: read
-
-jobs:
-  docker-build:
-    runs-on: ubuntu-latest
-    env:
-      TEST_TAG: zeek:latest
-      CONFFLAGS: --generator=Ninja --build-type=Release
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: "recursive"
-
-      # Create and boot a loader. This will e.g., provide caching
-      # so we avoid rebuilds of the same image after this step.
-      - uses: docker/setup-buildx-action@v2
-
-      - name: Build image
-        uses: docker/build-push-action@v3
-        with:
-          context: ./
-          file: docker/Dockerfile
-          build-args: |
-            CONFFLAGS=${{ env.CONFFLAGS }}
-          load: true
-          tags: ${{ env.TEST_TAG }}
-
-      - name: Run btests
-        run: make -C docker/btest
-
-      - name: Save image tarball
-        run: docker save -o ${{ env.IMAGE_FILE }} ${{ env.TEST_TAG }}
-
-      - name: Get version
-        id: version
-        run: echo "RELEASE_VERSION=$(cat VERSION)" >> $GITHUB_OUTPUT
-
-      - name: Compute target tag
-        id: target
-        env:
-          RELEASE_VERSION: ${{ steps.version.outputs.RELEASE_VERSION }}
-        run: |
-          # Translate the GitHub reference into a tag name.
-          #
-          # - `release` tag maps to `zeek:latest`
-          # - `v*` tag (excluding `v*-dev` tags) maps to `zeek:RELEASE_VERSION`
-          # - `master` branch maps to `zeek-dev:latest`
-          #
-          # Any other refs are not published below.
-          if [ "${GITHUB_REF}" = "refs/tags/release" ]; then
-            echo "tag=zeek:latest" >> $GITHUB_OUTPUT
-          elif [ "${GITHUB_REF}" = "refs/heads/master" ]; then
-            echo "tag=zeek-dev:latest" >> $GITHUB_OUTPUT
-          elif [[ "${GITHUB_REF}" = refs/tags/v* ]] && [[ "${GITHUB_REF}" != refs/tags/v*-dev ]]; then
-            echo "tag=zeek:${RELEASE_VERSION}" >> $GITHUB_OUTPUT
-          fi
-
-      - name: Login to ECR
-        # Don't publish on forks. Also note that secrets for the login are not
-        # available for pull requests, so trigger on pushes only.
-        if: github.repository == 'zeek/zeek' && github.event_name == 'push'
-        uses: docker/login-action@v1
-        with:
-          registry: public.ecr.aws
-          username: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          password: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v2
-        # Don't publish on forks. Also note that secrets for the login are not
-        # available for pull requests, so trigger on pushes only.
-        if: github.repository == 'zeek/zeek' && github.event_name == 'push'
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Push image
-        # Only publish if we did compute a tag.
-        if: github.repository == 'zeek/zeek' && github.event_name == 'push' && steps.target.outputs.tag != ''
-        uses: docker/build-push-action@v3
-        with:
-          context: ./
-          file: docker/Dockerfile
-          build-args: |
-            CONFFLAGS=${{ env.CONFFLAGS }}
-          push: true
-          tags: |
-            public.ecr.aws/zeek/${{ steps.target.outputs.tag }}
-            docker.io/zeekurity/${{ steps.target.outputs.tag }}
-            docker.io/zeek/${{ steps.target.outputs.tag }}
-
-      - name: Preserve image artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.IMAGE_NAME }}
-          path: ${{ env.IMAGE_FILE }}
-          retention-days: 1
-
-      - name: Preserve btest artifacts
-        uses: actions/upload-artifact@v3
-        if: failure()
-        with:
-          name: docker-btest
-          path: docker/btest/.tmp
-          if-no-files-found: ignore
-
-  cluster-testing:
-    # We need the Zeek Docker image build job to complete first, since we need
-    # the resulting image for our docker-compose setup.
-    needs: docker-build
-    runs-on: ubuntu-latest
-    steps:
-      # Grab the sources so we have access to btest. Could also use pip, but it
-      # seems appealing to be using the in-tree version of btest. btest is in a
-      # submodule; we check it out selectively to save time.
-      - uses: actions/checkout@v3
-      - name: Check out btest
-        run: git submodule update --init ./auxil/btest
-
-      - name: Download Docker image artifact
-        uses: actions/download-artifact@v3
-        with:
-          name: ${{ env.IMAGE_NAME }}
-          path: ${{ env.IMAGE_PATH }}
-
-      - name: Load Docker image
-        run: |
-          docker load --input ${{ env.IMAGE_FILE }}
-          docker tag zeek:latest zeektest:latest
-
-      # The testsuite ref to use for this version of Zeek is stored in a file in
-      # the Zeek source tree.
-      - name: Get testsuite version
-        run: |
-          echo "TESTSUITE_COMMIT=$(cat ./testing/external/commit-hash.zeek-testing-cluster)" >> $GITHUB_ENV
-
-      - name: Retrieve cluster testsuite
-        uses: actions/checkout@v3
-        with:
-          repository: zeek/zeek-testing-cluster
-          path: testing/external/zeek-testing-cluster
-          ref: ${{ ENV.TESTSUITE_COMMIT }}
-
-      - name: Run testsuite
-        run: make -C testing/external/zeek-testing-cluster
-
-      # upload-artifact balks at certain characters in artifact
-      # filenames, so substitute them for dots.
-      - name: Sanitize artifacts
-        if: failure()
-        run: |
-          sudo apt-get -q update && sudo apt-get install -q -y rename
-          find testing/external/zeek-testing-cluster/.tmp -depth -execdir rename 's/[":<>|*?\r\n]/./g' "{}" \;
-
-      - name: Preserve btest artifacts
-        uses: actions/upload-artifact@v3
-        if: failure()
-        with:
-          name: cluster-btest
-          path: testing/external/zeek-testing-cluster/.tmp
-          if-no-files-found: ignore
-
-      # To save storage space, truncate the Docker image artifact
-      # when this run was successful.
-      - name: Truncate Docker image
-        run: |
-          truncate -s0 ${{ env.IMAGE_FILE }}
-
-      - name: Store truncated image artifact
-        uses: actions/upload-artifact@v3
-        with:
-          name: ${{ env.IMAGE_NAME }}
-          path: ${{ env.IMAGE_FILE }}
-          retention-days: 1

CHANGES

@@ -1,3 +1,19 @@
+5.2.0-dev.589 | 2023-02-01 09:02:00 +0100
+
+  * cirrus/containers: Do not install btest pcaps (Arne Welzel, Corelight)
+
+  * cirrus: Replace curl with _cache (Arne Welzel, Corelight)
+
+  * cirrus: Add cluster testing (Arne Welzel, Corelight)
+
+  * cirrus: Polish container_image_manifest_docker_builder (Arne Welzel, Corelight)
+
+  * cirrus: Use ccache for faster container builds (Arne Welzel, Corelight)
+
+  * cirrus: Add docker_builder tasks to build and push images (Arne Welzel, Corelight)
+
+  * github: No more docker workflow (Arne Welzel, Corelight)
+
 5.2.0-dev.579 | 2023-01-31 14:49:29 +0100
 
   * mysql: Recognize when client/server negotiate SSL (Arne Welzel, Corelight)

NEWS

@@ -59,6 +59,12 @@ New Functionality
   The Zeek team wants to give a huge thank you to the team at Microsoft for all
   of their effort in completing this port.
 
+- Zeek container images are now being published to zeek/zeek and zeek/zeek-dev
+  rather than zeekurity/zeek and zeekurity/zeek-dev on Docker Hub (and continue
+  to be published to public.ecr.aws) Further, container images for amd64 and
+  arm64 platforms are now available. Main driver for the latter was to allow
+  usage of the official container images on Apple's M1 systems.
+
 - New ``analyzer_confirmation_info`` and ``analyzer_violation_info`` events with
   accompanying record types ``AnalyzerConfirmationInfo`` and
   ``AnalyzerViolationInfo`` have been added. These supersede

VERSION

@@ -1 +1 @@
-5.2.0-dev.579
+5.2.0-dev.589

ci/container-images-tag-and-push.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# This script expects two local images in the local container registry:
+#
+#   zeek/zeek-multiarch:arm64
+#   zeek/zeek-multiarch:amd64
+#
+# It retags these according to the environment ARCH_IMAGE_NAME and
+# ARCH_IMAGE_TAG as zeek/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-{arm64,amd64},
+# pushes them to the registry, then creates a manifest based on MANIFEST_NAME
+# and MANIFEST_TAG environment variables as zeek/${MANIFEST_NAME}:${MANIFEST_TAG}
+# including the two tags.
+#
+# REGISTRY_PREFIX can be used to prefix images with a registry. Needs
+# to end with a slash.
+set -eux
+
+REGISTRY_PREFIX=${REGISTRY_PREFIX:-}
+ZEEK_IMAGE_REPO=${ZEEK_IMAGE_REPO:-zeek}
+
+# Check for ending slash in registry prefix
+if [ -n "${REGISTRY_PREFIX}" ]; then
+    if [[ ! "${REGISTRY_PREFIX}" =~ .+/$ ]]; then
+        echo "Missing slash in: ${REGISTRY_PREFIX}"
+        exit 1
+    fi
+fi
+
+docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64
+docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
+docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64
+docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
+
+docker manifest create ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} \
+    ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 \
+    ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
+
+docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG}

docker/builder.Dockerfile

@@ -0,0 +1,29 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+
+# Layer to build Zeek.
+FROM debian:bullseye-slim
+
+# Configure system for build.
+RUN apt-get -q update \
+ && apt-get install -q -y --no-install-recommends \
+     bind9 \
+     bison \
+     ccache \
+     cmake \
+     flex \
+     g++ \
+     gcc \
+     libfl2 \
+     libfl-dev \
+     libmaxminddb-dev \
+     libpcap-dev \
+     libssl-dev \
+     libz-dev \
+     make \
+     python3-minimal \
+     python3.9-dev \
+     swig \
+     ninja-build \
+     python3-pip \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*

docker/final.Dockerfile

@@ -0,0 +1,25 @@
+# See the file "COPYING" in the main distribution directory for copyright.
+
+# Final layer containing all artifacts.
+FROM debian:bullseye-slim
+
+RUN apt-get -q update \
+ && apt-get install -q -y --no-install-recommends \
+     ca-certificates \
+     git \
+     libmaxminddb0 \
+     libpython3.9 \
+     libpcap0.8 \
+     libssl1.1 \
+     libz1 \
+     python3-minimal \
+     python3-git \
+     python3-semantic-version \
+     python3-websocket \
+ && apt-get clean \
+ && rm -rf /var/lib/apt/lists/*
+
+# Copy over Zeek installation from build
+COPY --from=zeek-build /usr/local/zeek /usr/local/zeek
+ENV PATH "/usr/local/zeek/bin:${PATH}"
+ENV PYTHONPATH "/usr/local/zeek/lib/zeek/python:${PYTHONPATH}"