Merge remote-tracking branch 'origin/topic/awelzel/2765-cirrus-ci-container-images-follow-up'

* origin/topic/awelzel/2765-cirrus-ci-container-images-follow-up:
  docker/Makefile: Remove stray quotes, unused DOCKER
  cirrus/container: Polish, fix and automated pushing of images

.cirrus.yml

@@ -412,6 +412,9 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE
   cpu: *CPUS
   memory: *MEMORY
   set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV
+  only_if: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) ||
+        $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' || $CIRRUS_TAG != '' )
 
   env:
     ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps --disable-cpp-tests --disable-broker-tests
@@ -454,7 +457,6 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE
   test_script:
     - docker tag ${IMAGE_TAG} zeek:latest
     - make -C docker/btest
-  << : *BRANCH_WHITELIST
 
 arm64_container_image_docker_builder:
   env:
@@ -472,9 +474,7 @@ container_image_manifest_docker_builder:
   only_if: >
     ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' &&
       ( $CIRRUS_BRANCH == 'master' ||
-        ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '')
+        $CIRRUS_TAG =~ 'v[0-9]+\.[0-9]+\.[0-9]+$' ) )
-      )
-    )
   env:
     DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!]
     DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!]
@@ -491,29 +491,62 @@ container_image_manifest_docker_builder:
     #    zeek/zeek-dev:latest-<amd64|arm64>
     # and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG}
     # for tags, or zeek/zeek-dev:latest for pushes to master.
+    set -x
     if [ -n "${CIRRUS_TAG}" ]; then
-      echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV
+      echo "IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV
-      echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV
+      echo "IMAGE_NAME=zeek"  >> $CIRRUS_ENV
-      echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV
-      echo "ARCH_IMAGE_NAME=zeek"  >> $CIRRUS_ENV
     elif [ "${CIRRUS_BRANCH}" = "master" ]; then
-      echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV
+      echo "IMAGE_NAME=zeek-dev"  >> $CIRRUS_ENV
-      echo "MANIFEST_TAG=latest"  >> $CIRRUS_ENV
+      echo "IMAGE_TAG=latest" >> $CIRRUS_ENV
-      echo "ARCH_IMAGE_NAME=zeek-dev"  >> $CIRRUS_ENV
-      echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
     # Hunk for testing and pushing into zeek/zeek-next. Make sure
     # to allow the branch in the above only_if attribute of this task.
     # elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then
-    #  echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV
+    #  echo "IMAGE_NAME=zeek-next"  >> $CIRRUS_ENV
-    #  echo "MANIFEST_TAG=latest"  >> $CIRRUS_ENV
+    #  echo "IMAGE_TAG=latest" >> $CIRRUS_ENV
-    #  echo "ARCH_IMAGE_NAME=zeek-next"  >> $CIRRUS_ENV
-    #  echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
     else
       echo "Bad tag/branch for container_image_manifest"
       env
       exit 1
     fi
 
+  set_additional_manifest_tags_script: |
+    set -x
+    if [ -z "${CIRRUS_TAG}" ]; then
+      exit 0
+    fi
+
+    # Populate the checkout with all the repository information we need
+    # to determine what the current feature and lts versions are.
+    git fetch --tags origin \
+        '+refs/heads/release/*:refs/remotes/origin/release/*' \
+        '+refs/heads/master:refs/remotes/origin/master'
+
+    # Find current versions for lts and feature depending on branches and
+    # tags in the repo. sed for escaping the dot in the version for using
+    # it in the regex below to match against CIRRUS_TAG.
+    lts_ver=$(./ci/find-current-version.sh lts)
+    lts_pat="^v$(echo $lts_ver | sed 's,\.,\\.,g')\.[0-9]+\$"
+    feature_ver=$(./ci/find-current-version.sh feature)
+    feature_pat="^v$(echo $feature_ver | sed 's,\.,\\.,g')\.[0-9]+\$"
+
+    # Construct additional tags for the image. At most this will
+    # be "lts x.0 feature" for an lts branch x.0 that is currently
+    # also the latest feature branch.
+    ADDL_MANIFEST_TAGS=
+    if echo "${CIRRUS_TAG}" | grep -E "${lts_pat}"; then
+        ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} lts ${lts_ver}"
+    fi
+
+    if echo "${CIRRUS_TAG}" | grep -E "${feature_pat}"; then
+        ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} latest"
+        if [ "${feature_ver}" != "${lts_ver}" ]; then
+          ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} ${feature_ver}"
+        fi
+    fi
+
+    # Let downstream know about it.
+    echo "ADDITIONAL_MANIFEST_TAGS=${ADDL_MANIFEST_TAGS}" >> $CIRRUS_ENV
+
   # These should've been populated by the previous jobs
   zeek_image_arm64_cache:
     folder: /tmp/zeek-image-cache-arm64
@@ -534,20 +567,17 @@ container_image_manifest_docker_builder:
     - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh
 
     # Continue to push tags to the "zeekurity" account as well.
-    - docker tag zeek/zeek-multiarch:arm64 zeekurity/zeek-multiarch:arm64
-    - docker tag zeek/zeek-multiarch:amd64 zeekurity/zeek-multiarch:amd64
     - ZEEK_IMAGE_REPO=zeekurity ./ci/container-images-tag-and-push.sh
   depends_on:
-    # Only push out the image if all the btests succeeded and the
-    # images have been built.
-    - arm_debian11
-    - debian11
     - arm64_container_image
     - amd64_container_image
 
 cluster_testing_docker_builder:
   cpu: *CPUS
   memory: *MEMORY
+  only_if: >
+    ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) ||
+        $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' || $CIRRUS_TAG != '' )
   env:
     CIRRUS_LOG_TIMESTAMP: true
     # At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to
@@ -572,4 +602,3 @@ cluster_testing_docker_builder:
       path: "testing/external/zeek-testing-cluster/.tmp/**"
   depends_on:
     - amd64_container_image
-  << : *BRANCH_WHITELIST

CHANGES

@@ -1,3 +1,24 @@
+6.0.0-dev.30 | 2023-02-13 12:10:50 +0100
+
+  * docker/Makefile: Remove stray quotes, unused DOCKER (Arne Welzel, Corelight)
+
+  * cirrus/container: Polish, fix and automated pushing of images (Arne Welzel, Corelight)
+
+    * Add ci/find-current-version.sh to figure out versions of lts and
+      feature branch based on repository state.
+
+    * Do not use ZEEK_IMAGE_REPO for the local images that
+      are expected to exist.
+
+    * Only use IMAGE_NAME/IMAGE_TAG instead of the ARCH and MANIFEST
+      versions that contained the very same values
+
+    * Support and push additional tags
+
+      In addition to the lts and feature tags, also push the base version
+      tags. 5.0 or 5.1 such that someone pulling zeek/zeek:5.0 will get
+      the most recent available 5.0 patch release.
+
 6.0.0-dev.27 | 2023-02-11 22:07:31 -0700
 
   * better error reporting when ZAM code calls a function (Vern Paxson, Corelight)

VERSION

@@ -1 +1 @@
-6.0.0-dev.27
+6.0.0-dev.30

ci/container-images-tag-and-push.sh

@@ -1,23 +1,25 @@
 #!/bin/bash
 #
-# This script expects two local images in the local container registry:
+# This script expects two images in the local container registry:
 #
 #   zeek/zeek-multiarch:arm64
 #   zeek/zeek-multiarch:amd64
 #
-# It retags these according to the environment ARCH_IMAGE_NAME and
+# It retags these according to the environment variables IMAGE_NAME and
-# ARCH_IMAGE_TAG as zeek/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-{arm64,amd64},
+# IMAGE_TAG as zeek/${IMAGE_NAME}:${IMAGE_TAG}-{arm64,amd64}, pushes them
-# pushes them to the registry, then creates a manifest based on MANIFEST_NAME
+# to the registry, then creates a manifest as zeek/${IMAGE_NAME}:${IMAGE_TAG}
-# and MANIFEST_TAG environment variables as zeek/${MANIFEST_NAME}:${MANIFEST_TAG}
+# containing the arch specific tags and pushes it.
-# including the two tags.
 #
 # REGISTRY_PREFIX can be used to prefix images with a registry. Needs
 # to end with a slash.
+#
 set -eux
 
 REGISTRY_PREFIX=${REGISTRY_PREFIX:-}
 ZEEK_IMAGE_REPO=${ZEEK_IMAGE_REPO:-zeek}
 
+ADDITIONAL_MANIFEST_TAGS=${ADDITIONAL_MANIFEST_TAGS:-}
+
 # Check for ending slash in registry prefix
 if [ -n "${REGISTRY_PREFIX}" ]; then
     if [[ ! "${REGISTRY_PREFIX}" =~ .+/$ ]]; then
@@ -26,13 +28,34 @@ if [ -n "${REGISTRY_PREFIX}" ]; then
     fi
 fi
 
-docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64
+# Forward arguments to docker and retry the command once if failing (e.g network issues).
-docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
+function do_docker {
-docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64
+    if ! docker "$@"; then
-docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
+        echo "docker invocation failed. retrying in 5 seconds." >&2
+        sleep 5
+        docker "$@"
+    fi
+}
 
-docker manifest create ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} \
+function create_and_push_manifest {
-    ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 \
+    # Expects $1 to be the manifest tag, globals otherwise
-    ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
+    do_docker manifest create --amend ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${1} \
+        ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64 \
+        ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64
 
-docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG}
+    do_docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$IMAGE_NAME:${1}
+}
+
+do_docker tag zeek/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64
+do_docker tag zeek/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64
+do_docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64
+do_docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64
+
+create_and_push_manifest ${IMAGE_TAG}
+
+if [ -n "${ADDITIONAL_MANIFEST_TAGS}" ]; then
+    # Rely on default IFS splitting on space
+    for tag in ${ADDITIONAL_MANIFEST_TAGS}; do
+        create_and_push_manifest ${tag}
+    done
+fi

ci/find-current-version.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+#
+# Helper script to return the current lts or feature version based on
+# the branch and tags of Zeek's repository.
+#
+# * lts is the highest sorting release/x.0 branch which contains a tag
+#   of the form form '^v{x}.0.0$'.
+#
+# * feature is the highest sorting release/x.y branch that contains
+#   a '^v{x}.{y}.0$' tag.
+#
+set -euo pipefail
+
+REMOTE=${REMOTE:-origin}
+MAIN_BRANCH=${MAIN_BRANCH:-refs/remotes/${REMOTE}/master}
+
+function usage() {
+    echo "Usage $0 <lts|feature>" >&2
+    exit 1
+}
+
+if [ $# -ne 1 ]; then
+    usage
+fi
+
+if [ "${1}" = "lts" ]; then
+    PATTERN=".* refs/remotes/${REMOTE}/release/[0-9]+\.0\$"
+elif [ "${1}" = "feature" ]; then
+    PATTERN=".* refs/remotes/${REMOTE}/release/[0-9]+\.[0-9]+\$"
+else
+    usage
+fi
+
+# Iterate through all candidate branches, determine if a corresponding
+# v{x}.{y}.0 tag exists for that branch. If so, that'll be the most recent
+# (highest sorting) branch where we had a release.
+for ref in $(git show-ref | grep -E "${PATTERN}" | awk '{ print $2 }' | sort -rn); do
+    version=$(echo $ref | sed -E 's,^.*/(.+)$,\1,g')
+    tag_ref="refs/tags/v${version}.0"
+
+    # Find the commit for that tag.
+    tag_obj=$(git rev-list -n 1 "${tag_ref}" 2>/dev/null || true)
+
+    # If there's no .0 tag, there hasn't been an initial release on
+    # that branch yet, so move on to the next one.
+    if [ -z "${tag_obj}" ]; then
+        continue
+    fi
+
+    # We're probably safe, but do verify that the found tag_obj is
+    # somewhere between the merge base and the tip of the branch.
+    merge_base=$(git merge-base $MAIN_BRANCH $ref)
+    if git rev-list ${merge_base}..${ref} | grep -q "^${tag_obj}$"; then
+        echo "${version}"
+        exit 0
+    fi
+done
+
+exit 1

docker/Makefile

@@ -1,10 +1,9 @@
 # See the file "COPYING" in the main distribution directory for copyright.
 
 VERSION := $(shell cat ../VERSION)
-DOCKER ?= docker
 BUILD_IMAGE := zeek-builder:$(VERSION)
 BUILD_CONTAINER := zeek-builder-container-$(VERSION)
-ZEEK_IMAGE ?= zeek:$(VERSION)"
+ZEEK_IMAGE ?= zeek:$(VERSION)
 BUILD_DIR ?= build-docker
 ZEEK_CONFIGURE_FLAGS ?= \
 	--build-dir=$(BUILD_DIR) \