diff --git a/.cirrus.yml b/.cirrus.yml index 5c6c8fff73..f902d111b6 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -412,6 +412,9 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE cpu: *CPUS memory: *MEMORY set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV + only_if: > + ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) || + $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' || $CIRRUS_TAG != '' ) env: ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps --disable-cpp-tests --disable-broker-tests @@ -454,7 +457,6 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE test_script: - docker tag ${IMAGE_TAG} zeek:latest - make -C docker/btest - << : *BRANCH_WHITELIST arm64_container_image_docker_builder: env: @@ -472,9 +474,7 @@ container_image_manifest_docker_builder: only_if: > ( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' && ( $CIRRUS_BRANCH == 'master' || - ( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '') - ) - ) + $CIRRUS_TAG =~ 'v[0-9]+\.[0-9]+\.[0-9]+$' ) ) env: DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!] DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!] @@ -491,29 +491,62 @@ container_image_manifest_docker_builder: # zeek/zeek-dev:latest- # and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG} # for tags, or zeek/zeek-dev:latest for pushes to master. + set -x if [ -n "${CIRRUS_TAG}" ]; then - echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV - echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV - echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV - echo "ARCH_IMAGE_NAME=zeek" >> $CIRRUS_ENV + echo "IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV + echo "IMAGE_NAME=zeek" >> $CIRRUS_ENV elif [ "${CIRRUS_BRANCH}" = "master" ]; then - echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV - echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV - echo "ARCH_IMAGE_NAME=zeek-dev" >> $CIRRUS_ENV - echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV + echo "IMAGE_NAME=zeek-dev" >> $CIRRUS_ENV + echo "IMAGE_TAG=latest" >> $CIRRUS_ENV # Hunk for testing and pushing into zeek/zeek-next. Make sure # to allow the branch in the above only_if attribute of this task. # elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then - # echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV - # echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV - # echo "ARCH_IMAGE_NAME=zeek-next" >> $CIRRUS_ENV - # echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV + # echo "IMAGE_NAME=zeek-next" >> $CIRRUS_ENV + # echo "IMAGE_TAG=latest" >> $CIRRUS_ENV else echo "Bad tag/branch for container_image_manifest" env exit 1 fi + set_additional_manifest_tags_script: | + set -x + if [ -z "${CIRRUS_TAG}" ]; then + exit 0 + fi + + # Populate the checkout with all the repository information we need + # to determine what the current feature and lts versions are. + git fetch --tags origin \ + '+refs/heads/release/*:refs/remotes/origin/release/*' \ + '+refs/heads/master:refs/remotes/origin/master' + + # Find current versions for lts and feature depending on branches and + # tags in the repo. sed for escaping the dot in the version for using + # it in the regex below to match against CIRRUS_TAG. + lts_ver=$(./ci/find-current-version.sh lts) + lts_pat="^v$(echo $lts_ver | sed 's,\.,\\.,g')\.[0-9]+\$" + feature_ver=$(./ci/find-current-version.sh feature) + feature_pat="^v$(echo $feature_ver | sed 's,\.,\\.,g')\.[0-9]+\$" + + # Construct additional tags for the image. At most this will + # be "lts x.0 feature" for an lts branch x.0 that is currently + # also the latest feature branch. + ADDL_MANIFEST_TAGS= + if echo "${CIRRUS_TAG}" | grep -E "${lts_pat}"; then + ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} lts ${lts_ver}" + fi + + if echo "${CIRRUS_TAG}" | grep -E "${feature_pat}"; then + ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} latest" + if [ "${feature_ver}" != "${lts_ver}" ]; then + ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} ${feature_ver}" + fi + fi + + # Let downstream know about it. + echo "ADDITIONAL_MANIFEST_TAGS=${ADDL_MANIFEST_TAGS}" >> $CIRRUS_ENV + # These should've been populated by the previous jobs zeek_image_arm64_cache: folder: /tmp/zeek-image-cache-arm64 @@ -534,20 +567,17 @@ container_image_manifest_docker_builder: - REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh # Continue to push tags to the "zeekurity" account as well. - - docker tag zeek/zeek-multiarch:arm64 zeekurity/zeek-multiarch:arm64 - - docker tag zeek/zeek-multiarch:amd64 zeekurity/zeek-multiarch:amd64 - ZEEK_IMAGE_REPO=zeekurity ./ci/container-images-tag-and-push.sh depends_on: - # Only push out the image if all the btests succeeded and the - # images have been built. - - arm_debian11 - - debian11 - arm64_container_image - amd64_container_image cluster_testing_docker_builder: cpu: *CPUS memory: *MEMORY + only_if: > + ( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) || + $CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' || $CIRRUS_TAG != '' ) env: CIRRUS_LOG_TIMESTAMP: true # At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to @@ -572,4 +602,3 @@ cluster_testing_docker_builder: path: "testing/external/zeek-testing-cluster/.tmp/**" depends_on: - amd64_container_image - << : *BRANCH_WHITELIST diff --git a/CHANGES b/CHANGES index f7d1cb95a7..333ad44260 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,24 @@ +6.0.0-dev.30 | 2023-02-13 12:10:50 +0100 + + * docker/Makefile: Remove stray quotes, unused DOCKER (Arne Welzel, Corelight) + + * cirrus/container: Polish, fix and automated pushing of images (Arne Welzel, Corelight) + + * Add ci/find-current-version.sh to figure out versions of lts and + feature branch based on repository state. + + * Do not use ZEEK_IMAGE_REPO for the local images that + are expected to exist. + + * Only use IMAGE_NAME/IMAGE_TAG instead of the ARCH and MANIFEST + versions that contained the very same values + + * Support and push additional tags + + In addition to the lts and feature tags, also push the base version + tags. 5.0 or 5.1 such that someone pulling zeek/zeek:5.0 will get + the most recent available 5.0 patch release. + 6.0.0-dev.27 | 2023-02-11 22:07:31 -0700 * better error reporting when ZAM code calls a function (Vern Paxson, Corelight) diff --git a/VERSION b/VERSION index c49565a362..3e84ff5620 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -6.0.0-dev.27 +6.0.0-dev.30 diff --git a/ci/container-images-tag-and-push.sh b/ci/container-images-tag-and-push.sh index 3eef706c34..10177dbe64 100755 --- a/ci/container-images-tag-and-push.sh +++ b/ci/container-images-tag-and-push.sh @@ -1,23 +1,25 @@ #!/bin/bash # -# This script expects two local images in the local container registry: +# This script expects two images in the local container registry: # # zeek/zeek-multiarch:arm64 # zeek/zeek-multiarch:amd64 # -# It retags these according to the environment ARCH_IMAGE_NAME and -# ARCH_IMAGE_TAG as zeek/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-{arm64,amd64}, -# pushes them to the registry, then creates a manifest based on MANIFEST_NAME -# and MANIFEST_TAG environment variables as zeek/${MANIFEST_NAME}:${MANIFEST_TAG} -# including the two tags. +# It retags these according to the environment variables IMAGE_NAME and +# IMAGE_TAG as zeek/${IMAGE_NAME}:${IMAGE_TAG}-{arm64,amd64}, pushes them +# to the registry, then creates a manifest as zeek/${IMAGE_NAME}:${IMAGE_TAG} +# containing the arch specific tags and pushes it. # # REGISTRY_PREFIX can be used to prefix images with a registry. Needs # to end with a slash. +# set -eux REGISTRY_PREFIX=${REGISTRY_PREFIX:-} ZEEK_IMAGE_REPO=${ZEEK_IMAGE_REPO:-zeek} +ADDITIONAL_MANIFEST_TAGS=${ADDITIONAL_MANIFEST_TAGS:-} + # Check for ending slash in registry prefix if [ -n "${REGISTRY_PREFIX}" ]; then if [[ ! "${REGISTRY_PREFIX}" =~ .+/$ ]]; then @@ -26,13 +28,34 @@ if [ -n "${REGISTRY_PREFIX}" ]; then fi fi -docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 -docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 -docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 -docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 +# Forward arguments to docker and retry the command once if failing (e.g network issues). +function do_docker { + if ! docker "$@"; then + echo "docker invocation failed. retrying in 5 seconds." >&2 + sleep 5 + docker "$@" + fi +} -docker manifest create ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} \ - ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 \ - ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64 +function create_and_push_manifest { + # Expects $1 to be the manifest tag, globals otherwise + do_docker manifest create --amend ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${1} \ + ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64 \ + ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64 -docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} + do_docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$IMAGE_NAME:${1} +} + +do_docker tag zeek/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64 +do_docker tag zeek/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64 +do_docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64 +do_docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64 + +create_and_push_manifest ${IMAGE_TAG} + +if [ -n "${ADDITIONAL_MANIFEST_TAGS}" ]; then + # Rely on default IFS splitting on space + for tag in ${ADDITIONAL_MANIFEST_TAGS}; do + create_and_push_manifest ${tag} + done +fi diff --git a/ci/find-current-version.sh b/ci/find-current-version.sh new file mode 100755 index 0000000000..85ea5edf56 --- /dev/null +++ b/ci/find-current-version.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# +# Helper script to return the current lts or feature version based on +# the branch and tags of Zeek's repository. +# +# * lts is the highest sorting release/x.0 branch which contains a tag +# of the form form '^v{x}.0.0$'. +# +# * feature is the highest sorting release/x.y branch that contains +# a '^v{x}.{y}.0$' tag. +# +set -euo pipefail + +REMOTE=${REMOTE:-origin} +MAIN_BRANCH=${MAIN_BRANCH:-refs/remotes/${REMOTE}/master} + +function usage() { + echo "Usage $0 " >&2 + exit 1 +} + +if [ $# -ne 1 ]; then + usage +fi + +if [ "${1}" = "lts" ]; then + PATTERN=".* refs/remotes/${REMOTE}/release/[0-9]+\.0\$" +elif [ "${1}" = "feature" ]; then + PATTERN=".* refs/remotes/${REMOTE}/release/[0-9]+\.[0-9]+\$" +else + usage +fi + +# Iterate through all candidate branches, determine if a corresponding +# v{x}.{y}.0 tag exists for that branch. If so, that'll be the most recent +# (highest sorting) branch where we had a release. +for ref in $(git show-ref | grep -E "${PATTERN}" | awk '{ print $2 }' | sort -rn); do + version=$(echo $ref | sed -E 's,^.*/(.+)$,\1,g') + tag_ref="refs/tags/v${version}.0" + + # Find the commit for that tag. + tag_obj=$(git rev-list -n 1 "${tag_ref}" 2>/dev/null || true) + + # If there's no .0 tag, there hasn't been an initial release on + # that branch yet, so move on to the next one. + if [ -z "${tag_obj}" ]; then + continue + fi + + # We're probably safe, but do verify that the found tag_obj is + # somewhere between the merge base and the tip of the branch. + merge_base=$(git merge-base $MAIN_BRANCH $ref) + if git rev-list ${merge_base}..${ref} | grep -q "^${tag_obj}$"; then + echo "${version}" + exit 0 + fi +done + +exit 1 diff --git a/docker/Makefile b/docker/Makefile index 9fdd002b5b..93d908d621 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,10 +1,9 @@ # See the file "COPYING" in the main distribution directory for copyright. VERSION := $(shell cat ../VERSION) -DOCKER ?= docker BUILD_IMAGE := zeek-builder:$(VERSION) BUILD_CONTAINER := zeek-builder-container-$(VERSION) -ZEEK_IMAGE ?= zeek:$(VERSION)" +ZEEK_IMAGE ?= zeek:$(VERSION) BUILD_DIR ?= build-docker ZEEK_CONFIGURE_FLAGS ?= \ --build-dir=$(BUILD_DIR) \