mirror of
https://github.com/zeek/zeek.git
synced 2025-10-01 22:28:20 +00:00
cirrus/container: Polish, fix and automated pushing of images
* Add ci/find-current-version.sh to figure out versions of lts and feature branch based on repository state. * Do not use ZEEK_IMAGE_REPO for the local images that are expected to exist. * Only use IMAGE_NAME/IMAGE_TAG instead of the ARCH and MANIFEST versions that contained the very same values * Support and push additional tags In addition to the lts and feature tags, also push the base version tags. 5.0 or 5.1 such that someone pulling zeek/zeek:5.0 will get the most recent available 5.0 patch release.
This commit is contained in:
Arne Welzel
parent
6047846353
commit
357bdc71ad
3 changed files with 148 additions and 37 deletions
75
.cirrus.yml
75
.cirrus.yml
|
|
@ -412,6 +412,9 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE
|
|||
cpu: *CPUS
|
||||
memory: *MEMORY
|
||||
set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV
|
||||
only_if: >
|
||||
( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) ||
|
||||
$CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' || $CIRRUS_TAG != '' )
|
||||
|
||||
env:
|
||||
ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps --disable-cpp-tests --disable-broker-tests
|
||||
|
|
@ -454,7 +457,6 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE
|
|||
test_script:
|
||||
- docker tag ${IMAGE_TAG} zeek:latest
|
||||
- make -C docker/btest
|
||||
<< : *BRANCH_WHITELIST
|
||||
|
||||
arm64_container_image_docker_builder:
|
||||
env:
|
||||
|
|
@ -472,9 +474,7 @@ container_image_manifest_docker_builder:
|
|||
only_if: >
|
||||
( $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' &&
|
||||
( $CIRRUS_BRANCH == 'master' ||
|
||||
( $CIRRUS_BRANCH =~ 'release/.*' && $CIRRUS_TAG != '')
|
||||
)
|
||||
)
|
||||
$CIRRUS_TAG =~ 'v[0-9]+\.[0-9]+\.[0-9]+$' ) )
|
||||
env:
|
||||
DOCKER_USERNAME: ENCRYPTED[!505b3dee552a395730a7e79e6aab280ffbe1b84ec62ae7616774dfefe104e34f896d2e20ce3ad701f338987c13c33533!]
|
||||
DOCKER_PASSWORD: ENCRYPTED[!6c4b2f6f0e5379ef1091719cc5d2d74c90cfd2665ac786942033d6d924597ffb95dbbc1df45a30cc9ddeec76c07ac620!]
|
||||
|
|
@ -491,29 +491,62 @@ container_image_manifest_docker_builder:
|
|||
# zeek/zeek-dev:latest-<amd64|arm64>
|
||||
# and using these, create a manifest of the form zeek/zeek:${CIRRUS_TAG}
|
||||
# for tags, or zeek/zeek-dev:latest for pushes to master.
|
||||
set -x
|
||||
if [ -n "${CIRRUS_TAG}" ]; then
|
||||
echo "MANIFEST_NAME=zeek" >> $CIRRUS_ENV
|
||||
echo "MANIFEST_TAG=$(cat VERSION)" >> $CIRRUS_ENV
|
||||
echo "ARCH_IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV
|
||||
echo "ARCH_IMAGE_NAME=zeek" >> $CIRRUS_ENV
|
||||
echo "IMAGE_TAG=$(cat VERSION)" >> $CIRRUS_ENV
|
||||
echo "IMAGE_NAME=zeek" >> $CIRRUS_ENV
|
||||
elif [ "${CIRRUS_BRANCH}" = "master" ]; then
|
||||
echo "MANIFEST_NAME=zeek-dev" >> $CIRRUS_ENV
|
||||
echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV
|
||||
echo "ARCH_IMAGE_NAME=zeek-dev" >> $CIRRUS_ENV
|
||||
echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
|
||||
echo "IMAGE_NAME=zeek-dev" >> $CIRRUS_ENV
|
||||
echo "IMAGE_TAG=latest" >> $CIRRUS_ENV
|
||||
# Hunk for testing and pushing into zeek/zeek-next. Make sure
|
||||
# to allow the branch in the above only_if attribute of this task.
|
||||
# elif [ "${CIRRUS_BRANCH}" = "topic/awelzel/2674-arm64-containers-on-cirrus" ]; then
|
||||
# echo "MANIFEST_NAME=zeek-next" >> $CIRRUS_ENV
|
||||
# echo "MANIFEST_TAG=latest" >> $CIRRUS_ENV
|
||||
# echo "ARCH_IMAGE_NAME=zeek-next" >> $CIRRUS_ENV
|
||||
# echo "ARCH_IMAGE_TAG=latest" >> $CIRRUS_ENV
|
||||
# echo "IMAGE_NAME=zeek-next" >> $CIRRUS_ENV
|
||||
# echo "IMAGE_TAG=latest" >> $CIRRUS_ENV
|
||||
else
|
||||
echo "Bad tag/branch for container_image_manifest"
|
||||
env
|
||||
exit 1
|
||||
fi
|
||||
|
||||
set_additional_manifest_tags_script: |
|
||||
set -x
|
||||
if [ -z "${CIRRUS_TAG}" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Populate the checkout with all the repository information we need
|
||||
# to determine what the current feature and lts versions are.
|
||||
git fetch --tags origin \
|
||||
'+refs/heads/release/*:refs/remotes/origin/release/*' \
|
||||
'+refs/heads/master:refs/remotes/origin/master'
|
||||
|
||||
# Find current versions for lts and feature depending on branches and
|
||||
# tags in the repo. sed for escaping the dot in the version for using
|
||||
# it in the regex below to match against CIRRUS_TAG.
|
||||
lts_ver=$(./ci/find-current-version.sh lts)
|
||||
lts_pat="^v$(echo $lts_ver | sed 's,\.,\\.,g')\.[0-9]+\$"
|
||||
feature_ver=$(./ci/find-current-version.sh feature)
|
||||
feature_pat="^v$(echo $feature_ver | sed 's,\.,\\.,g')\.[0-9]+\$"
|
||||
|
||||
# Construct additional tags for the image. At most this will
|
||||
# be "lts x.0 feature" for an lts branch x.0 that is currently
|
||||
# also the latest feature branch.
|
||||
ADDL_MANIFEST_TAGS=
|
||||
if echo "${CIRRUS_TAG}" | grep -E "${lts_pat}"; then
|
||||
ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} lts ${lts_ver}"
|
||||
fi
|
||||
|
||||
if echo "${CIRRUS_TAG}" | grep -E "${feature_pat}"; then
|
||||
ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} latest"
|
||||
if [ "${feature_ver}" != "${lts_ver}" ]; then
|
||||
ADDL_MANIFEST_TAGS="${ADDL_MANIFEST_TAGS} ${feature_ver}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Let downstream know about it.
|
||||
echo "ADDITIONAL_MANIFEST_TAGS=${ADDL_MANIFEST_TAGS}" >> $CIRRUS_ENV
|
||||
|
||||
# These should've been populated by the previous jobs
|
||||
zeek_image_arm64_cache:
|
||||
folder: /tmp/zeek-image-cache-arm64
|
||||
|
|
@ -534,20 +567,17 @@ container_image_manifest_docker_builder:
|
|||
- REGISTRY_PREFIX=public.ecr.aws/ ./ci/container-images-tag-and-push.sh
|
||||
|
||||
# Continue to push tags to the "zeekurity" account as well.
|
||||
- docker tag zeek/zeek-multiarch:arm64 zeekurity/zeek-multiarch:arm64
|
||||
- docker tag zeek/zeek-multiarch:amd64 zeekurity/zeek-multiarch:amd64
|
||||
- ZEEK_IMAGE_REPO=zeekurity ./ci/container-images-tag-and-push.sh
|
||||
depends_on:
|
||||
# Only push out the image if all the btests succeeded and the
|
||||
# images have been built.
|
||||
- arm_debian11
|
||||
- debian11
|
||||
- arm64_container_image
|
||||
- amd64_container_image
|
||||
|
||||
cluster_testing_docker_builder:
|
||||
cpu: *CPUS
|
||||
memory: *MEMORY
|
||||
only_if: >
|
||||
( ( $CIRRUS_PR != '' && $CIRRUS_BRANCH !=~ 'dependabot/.*' ) ||
|
||||
$CIRRUS_BRANCH == 'master' || $CIRRUS_BRANCH =~ 'release/.*' || $CIRRUS_TAG != '' )
|
||||
env:
|
||||
CIRRUS_LOG_TIMESTAMP: true
|
||||
# At this point, zeek-testing-cluster checks for "GITHUB_ACTION" to
|
||||
|
|
@ -572,4 +602,3 @@ cluster_testing_docker_builder:
|
|||
path: "testing/external/zeek-testing-cluster/.tmp/**"
|
||||
depends_on:
|
||||
- amd64_container_image
|
||||
<< : *BRANCH_WHITELIST
|
||||
|
|
|
|||
|
|
@ -1,23 +1,25 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# This script expects two local images in the local container registry:
|
||||
# This script expects two images in the local container registry:
|
||||
#
|
||||
# zeek/zeek-multiarch:arm64
|
||||
# zeek/zeek-multiarch:amd64
|
||||
#
|
||||
# It retags these according to the environment ARCH_IMAGE_NAME and
|
||||
# ARCH_IMAGE_TAG as zeek/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-{arm64,amd64},
|
||||
# pushes them to the registry, then creates a manifest based on MANIFEST_NAME
|
||||
# and MANIFEST_TAG environment variables as zeek/${MANIFEST_NAME}:${MANIFEST_TAG}
|
||||
# including the two tags.
|
||||
# It retags these according to the environment variables IMAGE_NAME and
|
||||
# IMAGE_TAG as zeek/${IMAGE_NAME}:${IMAGE_TAG}-{arm64,amd64}, pushes them
|
||||
# to the registry, then creates a manifest as zeek/${IMAGE_NAME}:${IMAGE_TAG}
|
||||
# containing the arch specific tags and pushes it.
|
||||
#
|
||||
# REGISTRY_PREFIX can be used to prefix images with a registry. Needs
|
||||
# to end with a slash.
|
||||
#
|
||||
set -eux
|
||||
|
||||
REGISTRY_PREFIX=${REGISTRY_PREFIX:-}
|
||||
ZEEK_IMAGE_REPO=${ZEEK_IMAGE_REPO:-zeek}
|
||||
|
||||
ADDITIONAL_MANIFEST_TAGS=${ADDITIONAL_MANIFEST_TAGS:-}
|
||||
|
||||
# Check for ending slash in registry prefix
|
||||
if [ -n "${REGISTRY_PREFIX}" ]; then
|
||||
if [[ ! "${REGISTRY_PREFIX}" =~ .+/$ ]]; then
|
||||
|
|
@ -26,13 +28,34 @@ if [ -n "${REGISTRY_PREFIX}" ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64
|
||||
docker tag ${ZEEK_IMAGE_REPO}/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
|
||||
docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64
|
||||
docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
|
||||
# Forward arguments to docker and retry the command once if failing (e.g network issues).
|
||||
function do_docker {
|
||||
if ! docker "$@"; then
|
||||
echo "docker invocation failed. retrying in 5 seconds." >&2
|
||||
sleep 5
|
||||
docker "$@"
|
||||
fi
|
||||
}
|
||||
|
||||
docker manifest create ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG} \
|
||||
${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-arm64 \
|
||||
${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${ARCH_IMAGE_NAME}:${ARCH_IMAGE_TAG}-amd64
|
||||
function create_and_push_manifest {
|
||||
# Expects $1 to be the manifest tag, globals otherwise
|
||||
do_docker manifest create --amend ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${1} \
|
||||
${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64 \
|
||||
${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64
|
||||
|
||||
docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$MANIFEST_NAME:${MANIFEST_TAG}
|
||||
do_docker manifest push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/$IMAGE_NAME:${1}
|
||||
}
|
||||
|
||||
do_docker tag zeek/zeek-multiarch:arm64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64
|
||||
do_docker tag zeek/zeek-multiarch:amd64 ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64
|
||||
do_docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-arm64
|
||||
do_docker push ${REGISTRY_PREFIX}${ZEEK_IMAGE_REPO}/${IMAGE_NAME}:${IMAGE_TAG}-amd64
|
||||
|
||||
create_and_push_manifest ${IMAGE_TAG}
|
||||
|
||||
if [ -n "${ADDITIONAL_MANIFEST_TAGS}" ]; then
|
||||
# Rely on default IFS splitting on space
|
||||
for tag in ${ADDITIONAL_MANIFEST_TAGS}; do
|
||||
create_and_push_manifest ${tag}
|
||||
done
|
||||
fi
|
||||
|
|
|
|||
59
ci/find-current-version.sh
Executable file
59
ci/find-current-version.sh
Executable file
|
|
@ -0,0 +1,59 @@
|
|||
#!/bin/bash
|
||||
#
|
||||
# Helper script to return the current lts or feature version based on
|
||||
# the branch and tags of Zeek's repository.
|
||||
#
|
||||
# * lts is the highest sorting release/x.0 branch which contains a tag
|
||||
# of the form form '^v{x}.0.0$'.
|
||||
#
|
||||
# * feature is the highest sorting release/x.y branch that contains
|
||||
# a '^v{x}.{y}.0$' tag.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
REMOTE=${REMOTE:-origin}
|
||||
MAIN_BRANCH=${MAIN_BRANCH:-refs/remotes/${REMOTE}/master}
|
||||
|
||||
function usage() {
|
||||
echo "Usage $0 <lts|feature>" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
if [ $# -ne 1 ]; then
|
||||
usage
|
||||
fi
|
||||
|
||||
if [ "${1}" = "lts" ]; then
|
||||
PATTERN=".* refs/remotes/${REMOTE}/release/[0-9]+\.0\$"
|
||||
elif [ "${1}" = "feature" ]; then
|
||||
PATTERN=".* refs/remotes/${REMOTE}/release/[0-9]+\.[0-9]+\$"
|
||||
else
|
||||
usage
|
||||
fi
|
||||
|
||||
# Iterate through all candidate branches, determine if a corresponding
|
||||
# v{x}.{y}.0 tag exists for that branch. If so, that'll be the most recent
|
||||
# (highest sorting) branch where we had a release.
|
||||
for ref in $(git show-ref | grep -E "${PATTERN}" | awk '{ print $2 }' | sort -rn); do
|
||||
version=$(echo $ref | sed -E 's,^.*/(.+)$,\1,g')
|
||||
tag_ref="refs/tags/v${version}.0"
|
||||
|
||||
# Find the commit for that tag.
|
||||
tag_obj=$(git rev-list -n 1 "${tag_ref}" 2>/dev/null || true)
|
||||
|
||||
# If there's no .0 tag, there hasn't been an initial release on
|
||||
# that branch yet, so move on to the next one.
|
||||
if [ -z "${tag_obj}" ]; then
|
||||
continue
|
||||
fi
|
||||
|
||||
# We're probably safe, but do verify that the found tag_obj is
|
||||
# somewhere between the merge base and the tip of the branch.
|
||||
merge_base=$(git merge-base $MAIN_BRANCH $ref)
|
||||
if git rev-list ${merge_base}..${ref} | grep -q "^${tag_obj}$"; then
|
||||
echo "${version}"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
|
||||
exit 1
|
||||
Loading…
Add table
Add a link
Reference in a new issue