From 4387532613f30449b9aad0263da09e0b7db9e492 Mon Sep 17 00:00:00 2001
From: Arne Welzel <arne.welzel@corelight.com>
Date: Thu, 4 May 2023 14:58:51 +0200
Subject: [PATCH] ci: Add public-ecr-vacuum.sh

---
 .cirrus.yml              | 15 ++++++++++
 ci/public-ecr-cleanup.sh | 63 ++++++++++++++++++++++++++++++++++++++++
 2 files changed, 78 insertions(+)
 create mode 100755 ci/public-ecr-cleanup.sh

diff --git a/.cirrus.yml b/.cirrus.yml
index 32ab5adbd0..2948b6d740 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -612,6 +612,21 @@ container_image_manifest_docker_builder:
     - arm64_container_image
     - amd64_container_image
 
+# Once we've published new images in container_image_manifest, remove any untagged
+# images from the public ECR repository to stay within free-tier bounds.
+public_ecr_cleanup_docker_builder:
+  cpu: 1
+  only_if: >
+    $CIRRUS_CRON == '' && $CIRRUS_REPO_FULL_NAME == 'zeek/zeek' && $CIRRUS_BRANCH == 'master'
+  env:
+    AWS_ACCESS_KEY_ID: ENCRYPTED[!eff52f6442e1bc78bce5b15a23546344df41bf519f6201924cb70c7af12db23f442c0e5f2b3687c2d856ceb11fcb8c49!]
+    AWS_SECRET_ACCESS_KEY: ENCRYPTED[!748bc302dd196140a5fa8e89c9efd148882dc846d4e723787d2de152eb136fa98e8dea7e6d2d6779d94f72dd3c088228!]
+    AWS_REGION: us-east-1
+  cleanup_script:
+    - ./ci/public-ecr-cleanup.sh
+  depends_on:
+    - container_image_manifest
+
 cluster_testing_docker_builder:
   cpu: *CPUS
   memory: *MEMORY
diff --git a/ci/public-ecr-cleanup.sh b/ci/public-ecr-cleanup.sh
new file mode 100755
index 0000000000..7a1f7c809e
--- /dev/null
+++ b/ci/public-ecr-cleanup.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+#
+# Script to batch-delete all untagged images from ECR public repositories,
+# defaulting to the zeek/zeek-dev repository.
+# First scans for manifest list images that are referencing other images and
+# deletes them, then deletes all remaining untagged images.
+set -eu
+
+if ! command -v aws >/dev/null; then
+    echo "missing aws command" >&2
+    exit 1
+fi
+
+REGISTRY_ID=${REGISTRY_ID:-103243056077}
+REPOSITORY_NAME=${REPOSITORY_NAME:-zeek-dev}
+BATCH_DELETE_SIZE=${BATCH_DELETE_SIZE:-50}
+
+# Chunk up "$1" into BATCH_DELETE_SIZE entries and batch-delete them at once
+# via aws batch-delete.
+#
+# Expected input looks as follows to keep things simple:
+#
+#    imageDigest=sha256:db6...366
+#    imageDigest=sha256:2ad...9b0
+#
+function batch_delete {
+    while read -r batch; do
+        if [ -z "${batch}" ]; then
+            break
+        fi
+
+        echo "Deleting ${batch}"
+        aws ecr-public batch-delete-image \
+            --registry-id "${REGISTRY_ID}" \
+            --repository-name "${REPOSITORY_NAME}" \
+            --image-ids ${batch}
+
+    done < <(xargs -L ${BATCH_DELETE_SIZE} <<<"$1")
+}
+
+# Find all untagged manifest lists with the following media types:
+#
+#    application/vnd.docker.distribution.manifest.list.v2+json
+#    application/vnd.oci.image.index.v1+json
+#
+# These reference other images, so we need to delete them first as
+# otherwise the referenced images can not be deleted.
+IMAGE_DIGESTS=$(aws ecr-public describe-images \
+    --registry-id "${REGISTRY_ID}" \
+    --repository-name "${REPOSITORY_NAME}" \
+    --query 'imageDetails[?!imageTags && (contains(imageManifestMediaType, `manifest.list.v2`) || contains(imageManifestMediaType, `image.index.v1`))].{imageDigest: join(`=`, [`imageDigest`, imageDigest])}' \
+    --output text)
+
+batch_delete "${IMAGE_DIGESTS}"
+
+# Now find all untagged manifests that are left.
+IMAGE_DIGESTS=$(aws ecr-public describe-images \
+    --registry-id "${REGISTRY_ID}" \
+    --repository-name "${REPOSITORY_NAME}" \
+    --query 'imageDetails[?!imageTags].{imageDigest: join(`=`, [`imageDigest`, imageDigest])}' \
+    --output text)
+
+batch_delete "${IMAGE_DIGESTS}"