From 44c19086e8dbde76ab25bf1ca94f910c5d6f5a3c Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 1 Feb 2023 11:41:50 +0100 Subject: [PATCH 1/4] cirrus: Add back testing of container images --- .cirrus.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.cirrus.yml b/.cirrus.yml index 9e4d40d3b6..ef9209f565 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -441,6 +441,9 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE build_final_script: - cd docker && docker build --cache-from ${IMAGE_TAG} -t ${IMAGE_TAG} -f final.Dockerfile . - docker save ${IMAGE_TAG} | zstd > ${ZEEK_IMAGE_CACHE_DIR}/final.zst + test_script: + - docker tag ${IMAGE_TAG} zeek:latest + - make -C docker/btest arm64_container_image_docker_builder: env: From 4523f678d2d511a99e6c51258598c0c4abc7d1f3 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 1 Feb 2023 12:23:05 +0100 Subject: [PATCH 2/4] docker: Update Makefile/README to align with the Cirrus CI setup --- docker/Dockerfile | 63 ----------------------------------------------- docker/Makefile | 24 +++++++++++++++--- docker/README | 19 ++++++++++---- 3 files changed, 35 insertions(+), 71 deletions(-) delete mode 100644 docker/Dockerfile diff --git a/docker/Dockerfile b/docker/Dockerfile deleted file mode 100644 index 8d8673f097..0000000000 --- a/docker/Dockerfile +++ /dev/null @@ -1,63 +0,0 @@ -# See the file "COPYING" in the main distribution directory for copyright. - -# Layer to build Zeek. -FROM debian:bullseye-slim AS build - -# Configure system for build. -RUN apt-get -q update \ - && apt-get install -q -y --no-install-recommends \ - bind9 \ - bison \ - cmake \ - flex \ - g++ \ - gcc \ - libfl2 \ - libfl-dev \ - libmaxminddb-dev \ - libpcap-dev \ - libssl-dev \ - libz-dev \ - make \ - python3-minimal \ - python3.9-dev \ - swig \ - ninja-build \ - python3-pip \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Copy over the Zeek source tree. -# NOTE: This assumes that we build in the context of the parent directory (the -# Git checkout root). We e.g., ensure that in the `Makefile` in this directory. -COPY . /src/zeek -RUN make -C /src/zeek distclean - -ARG CONFFLAGS="--generator=Ninja --build-type=Release" - -WORKDIR /src/zeek -RUN ./configure $CONFFLAGS && ninja -C build install - -# Final layer containing all artifacts. -FROM debian:bullseye-slim AS final - -RUN apt-get -q update \ - && apt-get install -q -y --no-install-recommends \ - ca-certificates \ - git \ - libmaxminddb0 \ - libpython3.9 \ - libpcap0.8 \ - libssl1.1 \ - libz1 \ - python3-minimal \ - python3-git \ - python3-semantic-version \ - python3-websocket \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - -# Copy over Zeek installation. -COPY --from=build /usr/local/zeek /usr/local/zeek -ENV PATH "/usr/local/zeek/bin:${PATH}" -ENV PYTHONPATH "/usr/local/zeek/lib/zeek/python:${PYTHONPATH}" diff --git a/docker/Makefile b/docker/Makefile index 696afc690d..9fdd002b5b 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,12 +1,30 @@ # See the file "COPYING" in the main distribution directory for copyright. -VERSION := $$(cat ../VERSION) +VERSION := $(shell cat ../VERSION) +DOCKER ?= docker +BUILD_IMAGE := zeek-builder:$(VERSION) +BUILD_CONTAINER := zeek-builder-container-$(VERSION) +ZEEK_IMAGE ?= zeek:$(VERSION)" +BUILD_DIR ?= build-docker +ZEEK_CONFIGURE_FLAGS ?= \ + --build-dir=$(BUILD_DIR) \ + --generator=Ninja \ + --build-type=Release \ + --disable-btest-pcaps \ + --disable-broker-tests \ + --disable-cpp-tests .PHONY: all all: - @docker build -t zeek:$(VERSION) -f Dockerfile .. - @docker tag zeek:$(VERSION) zeek:latest + -docker rm $(BUILD_CONTAINER) + docker build -t $(BUILD_IMAGE) -f builder.Dockerfile . + docker run --name $(BUILD_CONTAINER) \ + -v $(CURDIR)/../:/src/zeek -w /src/zeek \ + $(BUILD_IMAGE) bash -xeu -c "./configure $(ZEEK_CONFIGURE_FLAGS) && ninja -C $(BUILD_DIR) install" + docker commit $(BUILD_CONTAINER) zeek-build + docker build -t $(ZEEK_IMAGE) -f final.Dockerfile . + docker tag $(ZEEK_IMAGE) zeek:latest test: @TEST_TAG=zeek:$(VERSION) $(MAKE) -C btest diff --git a/docker/README b/docker/README index 8d3280f161..ae0011d747 100644 --- a/docker/README +++ b/docker/README @@ -1,19 +1,28 @@ Container image for Zeek ======================== -This directory contains a minimal container image for Zeek. This image is -published automatically to [DockerHub](https://hub.docker.com/u/zeekurity) for +This directory allows building a minimal container image for Zeek. This image is +published automatically to [DockerHub](https://hub.docker.com/u/zeek) for releases and for commits on the `master` branch. - Images for release are published as - [`zeekurity/zeek`](https://hub.docker.com/r/zeekurity/zeek) with the `latest` + [`zeek/zeek`](https://hub.docker.com/r/zeek/zeek) with the `latest` tag pointing to the latest release. - Development images for the `master` branch are published as - [`zeekurity/zeek-dev`](https://hub.docker.com/r/zeekurity/zeek-dev). + [`zeek/zeek-dev`](https://hub.docker.com/r/zeek/zeek-dev). +These images are also pushed to Amazon ECR Public Gallery under +the [Zeek Project](https://gallery.ecr.aws/zeek). To run the image execute e.g.,: - docker run -it zeekurity/zeek + docker run -it zeek/zeek To build the image execute `make` from this directory. + +CI +-- + +Note, the image build process on Cirrus CI is not using the Makefile, +but instead open-codes the invocation due to leveraging Cirrus CI caching +and ccache heavily. From 7efd9ad1f5e473c75c3a6501b206683040f4186a Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 1 Feb 2023 12:27:18 +0100 Subject: [PATCH 3/4] cirrus: No broker-tests and no cpp-tests for images --- .cirrus.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cirrus.yml b/.cirrus.yml index ef9209f565..4c0909a807 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -404,7 +404,7 @@ docker_build_template: &DOCKER_BUILD_TEMPLATE set_image_tag_script: echo "IMAGE_TAG=zeek/zeek-multiarch:${CIRRUS_ARCH}" >> $CIRRUS_ENV env: - ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps + ZEEK_CONFIGURE_FLAGS: --ccache --generator=Ninja --build-type=Release --disable-btest-pcaps --disable-cpp-tests --disable-broker-tests CIRRUS_LOG_TIMESTAMP: true BUILDER_IMAGE_CACHE_DIR: /tmp/builder-image-cache ZEEK_IMAGE_CACHE_DIR: /tmp/zeek-image-cache-${CIRRUS_ARCH} From 13b0e7a2aa96af4d19810446869c85771cf1e9fb Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Wed, 1 Feb 2023 12:33:09 +0100 Subject: [PATCH 4/4] docker: Extend tests to run a very basic Zeek script Mostly because a `-v` shortcuts quite, while a zeek -e 'print zeek_version()' will also parse all base scripts, etc. --- docker/btest/Baseline/docker.structure_tests/output | 1 + docker/btest/docker/structure_tests.sh | 3 +++ 2 files changed, 4 insertions(+) diff --git a/docker/btest/Baseline/docker.structure_tests/output b/docker/btest/Baseline/docker.structure_tests/output index 0c8a07377a..9caa011cea 100644 --- a/docker/btest/Baseline/docker.structure_tests/output +++ b/docker/btest/Baseline/docker.structure_tests/output @@ -1,5 +1,6 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. zeek version xxx +zeek version xxx XXX [sources] zeek = https://github.com/zeek/packages diff --git a/docker/btest/docker/structure_tests.sh b/docker/btest/docker/structure_tests.sh index 17a54aa86d..0a890a3488 100644 --- a/docker/btest/docker/structure_tests.sh +++ b/docker/btest/docker/structure_tests.sh @@ -7,6 +7,9 @@ TEST_TAG=${TEST_TAG:-zeek:latest} # Check that `zeek` can be run. docker run --rm "${TEST_TAG}" zeek -v | sed 's/\(zeek version\) .*/\1 xxx/' +# ...and load and execute some basic scripts, too. +docker run --rm "${TEST_TAG}" zeek -e 'print fmt("zeek version %s", zeek_version())' | sed 's/\(zeek version\) .*/\1 xxx/' + # Check that this is a release build. docker run --rm "${TEST_TAG}" zeek-config --build_type | grep -q 'release'