From ddae1398f1d8a9fe2b0253b2a54f4362717e0ec5 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 20 Aug 2021 07:54:41 +0200 Subject: [PATCH 1/3] Add a minimal containerized environment This patch adds a minimal Zeek environment packaged as a container. Since this is intended both as a base layer for other images and as a quick way to explore Zeek we install only zeek and zkg as basic functionality. Closes #1625. --- .github/workflows/docker.yml | 86 ++++++++++++++++++++++++++++ docker/Dockerfile | 60 +++++++++++++++++++ docker/Makefile | 8 +++ docker/README | 19 ++++++ docker/container-structure-test.yaml | 37 ++++++++++++ 5 files changed, 210 insertions(+) create mode 100644 .github/workflows/docker.yml create mode 100644 docker/Dockerfile create mode 100644 docker/Makefile create mode 100644 docker/README create mode 100644 docker/container-structure-test.yaml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 0000000000..79afc88acd --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,86 @@ +name: Check and publish Docker images + +on: + pull_request: + push: + branches: [master] + tags: + - 'v*' + - '!v*-dev' + - 'release' + +defaults: + run: + shell: bash + + +jobs: + build: + runs-on: ubuntu-latest + env: + TEST_TAG: zeek:latest + steps: + - uses: actions/checkout@v2 + with: + submodules: "recursive" + + # Create and boot a loader. This will e.g., provide caching + # so we avoid rebuilds of the same image after this step. + - uses: docker/setup-buildx-action@v1 + - name: Build + uses: docker/build-push-action@v2 + with: + context: ./ + file: docker/Dockerfile + # Load and tag the image so it can be used by the test job below. + load: true + tags: ${{ env.TEST_TAG }} + + # Run tests on the just created image. + - name: Run tests + uses: plexsystems/container-structure-test-action@v0.2.0 + with: + image: ${{ env.TEST_TAG }} + config: docker/container-structure-test.yaml + + - name: Get Version + id: version + run: echo "::set-output name=RELEASE_VERSION::$(cat VERSION)" + - name: Compute target tag + id: target + env: + RELEASE_VERSION: ${{ steps.version.outputs.RELEASE_VERSION }} + run: | + # Translate the Github reference into a tag name. + # + # - `release` tag maps to `zeek:latest` + # - `v*` tag (excluding `v*-dev` tags) maps to `zeek:RELEASE_VERSION` + # - `master` branch maps to `zeek-dev:latest` + # + # Any other refs are not published below. + if [ "${GITHUB_REF}" = "refs/tags/release" ]; then + echo "::set-output name=tag::zeek:latest" + elif [ "${GITHUB_REF}" = "refs/heads/master" ]; then + echo "::set-output name=tag::zeek-dev:latest" + elif [[ "${GITHUB_REF}" = refs/heads/v* ]] && [[ "${GITHUB_REF}" != refs/heads/v*-dev ]]; then + echo "::set-output name=tag::zeek:${RELEASE_VERSION}" + fi + + - name: Login to DockerHub + uses: docker/login-action@v1 + # Secrets for the login are not available for pull requests. + if: github.event_name == 'push' + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + + - name: Push + # Only publish if we did compute a tag. + if: github.event_name == 'push' && steps.target.outputs.tag != '' + uses: docker/build-push-action@v2 + with: + context: ./ + file: docker/Dockerfile + push: true + tags: | + zeekurity/${{ steps.target.outputs.tag}} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000000..8ee9b62c42 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,60 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +# Layer to build Zeek. +FROM debian:buster-slim AS build_zeek + +# Configure system for build. +RUN apt-get -q update \ + && apt-get install -q -y --no-install-recommends \ + bind9 \ + bison \ + cmake \ + flex \ + g++ \ + gcc \ + libmaxminddb-dev \ + libpcap-dev \ + libssl-dev \ + libz-dev \ + make \ + python3-minimal \ + python3-dev \ + swig \ + ninja-build \ + python3-pip \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy over the Zeek source tree. +# NOTE: This assumes that we build in the context of the parent directory (the +# Git checkout root). We e.g., ensure that in the `Makefile` in this directory. +COPY . /src/zeek +RUN make -C /src/zeek distclean + +WORKDIR /src/zeek +RUN ./configure \ + --generator=Ninja \ + --build-type=Release \ + && ninja -C build install + +# Final layer containing all artifacts. +FROM debian:buster-slim AS final + +RUN apt-get -q update \ + && apt-get install -q -y --no-install-recommends \ + ca-certificates \ + git \ + libmaxminddb0 \ + libpython3.7 \ + libpcap0.8 \ + libssl1.1 \ + libz1 \ + python3-minimal \ + python3-git \ + python3-semantic-version \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* + +# Copy over Zeek installation. +COPY --from=build_zeek /usr/local/zeek /usr/local/zeek +ENV PATH "/usr/local/zeek/bin:${PATH}" diff --git a/docker/Makefile b/docker/Makefile new file mode 100644 index 0000000000..0a242c04bd --- /dev/null +++ b/docker/Makefile @@ -0,0 +1,8 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +VERSION := $$(cat ../VERSION) + +.PHONY: all + +all: + @docker build -t zeek:$(VERSION) -f Dockerfile .. diff --git a/docker/README b/docker/README new file mode 100644 index 0000000000..8d3280f161 --- /dev/null +++ b/docker/README @@ -0,0 +1,19 @@ +Container image for Zeek +======================== + +This directory contains a minimal container image for Zeek. This image is +published automatically to [DockerHub](https://hub.docker.com/u/zeekurity) for +releases and for commits on the `master` branch. + +- Images for release are published as + [`zeekurity/zeek`](https://hub.docker.com/r/zeekurity/zeek) with the `latest` + tag pointing to the latest release. +- Development images for the `master` branch are published as + [`zeekurity/zeek-dev`](https://hub.docker.com/r/zeekurity/zeek-dev). + + +To run the image execute e.g.,: + + docker run -it zeekurity/zeek + +To build the image execute `make` from this directory. diff --git a/docker/container-structure-test.yaml b/docker/container-structure-test.yaml new file mode 100644 index 0000000000..455f52358b --- /dev/null +++ b/docker/container-structure-test.yaml @@ -0,0 +1,37 @@ +# See the file "COPYING" in the main distribution directory for copyright. + +# This file contains a test configuration for +# https://github.com/GoogleContainerTools/container-structure-test. + +schemaVersion: 2.0.0 + +commandTests: + - name: zeek can be run + command: zeek + args: ["-v"] + expectedOutput: ["^zeek version .*"] + + - name: is release build + command: zeek-config + args: ["--build_type"] + expectedOutput: ["release"] + + - name: btest can be run + command: btest + args: ["--version"] + expectedOutput: ["\\d\\.\\d"] + + - name: valid zkg config + command: zkg + args: ["config"] + # Just validate that we some some valid entry here. + expectedOutput: ["state_dir = /usr/local/zeek/var/lib/zkg"] + + - name: plugin can be installed + command: zkg + # We pick any plugin with minimal deps here. + args: ["install", "--force", "sethhall/domain-tld"] + expectedOutput: + - "Installing \"zeek/sethhall/domain-tld\"" + - "Installed \"zeek/sethhall/domain-tld\" (.*)" + - "Loaded \"zeek/sethhall/domain-tld\"" From d6c80f6d2c8f8ebc07b1ba2c18a5dfbbf5e4e0a5 Mon Sep 17 00:00:00 2001 From: Christian Kreibich Date: Wed, 22 Sep 2021 16:37:52 -0700 Subject: [PATCH 2/3] Test container with BTest instead of container-structure-tests This patch replaces the container-strucuture-tests for the Zeek container with a BTest-driven approach to provide a more familiar experience. In addtion to `python3` for `btest` we also rely on `make` being available in Github-hosted runners which at least currently seems to be hold (probably pulled in as a dependency of CMake which is officially included, see https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md). --- .github/workflows/docker.yml | 16 +++++--- docker/Makefile | 4 ++ docker/btest/.gitignore | 3 ++ .../Baseline/docker.structure_tests/output | 20 ++++++++++ docker/btest/Makefile | 11 ++++++ docker/btest/btest.cfg | 13 +++++++ docker/btest/docker/structure_tests.sh | 20 ++++++++++ docker/container-structure-test.yaml | 37 ------------------- 8 files changed, 81 insertions(+), 43 deletions(-) create mode 100644 docker/btest/.gitignore create mode 100644 docker/btest/Baseline/docker.structure_tests/output create mode 100644 docker/btest/Makefile create mode 100644 docker/btest/btest.cfg create mode 100644 docker/btest/docker/structure_tests.sh delete mode 100644 docker/container-structure-test.yaml diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 79afc88acd..161e92cec2 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -36,12 +36,8 @@ jobs: load: true tags: ${{ env.TEST_TAG }} - # Run tests on the just created image. - - name: Run tests - uses: plexsystems/container-structure-test-action@v0.2.0 - with: - image: ${{ env.TEST_TAG }} - config: docker/container-structure-test.yaml + - name: Run btests + run: make -C docker/btest - name: Get Version id: version @@ -84,3 +80,11 @@ jobs: push: true tags: | zeekurity/${{ steps.target.outputs.tag}} + + - name: Preserve artifacts + uses: actions/upload-artifact@v2 + if: failure() + with: + name: docker-btest + path: docker/btest/.tmp + if-no-files-found: ignore diff --git a/docker/Makefile b/docker/Makefile index 0a242c04bd..696afc690d 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -6,3 +6,7 @@ VERSION := $$(cat ../VERSION) all: @docker build -t zeek:$(VERSION) -f Dockerfile .. + @docker tag zeek:$(VERSION) zeek:latest + +test: + @TEST_TAG=zeek:$(VERSION) $(MAKE) -C btest diff --git a/docker/btest/.gitignore b/docker/btest/.gitignore new file mode 100644 index 0000000000..5b0019848f --- /dev/null +++ b/docker/btest/.gitignore @@ -0,0 +1,3 @@ +.tmp +.btest.failed.dat +diag.log diff --git a/docker/btest/Baseline/docker.structure_tests/output b/docker/btest/Baseline/docker.structure_tests/output new file mode 100644 index 0000000000..0c8a07377a --- /dev/null +++ b/docker/btest/Baseline/docker.structure_tests/output @@ -0,0 +1,20 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +zeek version xxx +XXX +[sources] +zeek = https://github.com/zeek/packages + +[paths] +state_dir = /usr/local/zeek/var/lib/zkg +script_dir = /usr/local/zeek/share/zeek/site +plugin_dir = /usr/local/zeek/lib/zeek/plugins +bin_dir = /usr/local/zeek/bin +zeek_dist = + +[templates] +default = https://github.com/zeek/package-template + + +Installing "zeek/sethhall/domain-tld" +Installed "zeek/sethhall/domain-tld" (XXX) +Loaded "zeek/sethhall/domain-tld" diff --git a/docker/btest/Makefile b/docker/btest/Makefile new file mode 100644 index 0000000000..65d4d65c98 --- /dev/null +++ b/docker/btest/Makefile @@ -0,0 +1,11 @@ +DIAG=diag.log +BTEST=../../auxil/btest/btest + +all: cleanup btest-verbose + +# Showing all tests. +btest-verbose: + @$(BTEST) -d -j -f $(DIAG) + +cleanup: + @rm -f $(DIAG) diff --git a/docker/btest/btest.cfg b/docker/btest/btest.cfg new file mode 100644 index 0000000000..69b30b9184 --- /dev/null +++ b/docker/btest/btest.cfg @@ -0,0 +1,13 @@ +[btest] +TestDirs = docker +TmpDir = %(testbase)s/.tmp +BaselineDir = %(testbase)s/Baseline +IgnoreDirs = .tmp +IgnoreFiles = *.tmp *.swp #* *.trace .DS_Store +MinVersion = 0.63 + +[environment] +LC_ALL=C +PATH=%(testbase)s/../../auxil/btest:%(default_path)s +TEST_IMAGE=${TEST_TAG:-zeek:latest} +TZ=UTC diff --git a/docker/btest/docker/structure_tests.sh b/docker/btest/docker/structure_tests.sh new file mode 100644 index 0000000000..cb6cd2e6f0 --- /dev/null +++ b/docker/btest/docker/structure_tests.sh @@ -0,0 +1,20 @@ +# @TEST-REQUIRES: docker inspect ${TEST_TAG:-zeek:latest} +# @TEST-EXEC: bash -euxo pipefail %INPUT >output +# @TEST-EXEC: btest-diff output + +TEST_TAG=${TEST_TAG:-zeek:latest} + +# Check that `zeek` can be run. +docker run --rm "${TEST_TAG}" zeek -v | sed 's/\(zeek version\) .*/\1 xxx/' + +# Check that this is a release build. +docker run --rm "${TEST_TAG}" zeek-config --build_type | grep -q 'release' + +# Check that `btest` can be run. +docker run --rm "${TEST_TAG}" btest --version | sed 's/^[0-9].*/XXX/g' + +# Check that the zkg config looks valid. +docker run --rm "${TEST_TAG}" zkg config + +# Check that a plugin can be installed. We pick any plugin with minimal deps here. +docker run --rm "${TEST_TAG}" zkg install --force sethhall/domain-tld | sed 's/(.*)/(XXX)/' diff --git a/docker/container-structure-test.yaml b/docker/container-structure-test.yaml deleted file mode 100644 index 455f52358b..0000000000 --- a/docker/container-structure-test.yaml +++ /dev/null @@ -1,37 +0,0 @@ -# See the file "COPYING" in the main distribution directory for copyright. - -# This file contains a test configuration for -# https://github.com/GoogleContainerTools/container-structure-test. - -schemaVersion: 2.0.0 - -commandTests: - - name: zeek can be run - command: zeek - args: ["-v"] - expectedOutput: ["^zeek version .*"] - - - name: is release build - command: zeek-config - args: ["--build_type"] - expectedOutput: ["release"] - - - name: btest can be run - command: btest - args: ["--version"] - expectedOutput: ["\\d\\.\\d"] - - - name: valid zkg config - command: zkg - args: ["config"] - # Just validate that we some some valid entry here. - expectedOutput: ["state_dir = /usr/local/zeek/var/lib/zkg"] - - - name: plugin can be installed - command: zkg - # We pick any plugin with minimal deps here. - args: ["install", "--force", "sethhall/domain-tld"] - expectedOutput: - - "Installing \"zeek/sethhall/domain-tld\"" - - "Installed \"zeek/sethhall/domain-tld\" (.*)" - - "Loaded \"zeek/sethhall/domain-tld\"" From 2db1ebb2a21f73a62ee829b259afb4f06c0ddbd4 Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Fri, 24 Sep 2021 09:35:10 +0200 Subject: [PATCH 3/3] Switch published container image to debian:bullseye-slim --- docker/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 8ee9b62c42..4ee11c65d4 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # See the file "COPYING" in the main distribution directory for copyright. # Layer to build Zeek. -FROM debian:buster-slim AS build_zeek +FROM debian:bullseye-slim AS build # Configure system for build. RUN apt-get -q update \ @@ -38,7 +38,7 @@ RUN ./configure \ && ninja -C build install # Final layer containing all artifacts. -FROM debian:buster-slim AS final +FROM debian:bullseye-slim AS final RUN apt-get -q update \ && apt-get install -q -y --no-install-recommends \ @@ -56,5 +56,5 @@ RUN apt-get -q update \ && rm -rf /var/lib/apt/lists/* # Copy over Zeek installation. -COPY --from=build_zeek /usr/local/zeek /usr/local/zeek +COPY --from=build /usr/local/zeek /usr/local/zeek ENV PATH "/usr/local/zeek/bin:${PATH}"