Merge branch 'topic/christian/external-testsuite-tweaks'

* topic/christian/external-testsuite-tweaks:
  Add helpers for syncing commit files with external testsuites
  Fix typo in update-timing target for external testsuites

CHANGES

@@ -1,3 +1,11 @@
+5.1.0-dev.133 | 2022-06-30 09:56:20 -0700
+
+  * Add helpers for syncing commit files with external testsuites (Christian Kreibich, Corelight)
+
+  * Fix typo in update-timing target for external testsuites (Christian Kreibich, Corelight)
+
+  * Update broker submodule [nomail] (Tim Wojtulewicz, Corelight)
+
 5.1.0-dev.128 | 2022-06-27 13:04:47 -0700
 
   * removed deprecated capture-by-reference closures (Vern Paxson, Corelight)

VERSION

@@ -1 +1 @@
-5.1.0-dev.130
+5.1.0-dev.133

ci/init-external-repos.sh

@@ -13,7 +13,6 @@ set -e
 cd testing/external
 [[ ! -d zeek-testing ]] && make init
 cd zeek-testing
-git checkout -q $(cat ../commit-hash.zeek-testing)
 
 if [[ -n "${CIRRUS_CI}" ]]; then
     if [[ -d ../zeek-testing-traces ]]; then

testing/external/Makefile

@@ -6,14 +6,14 @@ DIAG=diag.log
 
 all:
 	@rm -f $(DIAG)
-	@for repo in $(REPOS); do (cd $$repo && make -s ); done
+	@for repo in $(REPOS); do ( cd $$repo && make -s ); done
 
 brief:
 	@rm -f $(DIAG)
-	@for repo in $(REPOS); do (cd $$repo && make -s brief ); done
+	@for repo in $(REPOS); do ( cd $$repo && make -s brief ); done
 
 init:
-	git clone $(PUBLIC_REPO)
+	git clone $(PUBLIC_REPO) && ./scripts/sync-repo $$(basename $(PUBLIC_REPO))
 
 pull:
 	@for repo in $(REPOS); do ( cd $$repo && git pull ); done
@@ -25,9 +25,15 @@ status:
 	@for repo in $(REPOS); do ( cd $$repo && echo '>>' $$repo && git status -bs && echo ); done
 
 coverage:
-	@for repo in $(REPOS); do (cd $$repo && echo "Coverage for '$$repo' repo:" && make coverage); done
+	@for repo in $(REPOS); do ( cd $$repo && echo "Coverage for '$$repo' repo:" && make coverage ); done
 
 update-timing:
-	@for repo in $(REPOS); do (cd $$repo && echo "Coverage for '$$repo' repo:" && make update-timing); done
+	@for repo in $(REPOS); do ( cd $$repo && echo "Updating timing for '$$repo' repo:" && make update-timing ); done
+
+sync-commits:
+	for repo in $(REPOS); do ./scripts/sync-commit $$repo; done
+
+sync-repos:
+	for repo in $(REPOS); do ./scripts/sync-repo $$repo; done
 
 .PHONY: all brief init pull push status coverage

testing/external/README

@@ -33,7 +33,6 @@ To later update to upstream changes:
 
 This updates the tests and the traces as necessary.
 
-
 Running Tests
 -------------
 
@@ -53,6 +52,18 @@ test repository:
 All the standard ``btest`` options can be used to run individual
 tests, get diagnostic output, etc.
 
+Versioning
+----------
+
+Since external testsuites live in separate repositories, we need a way to tie
+the local Zeek codebase to a particular version of the testsuites. Normally we'd
+use git submodules, but cloning the testsuites with the rest of the distribution
+isn't always desirable or feasible. We resort to "manual submodules": a file
+"commit-hash.<testsuite-name>" in this directory contains the commit hash to use
+for the respective testsuite. The Makefile target "sync-repos" brings the local
+repositories in line with commit files, while "sync-commits" updates the commit
+files for locally available testsuites to their HEAD commits.
+
 Updating Baseline
 -----------------
 

testing/external/scripts/sync-commit

@@ -0,0 +1,30 @@
+#! /usr/bin/env bash
+#
+# This updates the requested test repo's commit file to the current HEAD commit
+# of that repo. If the repo isn't available, this does nothing. If the commit
+# file doesn't exist yet, this creates it. It doesn't stage or commit the
+# updated commit files.
+[[ -z "$1" ]] && {
+    echo "sync-commit needs a local testsuite repository path as argument"
+    exit 1
+}
+
+repo="$1"
+reponame=$(basename $repo)
+commitfile=commit-hash.$reponame
+
+[[ ! -d $repo || ! -d $repo/.git ]] && exit 0
+
+commit=$(cd $repo && git log -1 --pretty=format:%H)
+
+[[ -f $commitfile && $(cat $commitfile) == $commit ]] && exit 0
+
+if [[ -n $commit ]]; then
+    echo "Pinning '$reponame' to $commit"
+    echo $commit >$commitfile
+
+    # If git knows the commit file, show diff:
+    if git ls-files --error-unmatch $commitfile >/dev/null 2>&1; then
+        git diff $commitfile
+    fi
+fi

testing/external/scripts/sync-repo

@@ -0,0 +1,23 @@
+#! /usr/bin/env bash
+#
+# This moves the requested test repo to the commit indicated in its commit file.
+# If repo or commit file do not exist, it does nothing.
+[[ -z "$1" ]] && {
+    echo "sync-repo needs a local testsuite repository path as argument"
+    exit 1
+}
+
+repo="$1"
+reponame=$(basename $repo)
+commitfile=commit-hash.$reponame
+
+[[ ! -d $repo || ! -f $commitfile ]] && exit 0
+
+commit=$(cat $commitfile)
+
+[[ $commit == $(cd $repo && git rev-parse HEAD) ]] && exit 0
+
+(
+    echo "Moving '$reponame' to $commit"
+    cd $repo && git -c advice.detachedHead=false checkout $commit
+)