Add zeek -V/--build-info

This adds a new utility called ci/collect-repo-info.py to produce a JSON
document that is then baked into the Zeek executable file. Further, when
creating a tarball via `make dist`, put a top-level repo-info.json file
in place that is picked when no .git directory exists.

Closes #1405

CMakeLists.txt

@@ -858,6 +858,22 @@ foreach(plugin_dir ${_build_in_plugins})
     endif ()
 endforeach()
 
+########################################################################
+## Populate the ZEEK_BUILD_INFO for use in src/version.c.in
+execute_process(COMMAND "${PROJECT_SOURCE_DIR}/ci/collect-repo-info.py" "${ZEEK_INCLUDE_PLUGINS}"
+                WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}"
+                OUTPUT_VARIABLE ZEEK_BUILD_INFO
+                RESULT_VARIABLE ZEEK_BUILD_INFO_RESULT
+                OUTPUT_STRIP_TRAILING_WHITESPACE)
+
+if ( NOT ZEEK_BUILD_INFO_RESULT EQUAL "0" )
+    message( FATAL_ERROR "Could not collect repository info")
+endif ()
+
+# string(JSON ... ) requires CMake 3.19, but then we could do something like:
+# string(JSON ZEEK_BUILD_INFO SET "${ZEEK_BUILD_INFO}"
+#        compile_options cxx_flags "${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}")
+
 ########################################################################
 ## Recurse on sub-directories
 

Makefile

@@ -44,6 +44,7 @@ dist:
 	@(cd ../$(VERSION_FULL) && find . -name \.git\* | xargs rm -rf)
 	@(cd ../$(VERSION_FULL) && find . -name \.idea -type d | xargs rm -rf)
 	@(cd ../$(VERSION_FULL) && find . -maxdepth 1 -name build\* | xargs rm -rf)
+	@python3 ./ci/collect-repo-info.py --only-git > ../$(VERSION_FULL)/repo-info.json
 	@mv ../$(VERSION_FULL) .
 	@COPYFILE_DISABLE=true tar -czf $(VERSION_FULL).tar.gz $(VERSION_FULL)
 	@echo Package: $(VERSION_FULL).tar.gz

ci/collect-repo-info.py

@@ -0,0 +1,183 @@
+#!/usr/bin/env python3
+"""
+Collect Git information from the Zeek repository and output a JSON
+document on stdout for inclusion into the executable.
+
+Example usage:
+
+    ./ci/collect-repo-info.py './auxil/spicy-plugin'
+"""
+
+import argparse
+import json
+import logging
+import pathlib
+import os
+import subprocess
+import sys
+
+GIT = "git"
+
+logger = logging.getLogger(__name__)
+
+
+def git(*args):
+    return subprocess.check_output([GIT, *args]).decode("utf-8")
+
+
+def git_is_dirty(d: pathlib.Path):
+    return (len(git("-C", str(d), "status", "--untracked=no", "--short").splitlines()) > 0)
+
+
+def git_generic_info(d: pathlib.Path):
+    """
+    Collect git information from directory d
+    """
+    info = {
+        "commit": git("-C", str(d), "rev-list", "-1", "HEAD").strip(),
+        "dirty": git_is_dirty(d),
+    }
+
+    # git describe fails on Cirrus CI due to no tags being available
+    # in the shallow clone. Instead of using --all, just skip over it.
+    try:
+        info["describe"] = git("-C", str(d), "describe", "--tags").strip()
+    except subprocess.CalledProcessError:
+        if "CIRRUS_CI" not in os.environ:
+            logger.warning("Could not git describe %s", d)
+
+    return info
+
+
+def collect_submodule_info(zeek_dir: pathlib.Path):
+    submodules = []
+    for sm in git("-C", str(zeek_dir), "submodule", "status").splitlines():
+        sm = sm.strip()
+        if sm.count(" ") != 2:
+            logger.error("submodules not updated: %s", sm)
+            sys.exit(1)
+
+        commit, path, describe = sm.split(" ")
+        flag = None
+        if commit[0] in "U+-":
+            flag = commit[0]
+            commit = commit[1:]
+
+        describe = describe.strip("()")
+        sm_info = {
+            "path": path,
+            "commit": commit,
+            "describe": describe,
+            "dirty": git_is_dirty(pathlib.Path(zeek_dir / path)),
+        }
+        if flag:
+            sm_info["flag"] = flag
+
+        try:
+            sm_info["version"] = (zeek_dir / path / "VERSION").read_text().strip()
+        except FileNotFoundError:
+            # The external ones usually don't have a version.
+            pass
+
+        submodules.append(sm_info)
+
+    return submodules
+
+
+def collect_git_info(zeek_dir: pathlib.Path):
+    """
+    Assume we have a git checkout.
+    """
+    info = git_generic_info(zeek_dir)
+    info["name"] = "zeek"
+    info["version"] = (zeek_dir / "VERSION").read_text().strip()
+    info["submodules"] = collect_submodule_info(zeek_dir)
+    info["branch"] = git("-C", str(zeek_dir), "rev-parse", "--abbrev-ref", "HEAD").strip()
+    info["source"] = "git"
+
+    return info
+
+
+def collect_plugin_info(plugin_dir: pathlib.Path):
+    """ """
+    # A plugin's name is not part of it's metadata/information, use
+    # the basename of its directory.
+    result = {
+        "name": plugin_dir.parts[-1],
+    }
+
+    try:
+        result["version"] = (plugin_dir / "VERSION").read_text().strip()
+    except FileNotFoundError:
+        logger.warning("No VERSION found in %s", plugin_dir)
+
+    if (plugin_dir / ".git").exists():
+        result.update(git_generic_info(plugin_dir))
+
+    return result
+
+
+def main():
+    parser = argparse.ArgumentParser()
+
+    def included_plugin_dir_conv(v):
+        for p in [p.strip() for p in v.split(";") if p.strip()]:
+            yield pathlib.Path(p)
+
+    parser.add_argument("included_plugin_dirs",
+                        default="",
+                        nargs="?",
+                        type=included_plugin_dir_conv)
+    parser.add_argument("--dir", default=".")
+    parser.add_argument("--only-git",
+                        action="store_true",
+                        help="Do not try repo-info.json fallback")
+    args = parser.parse_args()
+
+    logging.basicConfig(format="%(levelname)s: %(message)s")
+
+    zeek_dir = pathlib.Path(args.dir).absolute()
+
+    if not (zeek_dir / "zeek-config.h.in").exists():
+        logger.error("%s missing zeek-config.h.in", zeek_dir)
+        return 1
+
+    try:
+        git("--version")
+    except OSError as e:
+        logger.error("No git? (%s)", str(e))
+        return 1
+
+    # Attempt to collect info from git first and alternatively
+    # fall back to a repo-info.json file within what is assumed
+    # to be a tarball.
+    if (zeek_dir / ".git").is_dir():
+        info = collect_git_info(zeek_dir)
+    elif not args.only_git:
+        try:
+            with open(zeek_dir / "repo-info.json") as fp:
+                info = json.load(fp)
+                info["source"] = "repo-info.json"
+        except FileNotFoundError:
+            logger.error("%s is not a git repo and repo-info.json missing", zeek_dir)
+            return 1
+    else:
+        logger.error("Not a git repo and --only-git provided")
+        return 1
+
+    included_plugins_info = []
+    for plugin_dir in args.included_plugin_dirs:
+        if not plugin_dir.is_dir():
+            logger.error("Plugin directory %s does not exist", plugin_dir)
+            return 1
+
+        included_plugins_info.append(collect_plugin_info(plugin_dir))
+
+    info["included_plugins"] = included_plugins_info
+
+    json_str = json.dumps(info, indent=2, sort_keys=True)
+    print(json_str)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/CMakeLists.txt

@@ -30,6 +30,9 @@ set(bro_REGISTER_BIFS CACHE INTERNAL "BIFs for automatic registering" FORCE)
 set(bro_BASE_BIF_SCRIPTS CACHE INTERNAL "Zeek script stubs for BIFs in base distribution of Zeek" FORCE)
 set(bro_PLUGIN_BIF_SCRIPTS CACHE INTERNAL "Zeek script stubs for BIFs in Zeek plugins" FORCE)
 
+# Poor man's JSON escaping as this is rendered into a C string.
+string(REPLACE "\"" "\\\"" ZEEK_BUILD_INFO_ESCAPED "${ZEEK_BUILD_INFO}")
+string(REPLACE "\n" "\\n" ZEEK_BUILD_INFO_ESCAPED "${ZEEK_BUILD_INFO_ESCAPED}")
 configure_file(version.c.in ${CMAKE_CURRENT_BINARY_DIR}/version.c)
 configure_file(util-config.h.in ${CMAKE_CURRENT_BINARY_DIR}/util-config.h)
 

src/Options.cc

@@ -108,6 +108,7 @@ void usage(const char* prog, int code)
 	fprintf(stderr, "       --no-unused-warnings         | suppress warnings of unused "
 	                "functions/hooks/events\n");
 	fprintf(stderr, "    -v|--version                    | print version and exit\n");
+	fprintf(stderr, "    -V|--build-info                 | print build information and exit\n");
 	fprintf(stderr, "    -w|--writefile <writefile>      | write to given tcpdump file\n");
 #ifdef DEBUG
 	fprintf(stderr, "    -B|--debug <dbgstreams>         | Enable debugging output for selected "
@@ -387,6 +388,7 @@ Options parse_cmdline(int argc, char** argv)
 		{"writefile", required_argument, nullptr, 'w'},
 		{"usage-issues", no_argument, nullptr, 'u'},
 		{"version", no_argument, nullptr, 'v'},
+		{"build-info", no_argument, nullptr, 'V'},
 		{"no-checksums", no_argument, nullptr, 'C'},
 		{"force-dns", no_argument, nullptr, 'F'},
 		{"deterministic", no_argument, nullptr, 'D'},
@@ -421,7 +423,7 @@ Options parse_cmdline(int argc, char** argv)
 	};
 
 	char opts[256];
-	util::safe_strncpy(opts, "B:c:E:e:f:G:H:I:i:j::n:O:0:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuv",
+	util::safe_strncpy(opts, "B:c:E:e:f:G:H:I:i:j::n:O:0:o:p:r:s:T:t:U:w:X:CDFMNPQSWabdhmuvV",
 	                   sizeof(opts));
 
 	int op;
@@ -513,6 +515,9 @@ Options parse_cmdline(int argc, char** argv)
 			case 'v':
 				rval.print_version = true;
 				break;
+			case 'V':
+				rval.print_build_info = true;
+				break;
 			case 'w':
 				rval.pcap_output_file = optarg;
 				break;

src/Options.h

@@ -32,6 +32,7 @@ struct Options
 	void filter_supervised_node_options();
 
 	bool print_version = false;
+	bool print_build_info = false;
 	bool print_usage = false;
 	bool print_execution_time = false;
 	bool print_signature_debug_info = false;

src/version.c.in

@@ -11,3 +11,5 @@ const char* ZEEK_VERSION_FUNCTION()
 {
     return "@VERSION_C_IDENT@";
 }
+
+const char zeek_build_info[] = "@ZEEK_BUILD_INFO_ESCAPED@";

src/zeek-setup.cc

@@ -207,6 +207,7 @@ char version[] = VERSION;
 #else
 extern char version[];
 #endif
+extern const char zeek_build_info[];
 
 const char* zeek::detail::command_line_policy = nullptr;
 vector<string> zeek::detail::params;
@@ -535,6 +536,12 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 		exit(0);
 		}
 
+	if ( options.print_build_info )
+		{
+		fprintf(stdout, "%s", zeek_build_info);
+		exit(0);
+		}
+
 	if ( options.run_unit_tests )
 		options.deterministic_mode = true;
 

testing/btest/misc/build-info.sh

@@ -0,0 +1,5 @@
+# @TEST-DOC: Verify -V and --build-info work
+# @TEST-EXEC: zeek -V  | python3 -m json.tool > V.json
+# @TEST-EXEC: zeek --build-info | python3 -m json.tool > build-info.json
+# @TEST-EXEC: diff V.json build-info.json
+# @TEST-EXEC: grep -q '"commit"' V.json