Merge remote-tracking branch 'origin/topic/awelzel/2837-builtin-plugin-ci-take-two'

* origin/topic/awelzel/2837-builtin-plugin-ci-take-two:
  cirrus: Add smoke testing for builtin plugins
  ci/collect-repo-info: Make plugin VERSION reading more robust
  configure: Quote --include-plugins argument
  Fix --no-install-recommends typo

.cirrus.yml

@@ -661,7 +661,7 @@ zeekctl_debian11_task:
       reupload_on_changes: true
   install_iproute2_script:
     # No iproute2 in default Zeek build container, but zeekctl tests need it.
-    - apt-get update && apt-get install -y --on-install-recommends iproute2
+    - apt-get update && apt-get install -y --no-install-recommends iproute2
   build_script:
     - cd auxil/zeekctl/testing && ./Scripts/build-zeek
   test_script:
@@ -669,3 +669,26 @@ zeekctl_debian11_task:
   on_failure:
     upload_zeekctl_testing_artifacts:
       path: "auxil/zeekctl/testing/.tmp/**"
+
+# Test building Zeek with builtin plugins available in
+# testing/builtin-plugins/Files/
+include_plugins_debian11_task:
+  cpu: *CPUS
+  memory: *MEMORY
+  container:
+    # Debian 11 EOL: June 2026
+    dockerfile: ci/debian-11/Dockerfile
+    << : *RESOURCES_TEMPLATE
+  sync_submodules_script: git submodule update --recursive --init
+  always:
+    ccache_cache:
+      folder: /tmp/ccache
+      fingerprint_script: echo builtin-plugins-ccache-$ZEEK_CCACHE_EPOCH-$CIRRUS_TASK_NAME-$CIRRUS_OS
+      reupload_on_changes: true
+  build_script: ZEEK_CI_CONFIGURE_FLAGS="${ZEEK_CI_CONFIGURE_FLAGS} --include-plugins='/zeek/testing/builtin-plugins/Files/protocol-plugin;/zeek/testing/builtin-plugins/Files/py-lib-plugin'" ./ci/build.sh
+  test_script:
+    - cd testing/builtin-plugins && ../../auxil/btest/btest -d -b -j ${ZEEK_CI_BTEST_JOBS}
+  on_failure:
+    upload_include_plugins_testing_artifacts:
+      path: "testing/builtin-plugins/.tmp/**"
+  << : *BUILDS_ONLY_IF_TEMPLATE

CHANGES

@@ -1,3 +1,11 @@
+6.0.0-dev.185 | 2023-03-13 09:38:34 +0100
+
+  * GH-2837: cirrus: Add smoke testing for builtin plugins (Arne Welzel, Corelight)
+
+  * ci/collect-repo-info: Make plugin VERSION reading more robust (Arne Welzel, Corelight)
+
+  * configure: Quote --include-plugins argument (Arne Welzel, Corelight)
+
 6.0.0-dev.179 | 2023-03-12 13:52:15 -0700
 
   * Force rebuild of CentOS 7 CI image to pick up git install (Tim Wojtulewicz, Corelight)

VERSION

@@ -1 +1 @@
-6.0.0-dev.179
+6.0.0-dev.185

ci/collect-repo-info.py

@@ -116,6 +116,19 @@ def collect_git_info(zeek_dir: pathlib.Path):
     return info
 
 
+def read_plugin_version(plugin_dir: pathlib.Path):
+    """
+    Open the VERSION file, look for the first non empty
+    non comment line and return it.
+    """
+    with (plugin_dir / "VERSION").open() as f:
+        for line in f:
+            line = line.strip()
+            if line and not line.startswith("#"):
+                return line
+    return ""
+
+
 def collect_plugin_info(plugin_dir: pathlib.Path):
     """ """
     # A plugin's name is not part of it's metadata/information, use
@@ -125,7 +138,7 @@ def collect_plugin_info(plugin_dir: pathlib.Path):
     }
 
     try:
-        result["version"] = (plugin_dir / "VERSION").read_text().strip()
+        result["version"] = read_plugin_version(plugin_dir)
     except FileNotFoundError:
         logger.warning("No VERSION found in %s", plugin_dir)
 

configure

@@ -230,7 +230,7 @@ while [ $# -ne 0 ]; do
             append_cache_entry CMAKE_TOOLCHAIN_FILE PATH $optarg
             ;;
         --include-plugins=*)
-            append_cache_entry ZEEK_INCLUDE_PLUGINS STRING $optarg
+            append_cache_entry ZEEK_INCLUDE_PLUGINS STRING \"$optarg\"
             ;;
         --prefix=*)
             prefix=$optarg

testing/builtin-plugins/.gitignore

@@ -0,0 +1,4 @@
+.tmp
+.btest.failed.dat
+diag.log
+coverage.log

testing/builtin-plugins/Baseline/tests.plugins-exist/out

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Zeek::PyLib - Plugin embedding Python, whoosh. (built-in)
+Demo::Foo - A Foo test analyzer (built-in)

testing/builtin-plugins/Baseline/tests.python/out

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Python 3
+<module 'sys' (built-in)>

testing/builtin-plugins/Files/protocol-plugin/CMakeLists.txt

@@ -0,0 +1,19 @@
+
+project(Zeek-Plugin-Demo-Foo)
+
+cmake_minimum_required(VERSION 3.5)
+
+if ( NOT ZEEK_DIST )
+    message(FATAL_ERROR "ZEEK_DIST not set")
+endif ()
+
+set(CMAKE_MODULE_PATH ${ZEEK_DIST}/cmake)
+
+include(ZeekPlugin)
+
+zeek_plugin_begin(Demo Foo)
+zeek_plugin_cc(src/Plugin.cc)
+zeek_plugin_cc(src/Foo.cc)
+zeek_plugin_bif(src/events.bif)
+zeek_plugin_pac(src/foo.pac src/foo-protocol.pac src/foo-analyzer.pac)
+zeek_plugin_end()

testing/builtin-plugins/Files/protocol-plugin/VERSION

@@ -0,0 +1,5 @@
+# Some people put comments here
+
+# which isn't really recommended...
+
+0.1.0

testing/builtin-plugins/Files/protocol-plugin/scripts/Demo/Foo/base/main.zeek

@@ -0,0 +1,7 @@
+
+const ports = { 4242/tcp };
+
+event zeek_init() &priority=5
+	{
+	Analyzer::register_for_ports(Analyzer::ANALYZER_FOO, ports);
+	}

testing/builtin-plugins/Files/protocol-plugin/scripts/__load__.zeek

@@ -0,0 +1 @@
+@load Demo/Foo/base/main

testing/builtin-plugins/Files/protocol-plugin/src/Foo.cc

@@ -0,0 +1,56 @@
+
+#include "Foo.h"
+
+#include "zeek/analyzer/protocol/tcp/TCP_Reassembler.h"
+
+#include "events.bif.h"
+#include "foo_pac.h"
+
+using namespace btest::plugin::Demo_Foo;
+
+Foo::Foo(zeek::Connection* conn) : zeek::analyzer::tcp::TCP_ApplicationAnalyzer("Foo", conn)
+	{
+	interp = new binpac::Foo::Foo_Conn(this);
+	}
+
+Foo::~Foo()
+	{
+	delete interp;
+	}
+
+void Foo::Done()
+	{
+	zeek::analyzer::tcp::TCP_ApplicationAnalyzer::Done();
+
+	interp->FlowEOF(true);
+	interp->FlowEOF(false);
+	}
+
+void Foo::EndpointEOF(bool is_orig)
+	{
+	zeek::analyzer::tcp::TCP_ApplicationAnalyzer::EndpointEOF(is_orig);
+	interp->FlowEOF(is_orig);
+	}
+
+void Foo::DeliverStream(int len, const u_char* data, bool orig)
+	{
+	zeek::analyzer::tcp::TCP_ApplicationAnalyzer::DeliverStream(len, data, orig);
+
+	if ( TCP() && TCP()->IsPartial() )
+		return;
+
+	try
+		{
+		interp->NewData(orig, data, data + len);
+		}
+	catch ( const binpac::Exception& e )
+		{
+		AnalyzerViolation(zeek::util::fmt("Binpac exception: %s", e.c_msg()));
+		}
+	}
+
+void Foo::Undelivered(uint64_t seq, int len, bool orig)
+	{
+	zeek::analyzer::tcp::TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	interp->NewGap(orig, len);
+	}

testing/builtin-plugins/Files/protocol-plugin/src/Foo.h

@@ -0,0 +1,35 @@
+
+#pragma once
+
+#include "analyzer/protocol/pia/PIA.h"
+#include "analyzer/protocol/tcp/TCP.h"
+
+namespace binpac
+	{
+namespace Foo
+	{
+class Foo_Conn;
+	}
+	}
+
+namespace btest::plugin::Demo_Foo
+	{
+
+class Foo : public zeek::analyzer::tcp::TCP_ApplicationAnalyzer
+	{
+public:
+	Foo(zeek::Connection* conn);
+	~Foo();
+
+	virtual void Done();
+	virtual void DeliverStream(int len, const u_char* data, bool orig);
+	virtual void Undelivered(uint64_t seq, int len, bool orig);
+	virtual void EndpointEOF(bool is_orig);
+
+	static zeek::analyzer::Analyzer* Instantiate(zeek::Connection* conn) { return new Foo(conn); }
+
+protected:
+	binpac::Foo::Foo_Conn* interp;
+	};
+
+	}

testing/builtin-plugins/Files/protocol-plugin/src/Plugin.cc

@@ -0,0 +1,36 @@
+
+#include "Plugin.h"
+
+#include "Foo.h"
+#include "analyzer/Component.h"
+#include "analyzer/Manager.h"
+
+namespace btest::plugin::Demo_Foo
+	{
+Plugin plugin;
+	}
+
+using namespace btest::plugin::Demo_Foo;
+
+zeek::plugin::Configuration Plugin::Configure()
+	{
+	AddComponent(
+		new zeek::analyzer::Component("Foo", btest::plugin::Demo_Foo::Foo::Instantiate, 1));
+
+	zeek::plugin::Configuration config;
+	config.name = "Demo::Foo";
+	config.description = "A Foo test analyzer";
+	config.version.major = 1;
+	config.version.minor = 0;
+	config.version.patch = 0;
+	return config;
+	}
+
+void Plugin::InitPostScript()
+	{
+	auto tag = ::zeek::analyzer_mgr->GetAnalyzerTag("Foo");
+	if ( ! tag )
+		::zeek::reporter->FatalError("cannot get analyzer Tag");
+
+	zeek::analyzer_mgr->RegisterAnalyzerForPort(tag, TransportProto::TRANSPORT_TCP, 4243);
+	}

testing/builtin-plugins/Files/protocol-plugin/src/Plugin.h

@@ -0,0 +1,20 @@
+
+#pragma once
+
+#include <zeek/plugin/Plugin.h>
+
+namespace btest::plugin::Demo_Foo
+	{
+
+class Plugin : public zeek::plugin::Plugin
+	{
+protected:
+	// Overridden from zeek::plugin::Plugin.
+	zeek::plugin::Configuration Configure() override;
+
+	void InitPostScript() override;
+	};
+
+extern Plugin plugin;
+
+	}

testing/builtin-plugins/Files/protocol-plugin/src/events.bif

@@ -0,0 +1,2 @@
+
+event foo_message%(c: connection, data: string%);

testing/builtin-plugins/Files/protocol-plugin/src/foo-analyzer.pac

@@ -0,0 +1,15 @@
+
+refine connection Foo_Conn += {
+
+	function Foo_data(msg: Foo_Message): bool
+		%{
+		auto data = zeek::make_intrusive<zeek::StringVal>(${msg.data}.length(), (const char*) ${msg.data}.data());
+		zeek::BifEvent::enqueue_foo_message(bro_analyzer(), bro_analyzer()->Conn(), std::move(data));
+		return true;
+		%}
+
+};
+
+refine typeattr Foo_Message += &let {
+	proc: bool = $context.connection.Foo_data(this);
+};

testing/builtin-plugins/Files/protocol-plugin/src/foo-protocol.pac

@@ -0,0 +1,4 @@
+
+type Foo_Message(is_orig: bool) = record {
+	data: bytestring &restofdata;
+};

testing/builtin-plugins/Files/protocol-plugin/src/foo.pac

@@ -0,0 +1,26 @@
+%include binpac.pac
+%include zeek.pac
+
+%extern{
+#include "Foo.h"
+
+#include "events.bif.h"
+%}
+
+analyzer Foo withcontext {
+    connection: Foo_Conn;
+    flow:       Foo_Flow;
+};
+
+connection Foo_Conn(bro_analyzer: ZeekAnalyzer) {
+    upflow   = Foo_Flow(true);
+    downflow = Foo_Flow(false);
+};
+
+%include foo-protocol.pac
+
+flow Foo_Flow(is_orig: bool) {
+	datagram = Foo_Message(is_orig) withcontext(connection, this);
+};
+
+%include foo-analyzer.pac

testing/builtin-plugins/Files/py-lib-plugin/CMakeLists.txt

@@ -0,0 +1,34 @@
+cmake_minimum_required(VERSION 3.5 FATAL_ERROR)
+
+find_package(PythonLibs REQUIRED)
+
+file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
+
+string(REGEX REPLACE "[.-]" " " version_numbers ${VERSION})
+separate_arguments(version_numbers)
+list(GET version_numbers 0 VERSION_MAJOR)
+list(GET version_numbers 1 VERSION_MINOR)
+list(GET version_numbers 2 VERSION_PATCH)
+
+# Our plugin source and scripts are in a subdirectory
+set(BRO_PLUGIN_BASE "${CMAKE_CURRENT_SOURCE_DIR}/plugin")
+
+include(ZeekPlugin)
+
+include_directories(BEFORE ${PYTHON_INCLUDE_DIR})
+
+zeek_plugin_begin(Zeek PyLib)
+
+file(GLOB cc_files RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "plugin/src/*.cc")
+foreach(file ${cc_files})
+    zeek_plugin_cc(${file})
+endforeach ()
+
+file(GLOB bif_files RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}" "plugin/src/*.bif")
+foreach(file ${bif_files})
+    zeek_plugin_bif(${file})
+endforeach ()
+
+zeek_plugin_dist_files(README VERSION)
+zeek_plugin_link_library(${PYTHON_LIBRARY})
+zeek_plugin_end()

testing/builtin-plugins/Files/py-lib-plugin/README

@@ -0,0 +1,8 @@
+PyLib
+=====
+
+Embedding Python with a plugin.
+
+    zeek -e 'Python::run_simple_string("import this")'
+
+Mostly to test a link dependency of a builtin plugin.

testing/builtin-plugins/Files/py-lib-plugin/VERSION

@@ -0,0 +1 @@
+0.0.1

testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.cc

@@ -0,0 +1,34 @@
+#include "Plugin.h"
+
+extern "C"
+	{
+#include <Python.h>
+	}
+
+namespace zeek::plugin::Zeek_PyLib
+	{
+Plugin plugin;
+	}
+
+using namespace zeek::plugin::Zeek_PyLib;
+
+zeek::plugin::Configuration Plugin::Configure()
+	{
+	zeek::plugin::Configuration config;
+	config.name = "Zeek::PyLib";
+	config.description = "Plugin embedding Python, whoosh.";
+	config.version.major = 0;
+	config.version.minor = 0;
+	config.version.patch = 1;
+	return config;
+	}
+
+void Plugin::InitPostScript()
+	{
+	Py_Initialize();
+	}
+
+void Plugin::Done()
+	{
+	Py_FinalizeEx();
+	}

testing/builtin-plugins/Files/py-lib-plugin/plugin/src/Plugin.h

@@ -0,0 +1,21 @@
+#pragma once
+
+#include <zeek/plugin/Plugin.h>
+
+namespace zeek::plugin
+	{
+namespace Zeek_PyLib
+	{
+
+class Plugin : public zeek::plugin::Plugin
+	{
+protected:
+	zeek::plugin::Configuration Configure() override;
+	void InitPostScript() override;
+	void Done() override;
+	};
+
+extern Plugin plugin;
+
+	}
+	}

testing/builtin-plugins/Files/py-lib-plugin/plugin/src/pylib.bif

@@ -0,0 +1,18 @@
+module Python;
+
+%%{
+extern "C" {
+#include <Python.h>
+}
+%%}
+
+function version%(%): string
+%{
+	return zeek::make_intrusive<zeek::StringVal>(Py_GetVersion());
+%}
+
+function run_simple_string%(s: string%): bool
+%{
+	PyRun_SimpleString(s->CheckString());
+	return zeek::val_mgr->True();
+%}

testing/builtin-plugins/README

@@ -0,0 +1 @@
+Smoke testing for builtin plugins from external sources.

testing/builtin-plugins/btest.cfg

@@ -0,0 +1,41 @@
+[DEFAULT]
+# This should be a path relative to the main repository directory, and
+# can be overridden with btest's -s command-line argument.
+build_dir = build
+
+[btest]
+TestDirs    = tests
+TmpDir      = %(testbase)s/.tmp
+BaselineDir = %(testbase)s/Baseline
+IgnoreDirs  = .svn CVS .tmp
+IgnoreFiles = *.tmp *.swp .clang-format #* *.trace .DS_Store
+MinVersion  = 0.63
+
+[environment]
+ZEEKPATH=`bash -c %(testbase)s/../../%(build_dir)s/zeek-path-dev`
+ZEEK_SEED_FILE=%(testbase)s/../btest/random.seed
+ZEEK_PLUGIN_PATH=
+TZ=UTC
+LC_ALL=C
+BTEST_PATH=%(testbase)s/../../auxil/btest
+PATH=%(testbase)s/../../%(build_dir)s/src%(pathsep)s%(testbase)s/../scripts%(pathsep)s%(testbase)s/../../auxil/btest%(pathsep)s%(testbase)s/../../%(build_dir)s/auxil/zeek-aux/zeek-cut%(pathsep)s%(testbase)s/../../auxil/btest/sphinx%(pathsep)s%(default_path)s%(pathsep)s/sbin
+TRACES=%(testbase)s/../btest/Traces
+FILES=%(testbase)s/Files
+SCRIPTS=%(testbase)s/../scripts
+DOC_ROOT=%(testbase)s/../../doc
+DIST=%(testbase)s/../..
+BUILD=%(testbase)s/../../%(build_dir)s
+TEST_DIFF_CANONIFIER=%(testbase)s/../scripts/diff-canonifier
+TMPDIR=%(testbase)s/.tmp
+ZEEK_PROFILER_FILE=%(testbase)s/.tmp/script-coverage/XXXXXX
+BTEST_RST_FILTER=$SCRIPTS/rst-filter
+ZEEK_DNS_FAKE=1
+ZEEK_DEFAULT_LISTEN_ADDRESS=127.0.0.1
+ZEEK_DEFAULT_LISTEN_RETRY=1
+ZEEK_DEFAULT_CONNECT_RETRY=1
+ZEEK_DISABLE_ZEEKYGEN=1
+ZEEK_ALLOW_INIT_ERRORS=1
+ZEEK_SUPERVISOR_NO_SIGKILL=1
+UBSAN_OPTIONS=print_stacktrace=1
+SPICY_PATH=`bash -c %(testbase)s/../../%(build_dir)s/spicy-path`
+HILTI_CXX_INCLUDE_DIRS=`bash -c %(testbase)s/../../%(build_dir)s/hilti-cxx-include-dirs`

testing/builtin-plugins/tests/plugins-exist.zeek

@@ -0,0 +1,5 @@
+# @TEST-DOC: Assumes the plugins within Files/ have been builtin
+
+# @TEST-EXEC: zeek -N Zeek::PyLib >>out
+# @TEST-EXEC: zeek -N Demo::Foo >>out
+# @TEST-EXEC: btest-diff out

testing/builtin-plugins/tests/python.zeek

@@ -0,0 +1,9 @@
+# @TEST-DOC: Run a bit of Python for the kicks, assume there'll be no Python 4 in the near future.
+# @TEST-EXEC: zeek %INPUT >out
+# @TEST-EXEC: btest-diff out
+
+event zeek_init()
+	{
+	Python::run_simple_string("import sys; print(\"Python\", sys.version_info.major)");
+	Python::run_simple_string("print(sys)");
+	}

testing/builtin-plugins/tests/zeek-ldd.zeek

@@ -0,0 +1,4 @@
+# @TEST-DOC: Run ldd on the zeek executable and check that it's linked against libpython
+
+# @TEST-EXEC: ldd $(which zeek) > ldd.out
+# @TEST-EXEC: grep libpython ldd.out