Merge branch 'topic/christian/doctest-in-packages'

* topic/christian/doctest-in-packages:
  Update cmake and auxil/zeek-aux submodules
  Remove the --enable-cpp-tests configure flag in Cirrus CI
  Add testcases for plugin with doctest usage
  Flip unit tests to being enabled by default
  Support for unit tests in plugins
  Alphabetically sort configure's enable/disable/with options

.cirrus.yml

@@ -10,11 +10,11 @@ btest_jobs: &BTEST_JOBS 4
 btest_retries: &BTEST_RETRIES 2
 memory: &MEMORY 4GB
 
-config: &CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
+config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
-static_config: &STATIC_CONFIG --build-type=release --enable-cpp-tests --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install
+static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install
-sanitizer_config: &SANITIZER_CONFIG --build-type=debug --enable-cpp-tests --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers --enable-coverage
+sanitizer_config: &SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address,undefined --enable-fuzzers --enable-coverage
-mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-cpp-tests --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
+mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
-openssl30_config: &OPENSSL30_CONFIG --build-type=release --enable-cpp-tests  --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install
+openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install
 
 resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS

CHANGES

@@ -1,3 +1,11 @@
+4.2.0-dev.417 | 2021-12-06 17:00:16 -0800
+
+  * Flip C++ unit tests to being enabled by default (Christian Kreibich, Corelight)
+
+    To disable them, configure with --disable-cpp-tests.
+
+  * Support for unit tests in plugins (Christian Kreibich, Corelight)
+
 4.2.0-dev.410 | 2021-12-06 11:29:32 -0700
 
   * Remove separate Tag types, note breaking change in NEWS (Tim Wojtulewicz, Corelight)

NEWS

@@ -113,6 +113,11 @@ Changed Functionality
 - The existing tunnel analyzers for AYIYA, Geneve, GTPv1, Teredo, and VXLAN
   are now packet analyzers.
 
+- C++ unit tests are now compiled in by default and can be disabled by
+  configuring the build with --disable-cpp-tests. We removed the former
+  --enable-cpp-tests configure flag. Unit tests now also work in (static and
+  dynamic) Zeek plugins.
+
 Removed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-4.2.0-dev.410
+4.2.0-dev.417

auxil/zeek-aux

@@ -1 +1 @@
-Subproject commit 9100b9d524dddfade02f1b4fceb54265a113b68c
+Subproject commit 614fb53023e97f71816beacba401f085d5dbdd6d

cmake

@@ -1 +1 @@
-Subproject commit 7ba5d4cb8115496ce88e6ec71a74ca3927a0b7fd
+Subproject commit 88cf658d059772b3df6dcd4a91c1a4c4afc113d9

configure

@@ -54,51 +54,51 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
                            install --home [PATH/lib/python]
 
   Optional Features:
-    --enable-debug         compile in debugging mode (like --build-type=Debug)
     --enable-coverage      compile with code coverage support (implies debugging mode)
+    --enable-debug         compile in debugging mode (like --build-type=Debug)
     --enable-fuzzers       build fuzzer targets
+    --enable-jemalloc      link against jemalloc
     --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
     --enable-perftools     enable use of Google perftools (use tcmalloc)
     --enable-perftools-debug use Google's perftools for debugging
-    --enable-jemalloc      link against jemalloc
-    --enable-static-broker build Broker statically (ignored if --with-broker is specified)
     --enable-static-binpac build binpac statically (ignored if --with-binpac is specified)
-    --enable-cpp-tests     build Zeek's C++ unit tests
+    --enable-static-broker build Broker statically (ignored if --with-broker is specified)
     --enable-zeek-client   install the Zeek cluster management client (experimental)
-    --disable-zeekctl      don't install ZeekControl
-    --disable-auxtools     don't build or install auxiliary tools
     --disable-archiver     don't build or install zeek-archiver tool
+    --disable-auxtools     don't build or install auxiliary tools
+    --disable-broker-tests don't try to build Broker unit tests
     --disable-btest        don't install BTest
     --disable-btest-pcaps  don't install Zeek's BTest input pcaps
+    --disable-cpp-tests    don't build Zeek's C++ unit tests
     --disable-python       don't try to build python bindings for Broker
-    --disable-broker-tests don't try to build Broker unit tests
+    --disable-zeekctl      don't install ZeekControl
     --disable-zkg          don't install zkg
 
   Required Packages in Non-Standard Locations:
-    --with-openssl=PATH    path to OpenSSL install root
-    --with-bind=PATH       path to BIND install root
-    --with-pcap=PATH       path to libpcap install root
-    --with-binpac=PATH     path to BinPAC executable
-                           (useful for cross-compiling)
     --with-bifcl=PATH      path to Zeek BIF compiler executable
                            (useful for cross-compiling)
-    --with-flex=PATH       path to flex executable
+    --with-bind=PATH       path to BIND install root
+    --with-binpac=PATH     path to BinPAC executable
+                           (useful for cross-compiling)
     --with-bison=PATH      path to bison executable
-    --with-python=PATH     path to Python executable
     --with-broker=PATH     path to Broker install root
                            (Zeek uses an embedded version by default)
     --with-caf=PATH        path to C++ Actor Framework install root
                            (a Broker dependency that is embedded by default)
+    --with-flex=PATH       path to flex executable
     --with-libkqueue=PATH  path to libkqueue install root
                            (Zeek uses an embedded version by default)
+    --with-openssl=PATH    path to OpenSSL install root
+    --with-pcap=PATH       path to libpcap install root
+    --with-python=PATH     path to Python executable
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libmaxminddb install root
+    --with-jemalloc=PATH   path to jemalloc install root
     --with-krb5=PATH       path to krb5 install root
     --with-perftools=PATH  path to Google Perftools install root
-    --with-jemalloc=PATH   path to jemalloc install root
-    --with-python-lib=PATH path to libpython
     --with-python-inc=PATH path to Python headers
+    --with-python-lib=PATH path to libpython
     --with-swig=PATH       path to SWIG executable
 
   Packaging Options (for developers):
@@ -163,6 +163,7 @@ append_cache_entry ZEEK_ETC_INSTALL_DIR PATH $prefix/etc
 append_cache_entry ENABLE_DEBUG BOOL false
 append_cache_entry ENABLE_PERFTOOLS BOOL false
 append_cache_entry ENABLE_JEMALLOC BOOL false
+append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL true
 append_cache_entry BUILD_SHARED_LIBS BOOL true
 append_cache_entry INSTALL_AUX_TOOLS BOOL true
 append_cache_entry INSTALL_BTEST BOOL true
@@ -260,11 +261,14 @@ while [ $# -ne 0 ]; do
             append_cache_entry ENABLE_COVERAGE BOOL true
             append_cache_entry ENABLE_DEBUG BOOL true
             ;;
+        --enable-debug)
+            append_cache_entry ENABLE_DEBUG BOOL true
+            ;;
         --enable-fuzzers)
             append_cache_entry ZEEK_ENABLE_FUZZERS BOOL true
             ;;
-        --enable-debug)
+        --enable-jemalloc)
-            append_cache_entry ENABLE_DEBUG BOOL true
+            append_cache_entry ENABLE_JEMALLOC BOOL true
             ;;
         --enable-mobile-ipv6)
             has_enable_mobile_ipv6=1
@@ -276,32 +280,24 @@ while [ $# -ne 0 ]; do
             append_cache_entry ENABLE_PERFTOOLS BOOL true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL true
             ;;
-        --sanitizers=*)
+        --enable-static-binpac)
-            append_cache_entry ZEEK_SANITIZERS STRING $optarg
+            append_cache_entry BUILD_STATIC_BINPAC BOOL true
-            ;;
-        --enable-jemalloc)
-            append_cache_entry ENABLE_JEMALLOC BOOL true
             ;;
         --enable-static-broker)
             append_cache_entry BUILD_STATIC_BROKER BOOL true
             ;;
-        --enable-static-binpac)
-            append_cache_entry BUILD_STATIC_BINPAC BOOL true
-            ;;
-        --enable-cpp-tests)
-            append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL true
-            ;;
         --enable-zeek-client)
             append_cache_entry INSTALL_ZEEK_CLIENT BOOL true
             ;;
-        --disable-zeekctl)
+        --disable-archiver)
-            append_cache_entry INSTALL_ZEEKCTL BOOL false
+            append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL false
             ;;
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS BOOL false
             ;;
-        --disable-archiver)
+        --disable-broker-tests)
-            append_cache_entry INSTALL_ZEEK_ARCHIVER BOOL false
+            append_cache_entry BROKER_DISABLE_TESTS BOOL true
+            append_cache_entry BROKER_DISABLE_DOC_EXAMPLES BOOL true
             ;;
         --disable-btest)
             append_cache_entry INSTALL_BTEST BOOL false
@@ -309,72 +305,74 @@ while [ $# -ne 0 ]; do
         --disable-btest-pcaps)
             append_cache_entry INSTALL_BTEST_PCAPS BOOL false
             ;;
+        --disable-cpp-tests)
+            append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL false
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true
             ;;
-        --disable-broker-tests)
-            append_cache_entry BROKER_DISABLE_TESTS BOOL true
-            append_cache_entry BROKER_DISABLE_DOC_EXAMPLES BOOL true
-            ;;
         --disable-zkg)
             append_cache_entry INSTALL_ZKG BOOL false
             ;;
-        --with-openssl=*)
+        --with-bifcl=*)
-            append_cache_entry OPENSSL_ROOT_DIR PATH $optarg
+            append_cache_entry BIFCL_EXE_PATH PATH $optarg
             ;;
         --with-bind=*)
             append_cache_entry BIND_ROOT_DIR PATH $optarg
             ;;
-        --with-pcap=*)
-            append_cache_entry PCAP_ROOT_DIR PATH $optarg
-            ;;
         --with-binpac=*)
             append_cache_entry BINPAC_EXE_PATH PATH $optarg
             ;;
-        --with-bifcl=*)
-            append_cache_entry BIFCL_EXE_PATH PATH $optarg
-            ;;
-        --with-flex=*)
-            append_cache_entry FLEX_EXECUTABLE PATH $optarg
-            ;;
         --with-bison=*)
             append_cache_entry BISON_EXECUTABLE PATH $optarg
             ;;
-        --with-geoip=*)
-            append_cache_entry LibMMDB_ROOT_DIR PATH $optarg
-            ;;
-        --with-krb5=*)
-            append_cache_entry LibKrb5_ROOT_DIR PATH $optarg
-            ;;
-        --with-perftools=*)
-            append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg
-            ;;
-        --with-jemalloc=*)
-            append_cache_entry JEMALLOC_ROOT_DIR PATH $optarg
-            append_cache_entry ENABLE_JEMALLOC BOOL true
-            ;;
-        --with-python=*)
-            append_cache_entry PYTHON_EXECUTABLE PATH $optarg
-            ;;
-        --with-python-lib=*)
-            append_cache_entry PYTHON_LIBRARY PATH $optarg
-            ;;
-        --with-python-inc=*)
-            append_cache_entry PYTHON_INCLUDE_DIR PATH $optarg
-            append_cache_entry PYTHON_INCLUDE_PATH PATH $optarg
-            ;;
-        --with-swig=*)
-            append_cache_entry SWIG_EXECUTABLE PATH $optarg
-            ;;
         --with-broker=*)
             append_cache_entry BROKER_ROOT_DIR PATH $optarg
             ;;
         --with-caf=*)
             append_cache_entry CAF_ROOT PATH $optarg
             ;;
+        --with-flex=*)
+            append_cache_entry FLEX_EXECUTABLE PATH $optarg
+            ;;
+        --with-geoip=*)
+            append_cache_entry LibMMDB_ROOT_DIR PATH $optarg
+            ;;
+        --with-jemalloc=*)
+            append_cache_entry JEMALLOC_ROOT_DIR PATH $optarg
+            append_cache_entry ENABLE_JEMALLOC BOOL true
+            ;;
+        --with-krb5=*)
+            append_cache_entry LibKrb5_ROOT_DIR PATH $optarg
+            ;;
         --with-libkqueue=*)
             append_cache_entry LIBKQUEUE_ROOT_DIR PATH $optarg
             ;;
+        --with-pcap=*)
+            append_cache_entry PCAP_ROOT_DIR PATH $optarg
+            ;;
+        --with-perftools=*)
+            append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg
+            ;;
+        --with-openssl=*)
+            append_cache_entry OPENSSL_ROOT_DIR PATH $optarg
+            ;;
+        --with-python=*)
+            append_cache_entry PYTHON_EXECUTABLE PATH $optarg
+            ;;
+        --with-python-inc=*)
+            append_cache_entry PYTHON_INCLUDE_DIR PATH $optarg
+            append_cache_entry PYTHON_INCLUDE_PATH PATH $optarg
+            ;;
+        --with-python-lib=*)
+            append_cache_entry PYTHON_LIBRARY PATH $optarg
+            ;;
+        --with-swig=*)
+            append_cache_entry SWIG_EXECUTABLE PATH $optarg
+            ;;
+        --sanitizers=*)
+            append_cache_entry ZEEK_SANITIZERS STRING $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

src/CMakeLists.txt

@@ -629,6 +629,7 @@ install(FILES
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/patricia.h
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/setsignal.h
         ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/sqlite3.h
+        ${CMAKE_CURRENT_SOURCE_DIR}/3rdparty/doctest.h
         DESTINATION include/zeek/3rdparty
 )
 

src/Options.cc

@@ -143,8 +143,9 @@ void usage(const char* prog, int code)
 	fprintf(stderr, "    --pseudo-realtime[=<speedup>]   | enable pseudo-realtime for performance "
 	                "evaluation (default 1)\n");
 	fprintf(stderr, "    -j|--jobs                       | enable supervisor mode\n");
+
 	fprintf(stderr, "    --test                          | run unit tests ('--test -h' for help, "
-	                "only when compiling with ENABLE_ZEEK_UNIT_TESTS)\n");
+	                "not available when built without ENABLE_ZEEK_UNIT_TESTS)\n");
 	fprintf(stderr, "    $ZEEKPATH                       | file search path (%s)\n",
 	        util::zeek_path().c_str());
 	fprintf(stderr, "    $ZEEK_PLUGIN_PATH               | plugin search path (%s)\n",

src/zeek-setup.cc

@@ -416,13 +416,7 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 		}
 
 	if ( options.run_unit_tests )
-		{
+		options.deterministic_mode = true;
-		doctest::Context context;
-		auto dargs = to_cargs(options.doctest_args);
-		context.applyCommandLine(dargs.size(), dargs.data());
-		ZEEK_LSAN_ENABLE();
-		exit(context.run());
-		}
 
 	auto stem = Supervisor::CreateStem(options.supervisor_mode);
 
@@ -601,6 +595,16 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 
 	plugin_mgr->ActivateDynamicPlugins(! options.bare_mode);
 
+	// Delay the unit test until here so that plugins have been loaded.
+	if ( options.run_unit_tests )
+		{
+		doctest::Context context;
+		auto dargs = to_cargs(options.doctest_args);
+		context.applyCommandLine(dargs.size(), dargs.data());
+		ZEEK_LSAN_ENABLE();
+		exit(context.run());
+		}
+
 	// Print usage after plugins load so that any path extensions are properly shown.
 	if ( options.print_usage )
 		usage(argv[0], 0);

testing/btest/Baseline/plugins.doctest-disabled/testnames

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/Baseline/plugins.doctest-supported/testnames

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+doctest-plugin/demotest

testing/btest/Baseline/plugins.doctest-supported/testresults

@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+[doctest] doctest version is "x.y.z"
+[doctest] run with "--help" for options
+===============================================================================
+[doctest] test cases: 1 | 1 passed | 0 failed | XX skipped
+[doctest] assertions: 1 | 1 passed | 0 failed |
+[doctest] Status: SUCCESS!

testing/btest/Baseline/plugins.doctest-unsupported/testnames

@@ -0,0 +1 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.

testing/btest/plugins/doctest-disabled.zeek

@@ -0,0 +1,12 @@
+# This requires Zeek with unit test support. The following errors if disabled.
+# @TEST-REQUIRES: zeek --test -h >/dev/null
+
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Doctest
+# @TEST-EXEC: cp -r %DIR/doctest-plugin/* .
+
+# Build the plugin without unit-test support.
+# @TEST-EXEC: ./configure --disable-cpp-tests --zeek-dist=${DIST} && make
+#
+# List the plugin's test names -- there shouldn't be any.
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Doctest" ZEEK_PLUGIN_PATH=`pwd` zeek --test -ltc | grep doctest-plugin >testnames || true
+# @TEST-EXEC: btest-diff testnames

testing/btest/plugins/doctest-plugin/src/Plugin.cc

@@ -0,0 +1,25 @@
+
+#include "Plugin.h"
+
+#include <zeek/3rdparty/doctest.h>
+
+
+namespace btest::plugin::Demo_Doctest { Plugin plugin; }
+
+using namespace btest::plugin::Demo_Doctest;
+
+zeek::plugin::Configuration Plugin::Configure()
+	{
+	zeek::plugin::Configuration config;
+	config.name = "Demo::Doctest";
+	config.description = "Run doctest in a unit-test enabled build";
+	config.version.major = 1;
+	config.version.minor = 0;
+	config.version.patch = 0;
+	return config;
+	}
+
+TEST_CASE("doctest-plugin/demotest")
+        {
+        CHECK(true);
+        }

testing/btest/plugins/doctest-plugin/src/Plugin.h

@@ -0,0 +1,17 @@
+
+#pragma once
+
+#include <zeek/plugin/Plugin.h>
+
+namespace btest::plugin::Demo_Doctest {
+
+class Plugin : public zeek::plugin::Plugin
+{
+protected:
+	// Overridden from zeek::plugin::Plugin.
+	zeek::plugin::Configuration Configure() override;
+};
+
+extern Plugin plugin;
+
+}

testing/btest/plugins/doctest-supported.zeek

@@ -0,0 +1,18 @@
+# This requires Zeek with unit test support. The following errors if disabled.
+# @TEST-REQUIRES: zeek --test -h >/dev/null
+
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Doctest
+# @TEST-EXEC: cp -r %DIR/doctest-plugin/* .
+
+# Build the plugin with unit-test support. Zeek supports it, so we should
+# get runnable tests.
+# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
+#
+# List the plugin's test names.
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Doctest" ZEEK_PLUGIN_PATH=`pwd` zeek --test -ltc | grep doctest-plugin >testnames
+# @TEST-EXEC: btest-diff testnames
+
+# The seed file affects some of the unit tests, so we unset it.
+# Running the unit tests implies deterministic mode, -D.
+# @TEST-EXEC: ZEEK_SEED_FILE= ZEEK_PLUGIN_ACTIVATE="Demo::Doctest" ZEEK_PLUGIN_PATH=`pwd` zeek --test --test-case='doctest-plugin/*' >testresults
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=diff-clean-doctest btest-diff testresults

testing/btest/plugins/doctest-unsupported.zeek

@@ -0,0 +1,13 @@
+# This requires Zeek without unit test support. The following errors if enabled.
+# @TEST-REQUIRES: ! zeek --test -h >/dev/null
+
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Doctest
+# @TEST-EXEC: cp -r %DIR/doctest-plugin/* .
+
+# Build the plugin without disabling unit testing. Zeek doesn't support it,
+# so the plugin should automatically build without it.
+# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
+#
+# List the plugin's test names -- there shouldn't be any.
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Doctest" ZEEK_PLUGIN_PATH=`pwd` zeek --test -ltc | grep doctest-plugin >testnames || true
+# @TEST-EXEC: btest-diff testnames

testing/scripts/diff-clean-doctest

@@ -0,0 +1,17 @@
+#! /usr/bin/env bash
+#
+# doctest's console reports contain several aspects that change over time:
+# - The total number of tests, which we replace with "XX"
+# - The version number, which becomes "x.y.z"
+# - Spacing in the report, which we normalize to single spaces
+
+# Get us "modern" regexps with sed.
+if [ $(uname) == "Linux" ]; then
+    sed="sed -r"
+else
+    sed="sed -E"
+fi
+
+$sed -e 's/[0-9]+ skipped/XX skipped/g' |
+    $sed -e 's/"[0-9]+\.[0-9]+\.[0-9]+"/"x.y.z"/g' |
+    $sed -e 's/ {2,}/ /g'