Added --include-plugins configure argument

.gitignore

@@ -1,4 +1,7 @@
-build
+# Ignore anything prefixed with build since people
+# tend to name all of their build directories prefixed that way.
+build*
+
 tmp
 *.gcov
 

CMakeLists.txt

@@ -573,6 +573,34 @@ if ( INSTALL_ZKG )
         DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config)
 endif ()
 
+########################################################################
+## Look for external plugins to build in
+
+string(REPLACE "," " " _build_in_plugins "${ZEEK_INCLUDE_PLUGINS}")
+separate_arguments(_build_in_plugins)
+foreach(plugin_dir ${_build_in_plugins})
+    if ( NOT IS_ABSOLUTE "${plugin_dir}/CMakeLists.txt" )
+      message(FATAL_ERROR "Plugins to build in need to be defined with absolute path! ${plugin_dir}")
+    endif()
+
+    if ( NOT EXISTS "${plugin_dir}/CMakeLists.txt" )
+      message(FATAL_ERROR "No plugin found at ${plugin_dir}!")
+    endif()
+
+    get_filename_component(plugin_name ${plugin_dir} NAME)
+
+    # Create a list of plugin directories that will then be added in the src/CMakeLists.txt
+    list(APPEND BUILTIN_PLUGIN_LIST ${plugin_dir})
+
+    message(STATUS "  Building in plugin: ${plugin_name} (${plugin_dir})")
+
+    if ( "${ZEEK_BUILTIN_PLUGINS}" STREQUAL "" )
+        set(ZEEK_BUILTIN_PLUGINS ${plugin_name})
+    else ()
+        set(ZEEK_BUILTIN_PLUGINS "${ZEEK_BUILTIN_PLUGINS}, ${plugin_name}")
+    endif ()
+endforeach()
+
 ########################################################################
 ## Recurse on sub-directories
 
@@ -629,6 +657,7 @@ message(
     "\nZeek Script Path:  ${ZEEK_SCRIPT_INSTALL_PATH}"
     "\nDebug mode:        ${ENABLE_DEBUG}"
     "\nUnit tests:        ${ENABLE_ZEEK_UNIT_TESTS}"
+    "\nBuiltin Plugins:   ${ZEEK_BUILTIN_PLUGINS}"
     "\n"
     "\nCC:                ${CMAKE_C_COMPILER}"
     "\nCFLAGS:            ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BuildType}}"

NEWS

@@ -63,6 +63,10 @@ New Functionality
 - The ASCII writer gained a new option LogAscii::logdir, which can be used to
   change the logging output directory.
 
+- Added a ``--include-plugins`` argument to ``configure``. This argument
+  takes a semicolon separated list of paths containing plugins that will be
+  statically built into Zeek.
+
 Changed Functionality
 ---------------------
 

configure

@@ -13,21 +13,23 @@ usage="\
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
   Build Options:
-    --cmake=PATH           custom path to a CMake binary
-    --builddir=DIR         place build files in directory [build]
-    --build-dir=DIR        alias for --builddir
-    --build-type=TYPE      set CMake build type [RelWithDebInfo]:
-                             - Debug: optimizations off, debug symbols + flags
-                             - MinSizeRel: size optimizations, debugging off
-                             - Release: optimizations on, debugging off
-                             - RelWithDebInfo: optimizations on,
-                                   debug symbols on, debug flags off
-    --generator=GENERATOR  CMake generator to use (see cmake --help)
-    --ccache               use ccache to speed up recompilation (requires
-                           ccache installation and CMake 3.10+)
-    --toolchain=PATH       path to a CMAKE_TOOLCHAIN_FILE
-                           (useful for cross-compiling)
-    --sanitizers=LIST      comma-separated list of sanitizer names to enable
+    --cmake=PATH                custom path to a CMake binary
+    --builddir=DIR              place build files in directory [build]
+    --build-dir=DIR             alias for --builddir
+    --build-type=TYPE           set CMake build type [RelWithDebInfo]:
+                                  - Debug: optimizations off, debug symbols + flags
+                                  - MinSizeRel: size optimizations, debugging off
+                                  - Release: optimizations on, debugging off
+                                  - RelWithDebInfo: optimizations on,
+                                        debug symbols on, debug flags off
+    --generator=GENERATOR       CMake generator to use (see cmake --help)
+    --ccache                    use ccache to speed up recompilation (requires
+                                ccache installation and CMake 3.10+)
+    --toolchain=PATH            path to a CMAKE_TOOLCHAIN_FILE
+                                (useful for cross-compiling)
+    --sanitizers=LIST           comma-separated list of sanitizer names to enable
+    --include-plugins=PATHS     paths containing plugins to build directly into Zeek
+                                (semicolon delimited and quoted when multiple)
 
   Installation Directories:
     --prefix=PREFIX        installation directory [/usr/local/zeek]
@@ -170,6 +172,7 @@ append_cache_entry INSTALL_ZKG          BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
 append_cache_entry ZEEK_SANITIZERS      STRING ""
+append_cache_entry ZEEK_INCLUDE_PLUGINS STRING ""
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -208,6 +211,9 @@ while [ $# -ne 0 ]; do
         --toolchain=*)
             append_cache_entry CMAKE_TOOLCHAIN_FILE  PATH   $optarg
             ;;
+        --include-plugins=*)
+            append_cache_entry ZEEK_INCLUDE_PLUGINS  STRING   $optarg
+            ;;
         --prefix=*)
             prefix=$optarg
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg

src/CMakeLists.txt

@@ -173,6 +173,42 @@ add_subdirectory(session)
 
 add_subdirectory(fuzzers)
 
+########################################################################
+## Build in the discovered external plugins and create the autogenerated scripts.
+
+set(PRELOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek)
+file(WRITE ${PRELOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
+set(LOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek)
+file(WRITE ${LOAD_SCRIPT} "# Warning, this is an autogenerated file!\n")
+
+foreach (plugin_dir ${BUILTIN_PLUGIN_LIST})
+    get_filename_component(plugin_name ${plugin_dir} NAME)
+
+    if(IS_DIRECTORY "${plugin_dir}/cmake")
+        set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${plugin_dir}/cmake")
+    endif()
+
+    # Setup the include path for built source artifacts.
+    include_directories(AFTER
+                        ${plugin_dir}/src
+                        ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
+
+    add_subdirectory(${plugin_dir} ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name})
+
+    set(preload_script ${plugin_dir}/scripts/__preload__.zeek)
+    if (EXISTS ${plugin_dir}/scripts/__preload__.zeek)
+        file(APPEND ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek "\n@load ${preload_script}")
+    endif()
+    set(load_script ${plugin_dir}/scripts/__load__.zeek)
+    if (EXISTS ${plugin_dir}/scripts/__load__.zeek)
+        file(APPEND ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek "\n@load ${load_script}")
+    endif()
+endforeach()
+
+install(FILES ${PRELOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
+install(FILES ${LOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/)
+
+
 ########################################################################
 ## bro target
 

src/plugin/Manager.cc

@@ -12,6 +12,7 @@
 #include <optional>
 #include <sstream>
 #include <fstream>
+#include <regex>
 
 #include "zeek/Reporter.h"
 #include "zeek/Func.h"
@@ -171,9 +172,9 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 		// this should be rare to begin with.
 		plugin_list* all_plugins = Manager::ActivePluginsInternal();
 
-		for ( Manager::plugin_list::const_iterator i = all_plugins->begin(); i != all_plugins->end(); i++ )
+		for ( const auto& p : *all_plugins )
 			{
-			if ( (*i)->Name() == name )
+			if ( p->Name() == name )
 				return true;
 			}
 
@@ -227,7 +228,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 
 			current_plugin->SetDynamic(true);
 			current_plugin->DoConfigure();
-			DBG_LOG(DBG_PLUGINS, "  InitialzingComponents");
+			DBG_LOG(DBG_PLUGINS, "  InitializingComponents");
 			current_plugin->InitializeComponents();
 
 			plugins_by_path.insert(std::make_pair(util::detail::normalize_path(dir), current_plugin));
@@ -274,6 +275,15 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 	// First load {scripts}/__preload__.zeek automatically.
 	init = dir + "scripts/__preload__.zeek";
 
+	if ( util::is_file(init) )
+		{
+		DBG_LOG(DBG_PLUGINS, "  Loading %s", init.c_str());
+		scripts_to_load.push_back(init);
+		}
+
+	// First load {scripts}/__preload__.zeek automatically.
+	init = dir + "builtin-plugins/__preload__.zeek";
+
 	if ( util::is_file(init) )
 		{
 		DBG_LOG(DBG_PLUGINS, "  Loading %s", init.c_str());
@@ -403,6 +413,25 @@ void Manager::RegisterBifFile(const char* plugin, bif_init_func c)
 	i->second->push_back(c);
 	}
 
+void Manager::ExtendZeekPathForPlugins()
+	{
+	for ( const auto& p : Manager::ActivePlugins() )
+		{
+		if ( p->DynamicPlugin() || p->Name().empty() )
+			continue;
+
+		string canon = std::regex_replace(p->Name(), std::regex("::"), "_");
+		string dir = "plugins/" + canon + "/";
+		// Use find_file to find the directory in the path.
+		string script_dir = util::find_file(dir, util::zeek_path());
+		if ( ! util::is_dir(script_dir) )
+			continue;
+
+		DBG_LOG(DBG_PLUGINS, "  Adding %s to ZEEKPATH", script_dir.c_str());
+		util::detail::add_to_zeek_path(script_dir);
+		}
+	}
+
 void Manager::InitPreScript()
 	{
 	assert(! init);

src/plugin/Manager.h

@@ -410,6 +410,12 @@ public:
 	 */
 	static void RegisterBifFile(const char* plugin, bif_init_func c);
 
+	/**
+	 * Adds the paths from built-in plugins (e.g. from --include-plugins)
+	 * to ZEEKPATH so they are loaded correctly.
+	 */
+	void ExtendZeekPathForPlugins();
+
 private:
 	bool ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found, std::vector<std::string>* errors);
 	void UpdateInputFiles();

src/zeek-setup.cc

@@ -413,9 +413,6 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	// options from deep within some modules.
 	analysis_options = options.analysis_options;
 
-	if ( options.print_usage )
-		usage(argv[0], 0);
-
 	if ( options.print_version )
 		{
 		fprintf(stdout, "%s version %s\n", argv[0], zeek_version());
@@ -551,6 +548,8 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	if ( ! options.bare_mode )
 		add_input_file("base/init-default.zeek");
 
+	add_input_file("builtin-plugins/__load__.zeek");
+
 	plugin_mgr->SearchDynamicPlugins(util::zeek_plugin_path());
 
 	if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() &&
@@ -598,6 +597,13 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	file_mgr->InitPreScript();
 	zeekygen_mgr->InitPreScript();
 
+	// This has to happen before ActivateDynamicPlugin() below or the list of plugins in the
+	// manager will be missing the plugins we want to try to add to the path.
+	plugin_mgr->ExtendZeekPathForPlugins();
+
+	if ( options.print_usage )
+		usage(argv[0], 0);
+
 	for ( const auto& x : requested_plugins )
 		plugin_mgr->ActivateDynamicPlugin(std::move(x));
 

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -231,6 +231,7 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek
     build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
     build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
+build/scripts/builtin-plugins/__load__.zeek
 scripts/policy/misc/loaded-scripts.zeek
   scripts/base/utils/paths.zeek
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -428,5 +428,6 @@ scripts/base/init-default.zeek
   scripts/base/misc/find-checksum-offloading.zeek
   scripts/base/misc/find-filtered-trace.zeek
   scripts/base/misc/version.zeek
+build/scripts/builtin-plugins/__load__.zeek
 scripts/policy/misc/loaded-scripts.zeek
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/plugins.hooks/output

@@ -1015,6 +1015,7 @@
 0.000000   MetaHookPost  LoadFile(0, base<...>/x509, <...>/x509) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/xmpp, <...>/xmpp) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(1, ./archive, <...>/archive.sig) -> -1
 0.000000   MetaHookPost  LoadFile(1, ./audio, <...>/audio.sig) -> -1
 0.000000   MetaHookPost  LoadFile(1, ./dpd.sig, <...>/dpd.sig) -> -1
@@ -2048,6 +2049,7 @@
 0.000000   MetaHookPre   LoadFile(0, base<...>/x509, <...>/x509)
 0.000000   MetaHookPre   LoadFile(0, base<...>/xmpp, <...>/xmpp)
 0.000000   MetaHookPre   LoadFile(0, base<...>/zeek.bif, <...>/zeek.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek)
 0.000000   MetaHookPre   LoadFile(1, ./archive, <...>/archive.sig)
 0.000000   MetaHookPre   LoadFile(1, ./audio, <...>/audio.sig)
 0.000000   MetaHookPre   LoadFile(1, ./dpd.sig, <...>/dpd.sig)
@@ -3092,6 +3094,7 @@
 0.000000 | HookLoadFile  base<...>/x509 <...>/x509
 0.000000 | HookLoadFile  base<...>/xmpp <...>/xmpp
 0.000000 | HookLoadFile  base<...>/zeek.bif <...>/zeek.bif.zeek
+0.000000 | HookLoadFile  builtin-plugins/__load__.zeek <...>/__load__.zeek
 0.000000 | HookLogInit   packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)}
 0.000000 | HookLogWrite  packet_filter [ts=XXXXXXXXXX.XXXXXX, node=zeek, filter=ip or not ip, init=T, success=T]
 0.000000 | HookQueueEvent NetControl::init()