diff --git a/.gitignore b/.gitignore index 75378c74ed..47d737f573 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,7 @@ -build +# Ignore anything prefixed with build since people +# tend to name all of their build directories prefixed that way. +build* + tmp *.gcov diff --git a/CMakeLists.txt b/CMakeLists.txt index e993b9c878..21db791f1e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -573,6 +573,34 @@ if ( INSTALL_ZKG ) DESTINATION ${ZEEK_ZKG_CONFIG_DIR} RENAME config) endif () +######################################################################## +## Look for external plugins to build in + +string(REPLACE "," " " _build_in_plugins "${ZEEK_INCLUDE_PLUGINS}") +separate_arguments(_build_in_plugins) +foreach(plugin_dir ${_build_in_plugins}) + if ( NOT IS_ABSOLUTE "${plugin_dir}/CMakeLists.txt" ) + message(FATAL_ERROR "Plugins to build in need to be defined with absolute path! ${plugin_dir}") + endif() + + if ( NOT EXISTS "${plugin_dir}/CMakeLists.txt" ) + message(FATAL_ERROR "No plugin found at ${plugin_dir}!") + endif() + + get_filename_component(plugin_name ${plugin_dir} NAME) + + # Create a list of plugin directories that will then be added in the src/CMakeLists.txt + list(APPEND BUILTIN_PLUGIN_LIST ${plugin_dir}) + + message(STATUS " Building in plugin: ${plugin_name} (${plugin_dir})") + + if ( "${ZEEK_BUILTIN_PLUGINS}" STREQUAL "" ) + set(ZEEK_BUILTIN_PLUGINS ${plugin_name}) + else () + set(ZEEK_BUILTIN_PLUGINS "${ZEEK_BUILTIN_PLUGINS}, ${plugin_name}") + endif () +endforeach() + ######################################################################## ## Recurse on sub-directories @@ -629,6 +657,7 @@ message( "\nZeek Script Path: ${ZEEK_SCRIPT_INSTALL_PATH}" "\nDebug mode: ${ENABLE_DEBUG}" "\nUnit tests: ${ENABLE_ZEEK_UNIT_TESTS}" + "\nBuiltin Plugins: ${ZEEK_BUILTIN_PLUGINS}" "\n" "\nCC: ${CMAKE_C_COMPILER}" "\nCFLAGS: ${CMAKE_C_FLAGS} ${CMAKE_C_FLAGS_${BuildType}}" diff --git a/NEWS b/NEWS index f9106bf7d3..b8fbdf1f6f 100644 --- a/NEWS +++ b/NEWS @@ -63,6 +63,10 @@ New Functionality - The ASCII writer gained a new option LogAscii::logdir, which can be used to change the logging output directory. +- Added a ``--include-plugins`` argument to ``configure``. This argument + takes a semicolon separated list of paths containing plugins that will be + statically built into Zeek. + Changed Functionality --------------------- diff --git a/configure b/configure index d75be8019c..3b07af0ee2 100755 --- a/configure +++ b/configure @@ -13,21 +13,23 @@ usage="\ Usage: $0 [OPTION]... [VAR=VALUE]... Build Options: - --cmake=PATH custom path to a CMake binary - --builddir=DIR place build files in directory [build] - --build-dir=DIR alias for --builddir - --build-type=TYPE set CMake build type [RelWithDebInfo]: - - Debug: optimizations off, debug symbols + flags - - MinSizeRel: size optimizations, debugging off - - Release: optimizations on, debugging off - - RelWithDebInfo: optimizations on, - debug symbols on, debug flags off - --generator=GENERATOR CMake generator to use (see cmake --help) - --ccache use ccache to speed up recompilation (requires - ccache installation and CMake 3.10+) - --toolchain=PATH path to a CMAKE_TOOLCHAIN_FILE - (useful for cross-compiling) - --sanitizers=LIST comma-separated list of sanitizer names to enable + --cmake=PATH custom path to a CMake binary + --builddir=DIR place build files in directory [build] + --build-dir=DIR alias for --builddir + --build-type=TYPE set CMake build type [RelWithDebInfo]: + - Debug: optimizations off, debug symbols + flags + - MinSizeRel: size optimizations, debugging off + - Release: optimizations on, debugging off + - RelWithDebInfo: optimizations on, + debug symbols on, debug flags off + --generator=GENERATOR CMake generator to use (see cmake --help) + --ccache use ccache to speed up recompilation (requires + ccache installation and CMake 3.10+) + --toolchain=PATH path to a CMAKE_TOOLCHAIN_FILE + (useful for cross-compiling) + --sanitizers=LIST comma-separated list of sanitizer names to enable + --include-plugins=PATHS paths containing plugins to build directly into Zeek + (semicolon delimited and quoted when multiple) Installation Directories: --prefix=PREFIX installation directory [/usr/local/zeek] @@ -170,6 +172,7 @@ append_cache_entry INSTALL_ZKG BOOL true append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING append_cache_entry ENABLE_MOBILE_IPV6 BOOL false append_cache_entry ZEEK_SANITIZERS STRING "" +append_cache_entry ZEEK_INCLUDE_PLUGINS STRING "" # parse arguments while [ $# -ne 0 ]; do @@ -208,6 +211,9 @@ while [ $# -ne 0 ]; do --toolchain=*) append_cache_entry CMAKE_TOOLCHAIN_FILE PATH $optarg ;; + --include-plugins=*) + append_cache_entry ZEEK_INCLUDE_PLUGINS STRING $optarg + ;; --prefix=*) prefix=$optarg append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 8c403330b7..b9462c5ea5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -173,6 +173,42 @@ add_subdirectory(session) add_subdirectory(fuzzers) +######################################################################## +## Build in the discovered external plugins and create the autogenerated scripts. + +set(PRELOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek) +file(WRITE ${PRELOAD_SCRIPT} "# Warning, this is an autogenerated file!\n") +set(LOAD_SCRIPT ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek) +file(WRITE ${LOAD_SCRIPT} "# Warning, this is an autogenerated file!\n") + +foreach (plugin_dir ${BUILTIN_PLUGIN_LIST}) + get_filename_component(plugin_name ${plugin_dir} NAME) + + if(IS_DIRECTORY "${plugin_dir}/cmake") + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${plugin_dir}/cmake") + endif() + + # Setup the include path for built source artifacts. + include_directories(AFTER + ${plugin_dir}/src + ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name}) + + add_subdirectory(${plugin_dir} ${CMAKE_CURRENT_BINARY_DIR}/builtin-plugins/${plugin_name}) + + set(preload_script ${plugin_dir}/scripts/__preload__.zeek) + if (EXISTS ${plugin_dir}/scripts/__preload__.zeek) + file(APPEND ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__preload__.zeek "\n@load ${preload_script}") + endif() + set(load_script ${plugin_dir}/scripts/__load__.zeek) + if (EXISTS ${plugin_dir}/scripts/__load__.zeek) + file(APPEND ${CMAKE_BINARY_DIR}/scripts/builtin-plugins/__load__.zeek "\n@load ${load_script}") + endif() +endforeach() + +install(FILES ${PRELOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/) +install(FILES ${LOAD_SCRIPT} DESTINATION ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins/) + + ######################################################################## ## bro target diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index da4ff5fe32..7880f6ea68 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -12,6 +12,7 @@ #include #include #include +#include #include "zeek/Reporter.h" #include "zeek/Func.h" @@ -171,9 +172,9 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ // this should be rare to begin with. plugin_list* all_plugins = Manager::ActivePluginsInternal(); - for ( Manager::plugin_list::const_iterator i = all_plugins->begin(); i != all_plugins->end(); i++ ) + for ( const auto& p : *all_plugins ) { - if ( (*i)->Name() == name ) + if ( p->Name() == name ) return true; } @@ -227,7 +228,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ current_plugin->SetDynamic(true); current_plugin->DoConfigure(); - DBG_LOG(DBG_PLUGINS, " InitialzingComponents"); + DBG_LOG(DBG_PLUGINS, " InitializingComponents"); current_plugin->InitializeComponents(); plugins_by_path.insert(std::make_pair(util::detail::normalize_path(dir), current_plugin)); @@ -274,6 +275,15 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ // First load {scripts}/__preload__.zeek automatically. init = dir + "scripts/__preload__.zeek"; + if ( util::is_file(init) ) + { + DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); + scripts_to_load.push_back(init); + } + + // First load {scripts}/__preload__.zeek automatically. + init = dir + "builtin-plugins/__preload__.zeek"; + if ( util::is_file(init) ) { DBG_LOG(DBG_PLUGINS, " Loading %s", init.c_str()); @@ -403,6 +413,25 @@ void Manager::RegisterBifFile(const char* plugin, bif_init_func c) i->second->push_back(c); } +void Manager::ExtendZeekPathForPlugins() + { + for ( const auto& p : Manager::ActivePlugins() ) + { + if ( p->DynamicPlugin() || p->Name().empty() ) + continue; + + string canon = std::regex_replace(p->Name(), std::regex("::"), "_"); + string dir = "plugins/" + canon + "/"; + // Use find_file to find the directory in the path. + string script_dir = util::find_file(dir, util::zeek_path()); + if ( ! util::is_dir(script_dir) ) + continue; + + DBG_LOG(DBG_PLUGINS, " Adding %s to ZEEKPATH", script_dir.c_str()); + util::detail::add_to_zeek_path(script_dir); + } + } + void Manager::InitPreScript() { assert(! init); diff --git a/src/plugin/Manager.h b/src/plugin/Manager.h index 6af81e7b96..371d4a8a8a 100644 --- a/src/plugin/Manager.h +++ b/src/plugin/Manager.h @@ -410,6 +410,12 @@ public: */ static void RegisterBifFile(const char* plugin, bif_init_func c); + /** + * Adds the paths from built-in plugins (e.g. from --include-plugins) + * to ZEEKPATH so they are loaded correctly. + */ + void ExtendZeekPathForPlugins(); + private: bool ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found, std::vector* errors); void UpdateInputFiles(); diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 4e1d046b56..ac706ecc4b 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -413,9 +413,6 @@ SetupResult setup(int argc, char** argv, Options* zopts) // options from deep within some modules. analysis_options = options.analysis_options; - if ( options.print_usage ) - usage(argv[0], 0); - if ( options.print_version ) { fprintf(stdout, "%s version %s\n", argv[0], zeek_version()); @@ -551,6 +548,8 @@ SetupResult setup(int argc, char** argv, Options* zopts) if ( ! options.bare_mode ) add_input_file("base/init-default.zeek"); + add_input_file("builtin-plugins/__load__.zeek"); + plugin_mgr->SearchDynamicPlugins(util::zeek_plugin_path()); if ( options.plugins_to_load.empty() && options.scripts_to_load.empty() && @@ -598,6 +597,13 @@ SetupResult setup(int argc, char** argv, Options* zopts) file_mgr->InitPreScript(); zeekygen_mgr->InitPreScript(); + // This has to happen before ActivateDynamicPlugin() below or the list of plugins in the + // manager will be missing the plugins we want to try to add to the path. + plugin_mgr->ExtendZeekPathForPlugins(); + + if ( options.print_usage ) + usage(argv[0], 0); + for ( const auto& x : requested_plugins ) plugin_mgr->ActivateDynamicPlugin(std::move(x)); diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 95228fc7ea..f0ea2b3140 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -231,6 +231,7 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek +build/scripts/builtin-plugins/__load__.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 4ebc37ef20..afb2e4649e 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -428,5 +428,6 @@ scripts/base/init-default.zeek scripts/base/misc/find-checksum-offloading.zeek scripts/base/misc/find-filtered-trace.zeek scripts/base/misc/version.zeek +build/scripts/builtin-plugins/__load__.zeek scripts/policy/misc/loaded-scripts.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index f9928d808e..87dc9570e5 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -1015,6 +1015,7 @@ 0.000000 MetaHookPost LoadFile(0, base<...>/x509, <...>/x509) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/xmpp, <...>/xmpp) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) -> -1 0.000000 MetaHookPost LoadFile(1, ./archive, <...>/archive.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./audio, <...>/audio.sig) -> -1 0.000000 MetaHookPost LoadFile(1, ./dpd.sig, <...>/dpd.sig) -> -1 @@ -2048,6 +2049,7 @@ 0.000000 MetaHookPre LoadFile(0, base<...>/x509, <...>/x509) 0.000000 MetaHookPre LoadFile(0, base<...>/xmpp, <...>/xmpp) 0.000000 MetaHookPre LoadFile(0, base<...>/zeek.bif, <...>/zeek.bif.zeek) +0.000000 MetaHookPre LoadFile(0, builtin-plugins/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFile(1, ./archive, <...>/archive.sig) 0.000000 MetaHookPre LoadFile(1, ./audio, <...>/audio.sig) 0.000000 MetaHookPre LoadFile(1, ./dpd.sig, <...>/dpd.sig) @@ -3092,6 +3094,7 @@ 0.000000 | HookLoadFile base<...>/x509 <...>/x509 0.000000 | HookLoadFile base<...>/xmpp <...>/xmpp 0.000000 | HookLoadFile base<...>/zeek.bif <...>/zeek.bif.zeek +0.000000 | HookLoadFile builtin-plugins/__load__.zeek <...>/__load__.zeek 0.000000 | HookLogInit packet_filter 1/1 {ts (time), node (string), filter (string), init (bool), success (bool)} 0.000000 | HookLogWrite packet_filter [ts=XXXXXXXXXX.XXXXXX, node=zeek, filter=ip or not ip, init=T, success=T] 0.000000 | HookQueueEvent NetControl::init()