Retry loading plugins on failure to resolve to dependencies.

Closes #1179.

src/plugin/Manager.cc

@@ -140,7 +140,7 @@ void Manager::SearchDynamicPlugins(const std::string& dir)
 	closedir(d);
 	}
 
-bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found)
+bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found, std::vector<std::string>* errors)
 	{
 	dynamic_plugin_map::iterator m = dynamic_plugins.find(util::strtolower(name));
 
@@ -160,7 +160,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 				return true;
 			}
 
-		reporter->Error("plugin %s is not available", name.c_str());
+		errors->push_back(util::fmt("plugin %s is not available", name.c_str()));
 		return false;
 		}
 
@@ -192,16 +192,19 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 			current_plugin = nullptr;
 			current_dir = dir.c_str();
 			current_sopath = path;
-			void* hdl = dlopen(path, RTLD_LAZY | RTLD_GLOBAL);
+			void* hdl = dlopen(path, RTLD_NOW | RTLD_GLOBAL);
 
 			if ( ! hdl )
 				{
 				const char* err = dlerror();
-				reporter->FatalError("cannot load plugin library %s: %s", path, err ? err : "<unknown error>");
+				errors->push_back(util::fmt("cannot load plugin library %s: %s", path, err ? err : "<unknown error>"));
+				return false;
 				}
 
-			if ( ! current_plugin )
-				reporter->FatalError("load plugin library %s did not instantiate a plugin", path);
+			if ( ! current_plugin ) {
+				errors->push_back(util::fmt("load plugin library %s did not instantiate a plugin", path));
+				return false;
+			}
 
 			current_plugin->SetDynamic(true);
 			current_plugin->DoConfigure();
@@ -217,9 +220,11 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 
 			// Make sure the name the plugin reports is consistent with
 			// what we expect from its magic file.
-			if ( util::strtolower(current_plugin->Name()) != util::strtolower(name) )
-				reporter->FatalError("inconsistent plugin name: %s vs %s",
-						     current_plugin->Name().c_str(), name.c_str());
+			if ( util::strtolower(current_plugin->Name()) != util::strtolower(name) ) {
+				errors->push_back(util::fmt("inconsistent plugin name: %s vs %s",
+						     current_plugin->Name().c_str(), name.c_str()));
+				return false;
+			}
 
 			current_dir = nullptr;
 			current_sopath = nullptr;
@@ -294,37 +299,66 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_
 	return true;
 	}
 
-bool Manager::ActivateDynamicPlugin(const std::string& name)
+void Manager::ActivateDynamicPlugin(const std::string& name)
 	{
-	if ( ! ActivateDynamicPluginInternal(name) )
-		return false;
-
+	std::vector<std::string> errors;
+	if ( ActivateDynamicPluginInternal(name, false, &errors) )
 		UpdateInputFiles();
-	return true;
+	else
+		// Reschedule for another attempt later.
+		requested_plugins.insert(std::move(name));
 	}
 
-bool Manager::ActivateDynamicPlugins(bool all)
-	{
+void Manager::ActivateDynamicPlugins(bool all) {
+	// Tracks plugins we need to activate as pairs of their names and booleans
+	// indicating whether an activation failure is to be deemed a fatal error.
+	std::set<std::pair<std::string, bool>> plugins_to_activate;
+
+	// Activate plugins that were specifically requested.
+	for ( const auto& x : requested_plugins )
+		plugins_to_activate.emplace(x, false);
+
 	// Activate plugins that our environment tells us to.
 	vector<string> p;
 	util::tokenize_string(util::zeek_plugin_activate(), ",", &p);
 
-	for ( size_t n = 0; n < p.size(); ++n )
-		ActivateDynamicPluginInternal(p[n], true);
+	for ( const auto& x : p )
+		plugins_to_activate.emplace(x, true);
 
 	if ( all )
 		{
-		for ( dynamic_plugin_map::const_iterator i = dynamic_plugins.begin();
-		      i != dynamic_plugins.end(); i++ )
-			{
-			if ( ! ActivateDynamicPluginInternal(i->first) )
-				return false;
+		// Activate all other ones we discovered.
+		for ( const auto& x : dynamic_plugins )
+			plugins_to_activate.emplace(x.first, false);
 		}
+
+	// Now we keep iterating over all the plugins, trying to load them, for as
+	// long as we're successful for at least one further of them each round.
+	// Doing so ensures that we can resolve (non-cyclic) load dependencies
+	// independent of any particular order.
+	while ( ! plugins_to_activate.empty() ) {
+		std::vector<std::string> errors;
+		auto plugins_left = plugins_to_activate;
+
+		for ( const auto& x : plugins_to_activate )
+			{
+			if ( ActivateDynamicPluginInternal(x.first, x.second, &errors) )
+				plugins_left.erase(x);
+			}
+
+		if ( plugins_left.size() == plugins_to_activate.size() )
+			{
+			// Could not load a single further plugin this round, that's fatal.
+			for ( const auto& msg : errors )
+				reporter->Error("%s", msg.c_str());
+
+			reporter->FatalError("aborting after plugin errors");
+			}
+
+		plugins_to_activate = std::move(plugins_left);
 		}
 
 	UpdateInputFiles();
-
-	return true;
 	}
 
 void Manager::UpdateInputFiles()

src/plugin/Manager.h

@@ -2,9 +2,10 @@
 
 #pragma once
 
-#include <utility>
 #include <map>
+#include <set>
 #include <string_view>
+#include <utility>
 
 #include "zeek/plugin/Plugin.h"
 #include "zeek/plugin/Component.h"
@@ -79,28 +80,25 @@ public:
 	 * Activating a plugin involves loading its dynamic module, making its
 	 * bifs available, and adding its script paths to ZEEKPATH.
 	 *
+	 * This attempts to activiate the plugin immediately. If that fails for
+	 * some reason, we schedule it to be retried later with
+	 * ActivateDynamicPlugins().
+	 *
 	 * @param name The name of the plugin, as found previously by
-	 * SearchPlugin().
-	 *
-	 * @return True if the plugin has been loaded successfully.
-	 *
+	·* SearchPlugin().
 	 */
-	bool ActivateDynamicPlugin(const std::string& name);
+	void ActivateDynamicPlugin(const std::string& name);
 
 	/**
-	 * Activates plugins that SearchDynamicPlugins() has previously discovered.
-	 * The effect is the same all calling \a ActivePlugin(name) for each plugin.
+	 * Activates plugins that SearchDynamicPlugins() has previously discovered,
+	 * including any that have failed to load in prior calls to
+	 * ActivateDynamicPlugin(). Aborts if any plugins fails to activate.
 	 *
 	 * @param all If true, activates all plugins that are found. If false,
 	 * activates only those that should always be activated unconditionally,
-	 * as specified via the ZEEK_PLUGIN_ACTIVATE enviroment variable. In other
-	 * words, it's \c true in standard mode and \c false in bare mode.
-	 *
-	 * @return True if all plugins have been loaded successfully. If one
-	 * fails to load, the method stops there without loading any further ones
-	 * and returns false.
+	 * as specified via the ZEEK_PLUGIN_ACTIVATE environment variable. In other
 	 */
-	bool ActivateDynamicPlugins(bool all);
+	void ActivateDynamicPlugins(bool all);
 
 	/**
 	 * First-stage initializion of the manager. This is called early on
@@ -413,11 +411,15 @@ public:
 	static void RegisterBifFile(const char* plugin, bif_init_func c);
 
 private:
-	bool ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found = false);
+	bool ActivateDynamicPluginInternal(const std::string& name, bool ok_if_not_found, std::vector<std::string>* errors);
 	void UpdateInputFiles();
 	void MetaHookPre(HookType hook, const HookArgumentList& args) const;
 	void MetaHookPost(HookType hook, const HookArgumentList& args, HookArgument result) const;
 
+	// Plugins that were explicitly requested to be activated, but failed to
+	// load at first.
+	std::set<std::string> requested_plugins;
+
 	 // All found dynamic plugins, mapping their names to base directory.
 	using dynamic_plugin_map = std::map<std::string, std::string>;
 	dynamic_plugin_map dynamic_plugins;

src/zeek-setup.cc

@@ -604,17 +604,8 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 	file_mgr->InitPreScript();
 	zeekygen_mgr->InitPreScript();
 
-	bool missing_plugin = false;
-
-	for ( set<string>::const_iterator i = requested_plugins.begin();
-	      i != requested_plugins.end(); i++ )
-		{
-		if ( ! plugin_mgr->ActivateDynamicPlugin(*i) )
-			missing_plugin = true;
-		}
-
-	if ( missing_plugin )
-		reporter->FatalError("Failed to activate requested dynamic plugin(s).");
+	for ( const auto& x : requested_plugins )
+		plugin_mgr->ActivateDynamicPlugin(std::move(x));
 
 	plugin_mgr->ActivateDynamicPlugins(! options.bare_mode);