Merge remote-tracking branch 'origin/topic/robin/gh-1757-loadfile2'

* origin/topic/robin/gh-1757-loadfile2:
  Tweaking a couple of debug message.
  Add new hook `HookLoadFileExtended` that allows plugins to supply Zeek script code to parse.
  Move logic to execute `HookLoadFile` for signatures into rule matcher code.

CHANGES

@@ -1,3 +1,18 @@
+4.2.0-dev.314 | 2021-11-10 11:16:28 +0100
+
+  * GH-1757: Add new hook `HookLoadFileExtended` that allows plugins
+    to supply Zeek script and signature code to parse. (Robin Sommer)
+
+    The new hook works similar to the existing `HookLoadFile` but,
+    additionally, allows the plugin to return a string that contains
+    the code to be used for the file being loaded. If the plugin does
+    so, the content of any actual file on disk will be ignored. This
+    works for both Zeek scripts and signatures.
+
+  * Fix an issue where signature files supplied on the command line
+    wouldn't pass through the file loading hooks. (Robin Sommer,
+    Corelight)
+
 4.2.0-dev.310 | 2021-11-09 10:29:59 -0700
 
   * Add Github action exercising pre-commit (Benjamin Bannier, Corelight)

VERSION

@@ -1 +1 @@
-4.2.0-dev.310
+4.2.0-dev.314

src/PolicyFile.cc

@@ -49,7 +49,7 @@ int how_many_lines_in(const char* policy_filename)
 	FILE* throwaway = fopen(policy_filename, "r");
 	if ( ! throwaway )
 		{
-		debug_msg("No such policy file: %s.\n", policy_filename);
+		debug_msg("Could not open policy file: %s.\n", policy_filename);
 		return -1;
 		}
 
@@ -72,49 +72,60 @@ int how_many_lines_in(const char* policy_filename)
 	return pf->lines.size();
 	}
 
-bool LoadPolicyFileText(const char* policy_filename)
+bool LoadPolicyFileText(const char* policy_filename,
+                        const std::optional<std::string>& preloaded_content)
 	{
 	if ( ! policy_filename )
 		return true;
 
-	FILE* f = fopen(policy_filename, "r");
-
-	if ( ! f )
-		{
-		debug_msg("No such policy file: %s.\n", policy_filename);
-		return false;
-		}
-
-	PolicyFile* pf = new PolicyFile;
-
 	if ( policy_files.find(policy_filename) != policy_files.end() )
 		debug_msg("Policy file %s already loaded\n", policy_filename);
 
+	PolicyFile* pf = new PolicyFile;
 	policy_files.insert(PolicyFileMap::value_type(policy_filename, pf));
 
-	struct stat st;
+	if ( preloaded_content )
-	if ( fstat(fileno(f), &st) != 0 )
 		{
-		char buf[256];
+		auto size = preloaded_content->size();
-		util::zeek_strerror_r(errno, buf, sizeof(buf));
+		pf->filedata = new char[size + 1];
-		reporter->Error("fstat failed on %s: %s", policy_filename, buf);
+		memcpy(pf->filedata, preloaded_content->data(), size);
-		fclose(f);
+		pf->filedata[size] = '\0';
-		return false;
 		}
+	else
+		{
+		FILE* f = fopen(policy_filename, "r");
 
-	pf->lmtime = st.st_mtime;
+		if ( ! f )
-	off_t size = st.st_size;
+			{
+			debug_msg("Could not open policy file: %s.\n", policy_filename);
+			return false;
+			}
 
-	// ### This code is not necessarily Unicode safe!
+		struct stat st;
-	// (probably fine with UTF-8)
+		if ( fstat(fileno(f), &st) != 0 )
-	pf->filedata = new char[size + 1];
+			{
-	if ( fread(pf->filedata, size, 1, f) != 1 )
+			char buf[256];
-		reporter->InternalError("Failed to fread() file data");
+			util::zeek_strerror_r(errno, buf, sizeof(buf));
-	pf->filedata[size] = 0;
+			reporter->Error("fstat failed on %s: %s", policy_filename, buf);
-	fclose(f);
+			fclose(f);
+			return false;
+			}
+
+		pf->lmtime = st.st_mtime;
+		off_t size = st.st_size;
+
+		// ### This code is not necessarily Unicode safe!
+		// (probably fine with UTF-8)
+		pf->filedata = new char[size + 1];
+		if ( fread(pf->filedata, size, 1, f) != 1 )
+			reporter->InternalError("Failed to fread() file data");
+		pf->filedata[size] = 0;
+		fclose(f);
+		}
 
 	// Separate the string by newlines.
 	pf->lines.push_back(pf->filedata);
+
 	for ( char* iter = pf->filedata; *iter; ++iter )
 		{
 		if ( *iter == '\n' )
@@ -141,7 +152,7 @@ bool PrintLines(const char* policy_filename, unsigned int start_line, unsigned i
 	FILE* throwaway = fopen(policy_filename, "r");
 	if ( ! throwaway )
 		{
-		debug_msg("No such policy file: %s.\n", policy_filename);
+		debug_msg("Could not open policy file: %s.\n", policy_filename);
 		return false;
 		}
 

src/PolicyFile.h

@@ -14,12 +14,16 @@
 // policy_filename arguments should be absolute or relative paths;
 // no expansion is done.
 
+#include <optional>
+#include <string>
+
 namespace zeek::detail
 	{
 
 int how_many_lines_in(const char* policy_filename);
 
-bool LoadPolicyFileText(const char* policy_filename);
+bool LoadPolicyFileText(const char* policy_filename,
+                        const std::optional<std::string>& preloaded_content = {});
 
 // start_line is 1-based (the intuitive way)
 bool PrintLines(const char* policy_filename, unsigned int start_line, unsigned int how_many_lines,

src/RuleMatcher.cc

@@ -27,6 +27,11 @@
 
 using namespace std;
 
+// Functions exposed by rule-scan.l
+extern void rules_set_input_from_buffer(const char* data, size_t size);
+extern void rules_set_input_from_file(FILE* f);
+extern void rules_parse_input();
+
 namespace zeek::detail
 	{
 
@@ -248,7 +253,7 @@ void RuleMatcher::Delete(RuleHdrTest* node)
 	delete node;
 	}
 
-bool RuleMatcher::ReadFiles(const std::vector<std::string>& files)
+bool RuleMatcher::ReadFiles(const std::vector<SignatureFile>& files)
 	{
 #ifdef USE_PERFTOOLS_DEBUG
 	HeapLeakChecker::Disabler disabler;
@@ -256,20 +261,82 @@ bool RuleMatcher::ReadFiles(const std::vector<std::string>& files)
 
 	parse_error = false;
 
-	for ( const auto& f : files )
+	for ( auto f : files )
 		{
-		rules_in = util::open_file(util::find_file(f, util::zeek_path(), ".sig"));
+		if ( ! f.full_path )
+			f.full_path = util::find_file(f.file, util::zeek_path(), ".sig");
 
-		if ( ! rules_in )
+		std::pair<int, std::optional<std::string>> rc = {-1, std::nullopt};
+		rc.first = PLUGIN_HOOK_WITH_RESULT(
+			HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SIGNATURES, f.file, *f.full_path),
+			-1);
+
+		if ( rc.first < 0 )
+			rc = PLUGIN_HOOK_WITH_RESULT(
+				HOOK_LOAD_FILE_EXT,
+				HookLoadFileExtended(zeek::plugin::Plugin::SIGNATURES, f.file, *f.full_path),
+				std::make_pair(-1, std::nullopt));
+
+		switch ( rc.first )
 			{
-			reporter->Error("Can't open signature file %s", f.data());
+			case -1:
-			return false;
+				// No plugin in charge of this file.
+				if ( f.full_path->empty() )
+					{
+					zeek::reporter->Error("failed to find file associated with @load-sigs %s",
+					                      f.file.c_str());
+					continue;
+					}
+				break;
+
+			case 0:
+				if ( ! zeek::reporter->Errors() )
+					zeek::reporter->Error("Plugin reported error loading signatures %s",
+					                      f.file.c_str());
+
+				exit(1);
+				break;
+
+			case 1:
+				if ( ! rc.second )
+					// A plugin took care of it, just skip.
+					continue;
+
+				break;
+
+			default:
+				assert(false);
+				break;
+			}
+
+		FILE* rules_in = nullptr;
+
+		if ( rc.first == 1 )
+			{
+			// Parse code provided by plugin.
+			assert(rc.second);
+			rules_set_input_from_buffer(rc.second->data(), rc.second->size());
+			}
+		else
+			{
+			// Parse from file.
+			rules_in = util::open_file(*f.full_path);
+
+			if ( ! rules_in )
+				{
+				reporter->Error("Can't open signature file %s", f.file.c_str());
+				return false;
+				}
+
+			rules_set_input_from_file(rules_in);
 			}
 
 		rules_line_number = 0;
-		current_rule_file = f.data();
+		current_rule_file = f.full_path->c_str();
-		rules_parse();
+		rules_parse_input();
-		fclose(rules_in);
+
+		if ( rules_in )
+			fclose(rules_in);
 		}
 
 	if ( parse_error )

src/RuleMatcher.h

@@ -11,6 +11,8 @@
 #include "zeek/CCL.h"
 #include "zeek/RE.h"
 #include "zeek/Rule.h"
+#include "zeek/ScannedFile.h"
+#include "zeek/plugin/Manager.h"
 
 //#define MATCHER_PRINT_STATS
 
@@ -22,7 +24,6 @@ extern void rules_error(zeek::detail::Rule* id, const char* msg);
 extern int rules_lex(void);
 extern int rules_parse(void);
 extern "C" int rules_wrap(void);
-extern FILE* rules_in;
 extern int rules_line_number;
 extern const char* current_rule_file;
 
@@ -259,7 +260,7 @@ public:
 	~RuleMatcher();
 
 	// Parse the given files and built up data structures.
-	bool ReadFiles(const std::vector<std::string>& files);
+	bool ReadFiles(const std::vector<SignatureFile>& files);
 
 	/**
 	 * Inititialize a state object for matching file magic signatures.

src/ScannedFile.cc

@@ -10,7 +10,7 @@ namespace zeek::detail
 	{
 
 std::list<ScannedFile> files_scanned;
-std::vector<std::string> sig_files;
+std::vector<SignatureFile> sig_files;
 
 ScannedFile::ScannedFile(int arg_include_level, std::string arg_name, bool arg_skipped,
                          bool arg_prefixes_checked)
@@ -47,4 +47,11 @@ bool ScannedFile::AlreadyScanned() const
 	return rval;
 	}
 
+SignatureFile::SignatureFile(std::string file) : file(std::move(file)) { }
+
+SignatureFile::SignatureFile(std::string file, std::string full_path)
+	: file(std::move(file)), full_path(std::move(full_path))
+	{
+	}
+
 	} // namespace zeek::detail

src/ScannedFile.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include <list>
+#include <optional>
 #include <string>
 #include <vector>
 
@@ -34,6 +35,16 @@ public:
 	};
 
 extern std::list<ScannedFile> files_scanned;
-extern std::vector<std::string> sig_files;
+
+struct SignatureFile
+	{
+	std::string file;
+	std::optional<std::string> full_path;
+
+	SignatureFile(std::string file);
+	SignatureFile(std::string file, std::string full_path);
+	};
+
+extern std::vector<SignatureFile> sig_files;
 
 	} // namespace zeek::detail

src/plugin/Manager.cc

@@ -692,6 +692,41 @@ int Manager::HookLoadFile(const Plugin::LoadType type, const string& file, const
 	return rc;
 	}
 
+std::pair<int, std::optional<std::string>>
+Manager::HookLoadFileExtended(const Plugin::LoadType type, const string& file,
+                              const string& resolved)
+	{
+	HookArgumentList args;
+
+	if ( HavePluginForHook(META_HOOK_PRE) )
+		{
+		args.push_back(HookArgument(type));
+		args.push_back(HookArgument(file));
+		args.push_back(HookArgument(resolved));
+		MetaHookPre(HOOK_LOAD_FILE_EXT, args);
+		}
+
+	hook_list* l = hooks[HOOK_LOAD_FILE_EXT];
+
+	std::pair<int, std::optional<std::string>> rc = {-1, std::nullopt};
+
+	if ( l )
+		for ( hook_list::iterator i = l->begin(); i != l->end(); ++i )
+			{
+			Plugin* p = (*i).second;
+
+			rc = p->HookLoadFileExtended(type, file, resolved);
+
+			if ( rc.first >= 0 )
+				break;
+			}
+
+	if ( HavePluginForHook(META_HOOK_POST) )
+		MetaHookPost(HOOK_LOAD_FILE_EXT, args, HookArgument(rc));
+
+	return rc;
+	}
+
 std::pair<bool, ValPtr> Manager::HookCallFunction(const Func* func, zeek::detail::Frame* parent,
                                                   Args* vecargs) const
 	{

src/plugin/Manager.h

@@ -245,6 +245,32 @@ public:
 	virtual int HookLoadFile(const Plugin::LoadType type, const std::string& file,
 	                         const std::string& resolved);
 
+	/**
+	 * Hook that gives plugins a chance to take over loading an input file,
+	 * including replacing the file's content. This method must be called
+	 * between InitPreScript() and InitPostScript() for each input file Bro is
+	 * about to load, either given on the command line or via @load script
+	 * directives. The hook can take over the file, in which case Bro must not
+	 * further process it otherwise; or provide its content, in which case Bro
+	 * must use that and ignore the original file.
+	 *
+	 * @return tuple where the first element is 1 if a plugin took over the
+	 * file; 0 if a plugin took over the file but had trouble loading it; and
+	 * -1 if no plugin was interested in the file at all.
+	 *
+	 * If the plugins takes over by returning 1, there are two cases: if the
+	 * second tuple element remains unset, the plugin handled the loading
+	 * completely internally; the caller must not process it any further.
+	 * Alternatively, the plugin may optionally return the acutal content to
+	 * use for the file as a string through the tuple's second element. If so,
+	 * the caller must ignore the file on disk and use that provided content
+	 * instead (including when there's actually no physical file in place on
+	 * disk at all).
+	 */
+	virtual std::pair<int, std::optional<std::string>>
+	HookLoadFileExtended(const Plugin::LoadType type, const std::string& file,
+	                     const std::string& resolved);
+
 	/**
 	 * Hook that filters calls to a script function/event/hook.
 	 *

src/plugin/Plugin.cc

@@ -22,6 +22,7 @@ const char* hook_name(HookType h)
 	static constexpr const char* hook_names[int(NUM_HOOKS) + 1] = {
 		// Order must match that of HookType.
 		"LoadFile",
+		"LoadFileExtended",
 		"CallFunction",
 		"QueueEvent",
 		"DrainEvents",
@@ -230,6 +231,21 @@ void HookArgument::Describe(ODesc* d) const
 				{
 				d->Add("<no location>");
 				}
+			break;
+
+		case INPUT_FILE:
+			{
+			d->Add("(");
+			d->Add(input_file.first);
+			d->Add(", ");
+			if ( input_file.second )
+				d->Add(input_file.second->substr(0, 20)); // cut content off
+			else
+				d->Add("<no content>");
+
+			d->Add(")");
+			}
+			break;
 		}
 	}
 
@@ -368,6 +384,13 @@ int Plugin::HookLoadFile(const LoadType type, const std::string& file, const std
 	return -1;
 	}
 
+std::pair<int, std::optional<std::string>> Plugin::HookLoadFileExtended(const LoadType type,
+                                                                        const std::string& file,
+                                                                        const std::string& resolved)
+	{
+	return std::make_pair(-1, std::nullopt);
+	}
+
 std::pair<bool, ValPtr> Plugin::HookFunctionCall(const Func* func, zeek::detail::Frame* parent,
                                                  Args* args)
 	{

src/plugin/Plugin.h

@@ -5,6 +5,7 @@
 #include "zeek/zeek-config.h"
 
 #include <list>
+#include <optional>
 #include <string>
 #include <utility>
 
@@ -57,6 +58,7 @@ enum HookType
 	{
 	// Note: when changing this table, update hook_name() in Plugin.cc.
 	HOOK_LOAD_FILE, //< Activates Plugin::HookLoadFile().
+	HOOK_LOAD_FILE_EXT, //< Activates Plugin::HookLoadFileExtended().
 	HOOK_CALL_FUNCTION, //< Activates Plugin::HookCallFunction().
 	HOOK_QUEUE_EVENT, //< Activates Plugin::HookQueueEvent().
 	HOOK_DRAIN_EVENTS, //< Activates Plugin::HookDrainEvents()
@@ -205,7 +207,8 @@ public:
 		CONN,
 		THREAD_FIELDS,
 		LOCATION,
-		ARG_LIST
+		ARG_LIST,
+		INPUT_FILE
 		};
 
 	/**
@@ -357,6 +360,15 @@ public:
 		arg.args = args;
 		}
 
+	/**
+	 * Constructor with HookLoadFileExtended result describing an input file.
+	 */
+	explicit HookArgument(std::pair<int, std::optional<std::string>> file)
+		{
+		type = INPUT_FILE;
+		input_file = std::move(file);
+		}
+
 	/**
 	 * Returns the value for a boolen argument. The argument's type must
 	 * match accordingly.
@@ -540,6 +552,7 @@ private:
 	std::pair<bool, Val*> func_result;
 	std::pair<int, const threading::Field* const*> tfields;
 	std::string arg_string;
+	std::pair<int, std::optional<std::string>> input_file;
 	};
 
 using HookArgumentList = std::list<HookArgument>;
@@ -815,6 +828,43 @@ protected:
 	virtual int HookLoadFile(const LoadType type, const std::string& file,
 	                         const std::string& resolved);
 
+	/**
+	 * Hook into loading input files, with extended capabilities. This method
+	 * will be called between InitPreScript() and InitPostScript(), but with no
+	 * further order or timing guaranteed. It will be called once for each
+	 * input file Bro is about to load, either given on the command line or via
+	 * @load script directives. The hook can take over the file, in which case
+	 * Bro will not further process it otherwise. It can, alternatively, also
+	 * provide the file content as a string, which Bro will then process just
+	 * as if it had read it from a file.
+	 *
+	 * @param type The type of load encountered: script load, signatures load,
+	 *             or plugin load.
+	 *
+	 * @param file The filename that was passed to @load. Only includes
+	 *             an extension if it was given in @load.
+	 *
+	 * @param resolved The file or directory name Bro resolved from
+	 *                 the given path and is going to load. Empty string
+	 *                 if Bro was not able to resolve a path.
+	 *
+	 * @return tuple of an integer and an optional string, where: the integer
+	 * must be 1 if the plugin takes over loading the file (see below); 0 if
+	 * the plugin wanted to take over the file but had trouble loading it
+	 * (processing will abort in this case, and the plugin should have printed
+	 * an error message); and -1 if the plugin wants Bro to proceeed processing
+	 * the file normally. If the plugins takes over by returning 1, there are
+	 * two cases: if the second tuple element remains unset, the plugin handled
+	 * the loading completely internally; Bro will not do anything further with
+	 * it. Alternatively, the plugin may optionally return the acutal content
+	 * to use for the file as a string through the tuple's second element. If
+	 * so, Bro will ignore the file on disk and use that provided content
+	 * instead (including when there's actually no physical file in place on
+	 * disk at all, and loading would have hence failed otherwise).
+	 */
+	virtual std::pair<int, std::optional<std::string>>
+	HookLoadFileExtended(const LoadType type, const std::string& file, const std::string& resolved);
+
 	/**
 	 * Hook into executing a script-level function/event/hook. Whenever
 	 * the script interpreter is about to execution a function, it first

src/rule-scan.l

@@ -228,3 +228,23 @@ void end_PS()
 	{
 	BEGIN(INITIAL);
 	}
+
+static YY_BUFFER_STATE rules_buffer;
+
+void rules_set_input_from_buffer(const char* data, size_t size)
+	{
+	rules_buffer = yy_scan_bytes(data, size); // this copies the data
+	}
+
+void rules_set_input_from_file(FILE* f)
+	{
+	rules_buffer = yy_create_buffer(f, YY_BUF_SIZE);
+	}
+
+void rules_parse_input()
+	{
+	yy_switch_to_buffer(rules_buffer);
+	rules_parse();
+	yy_delete_buffer(rules_buffer);
+	}
+

src/scan.l

@@ -348,42 +348,19 @@ when	return TOK_WHEN;
 @load-sigs{WS}{FILE} {
 	const char* file = zeek::util::skip_whitespace(yytext + 10);
 	std::string path = find_relative_file(file, ".sig");
-	int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SIGNATURES, file, path), -1);
+	sig_files.emplace_back(file, path);
-
-	switch ( rc ) {
-	case -1:
-		// No plugin in charge of this file.
-		if ( path.empty() )
-			zeek::reporter->Error("failed to find file associated with @load-sigs %s",
-			                      file);
-		else
-			zeek::detail::sig_files.push_back(std::move(path));
-		break;
-
-	case 0:
-		if ( ! zeek::reporter->Errors() )
-			zeek::reporter->Error("Plugin reported error loading signatures %s", file);
-
-		exit(1);
-		break;
-
-	case 1:
-		// A plugin took care of it, just skip.
-		break;
-
-	default:
-		assert(false);
-		break;
-	}
 	}
 
 @load-plugin{WS}{ID} {
 	const char* plugin = zeek::util::skip_whitespace(yytext + 12);
-	int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::PLUGIN, plugin, ""), -1);
+	std::pair<int, std::optional<std::string>> rc;
+	rc.first = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::PLUGIN, plugin, ""), -1);
+	if ( rc.first < 0 )
+		rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE_EXT, HookLoadFileExtended(zeek::plugin::Plugin::PLUGIN, plugin, ""), std::make_pair(-1, std::nullopt));
 
-	switch ( rc ) {
+	switch ( rc.first ) {
 	case -1:
-		// No plugin in charge of this file.
+		// No plugin in charge of this file. (We ignore any returned content.)
 		zeek::plugin_mgr->ActivateDynamicPlugin(plugin);
 		break;
 
@@ -586,12 +563,13 @@ YYLTYPE zeek::detail::GetCurrentLocation()
 static int load_files(const char* orig_file)
 	{
 	std::string file_path = find_relative_script_file(orig_file);
-	int rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SCRIPT, orig_file, file_path), -1);
 
-	if ( rc == 1 )
+	std::pair<int, std::optional<std::string>> rc = {-1, std::nullopt};
-		return 0; // A plugin took care of it, just skip.
+	rc.first = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE, HookLoadFile(zeek::plugin::Plugin::SCRIPT, orig_file, file_path), -1);
+	if ( rc.first < 0 )
+		rc = PLUGIN_HOOK_WITH_RESULT(HOOK_LOAD_FILE_EXT, HookLoadFileExtended(zeek::plugin::Plugin::SCRIPT, orig_file, file_path), std::make_pair(-1, std::nullopt));
 
-	if ( rc == 0 )
+	if ( rc.first == 0 )
 		{
 		if ( ! zeek::reporter->Errors() )
 			// This is just in case the plugin failed to report
@@ -602,55 +580,57 @@ static int load_files(const char* orig_file)
 		exit(1);
 		}
 
-	assert(rc == -1); // No plugin in charge of this file.
+	if ( rc.first == 1 && ! rc.second )
+		return 0; // A plugin took care of it, just skip.
 
 	FILE* f = nullptr;
 
-	if ( zeek::util::streq(orig_file, "-") )
+	if ( rc.first == -1 )
 		{
-		f = stdin;
+		if ( zeek::util::streq(orig_file, "-") )
-		file_path = zeek::detail::ScannedFile::canonical_stdin_path;
-
-		if ( zeek::detail::g_policy_debug )
 			{
-			zeek::detail::debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n");
+			f = stdin;
-			zeek::detail::g_policy_debug = false;
+			file_path = zeek::detail::ScannedFile::canonical_stdin_path;
+
+			if ( zeek::detail::g_policy_debug )
+				{
+				zeek::detail::debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n");
+				zeek::detail::g_policy_debug = false;
+				}
 			}
-		}
 
-	else
-		{
-		if ( file_path.empty() )
-			zeek::reporter->FatalError("can't find %s", orig_file);
-
-		if ( zeek::util::is_dir(file_path.c_str()) )
-			f = zeek::util::detail::open_package(file_path);
 		else
-			f = zeek::util::open_file(file_path);
+			{
+			if ( file_path.empty() )
+				zeek::reporter->FatalError("can't find %s", orig_file);
 
-		if ( ! f )
+			if ( zeek::util::is_dir(file_path.c_str()) )
-			zeek::reporter->FatalError("can't open %s", file_path.c_str());
+				f = zeek::util::detail::open_package(file_path);
+			else
+				f = zeek::util::open_file(file_path);
+
+			if ( ! f )
+				zeek::reporter->FatalError("can't open %s", file_path.c_str());
+			}
+
+		zeek::detail::ScannedFile sf(file_stack.length(), file_path);
+		if ( sf.AlreadyScanned() )
+			{
+			if ( rc.first == -1 && f != stdin )
+				fclose(f);
+
+			return 0;
+			}
+
+		zeek::detail::files_scanned.push_back(std::move(sf));
 		}
 
-	zeek::detail::ScannedFile sf(file_stack.length(), file_path);
-
-	if ( sf.AlreadyScanned() )
-		{
-		if ( f != stdin )
-			fclose(f);
-
-		return 0;
-		}
-
-	zeek::detail::files_scanned.push_back(std::move(sf));
-
 	if ( zeek::detail::g_policy_debug && ! file_path.empty() )
 		{
 		// Add the filename to the file mapping table (Debug.h).
 		zeek::detail::Filemap* map = new zeek::detail::Filemap;
-		zeek::detail::HashKey* key = new zeek::detail::HashKey(file_path.c_str());
 		zeek::detail::g_dbgfilemaps.emplace(file_path, map);
-		LoadPolicyFileText(file_path.c_str());
+		LoadPolicyFileText(file_path.c_str(), rc.second);
 		}
 
 	// Remember where we were to restore the module scope in which
@@ -659,12 +639,24 @@ static int load_files(const char* orig_file)
 
 	zeek::detail::zeekygen_mgr->Script(file_path);
 
-	DBG_LOG(zeek::DBG_SCRIPTS, "Loading %s", file_path.c_str());
-
 	// "orig_file", could be an alias for yytext, which is ephemeral
 	//  and will be zapped after the yy_switch_to_buffer() below.
-	yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE));
+	YY_BUFFER_STATE buffer;
 
+	if ( rc.first == 1 ) {
+		// Parse code provided by plugin.
+		assert(rc.second);
+		DBG_LOG(zeek::DBG_SCRIPTS, "Loading %s from code supplied by plugin ", file_path.c_str());
+		buffer = yy_scan_bytes(rc.second->data(), rc.second->size()); // this copies the data
+		}
+	else {
+		// Parse from file.
+		assert(f);
+		DBG_LOG(zeek::DBG_SCRIPTS, "Loading %s", file_path.c_str());
+		buffer = yy_create_buffer(f, YY_BUF_SIZE);
+		}
+
+	yy_switch_to_buffer(buffer);
 	yylloc.first_line = yylloc.last_line = line_number = 1;
 
 	// Don't delete the old filename - it's pointed to by

src/zeek-setup.cc

@@ -737,7 +737,11 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 			id->SetVal(make_intrusive<StringVal>(*options.pcap_filter));
 			}
 
-		auto all_signature_files = options.signature_files;
+		std::vector<SignatureFile> all_signature_files;
+
+		// Append signature files given on the command line
+		for ( const auto& sf : options.signature_files )
+			all_signature_files.emplace_back(sf);
 
 		// Append signature files defined in "signature_files" script option
 		for ( auto&& sf : get_script_signature_files() )

testing/btest/Baseline/core.parse-only-signature-file-issues/missing-sig-file

@@ -1,2 +1,2 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: Can't open signature file nope
+error: failed to find file associated with @load-sigs nope

testing/btest/Baseline/plugins.plugin-load-file-extended/output

@@ -0,0 +1,7 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+HookLoadExtended/script: file=|xxx| resolved=|./xxx.zeek|
+HookLoadExtended/script: file=|yyy| resolved=||
+HookLoadExtended/signature: file=|abc.sig| resolved=|./abc.sig|
+new zeek_init(): script has been replaced
+new zeek_init(): script has been added
+signature works!

testing/btest/Baseline/signatures.udp-state/reject

@@ -1,3 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
-error: Error in signature (udp-established.sig:5): 'established' is not a valid 'udp-state'
+error: Error in signature (./udp-established.sig:5): 'established' is not a valid 'udp-state'
 

testing/btest/plugins/hooks-plugin/src/Plugin.cc

@@ -15,6 +15,7 @@ using namespace btest::plugin::Demo_Hooks;
 zeek::plugin::Configuration Plugin::Configure()
 	{
 	EnableHook(zeek::plugin::HOOK_LOAD_FILE);
+	EnableHook(zeek::plugin::HOOK_LOAD_FILE_EXT);
 	EnableHook(zeek::plugin::HOOK_CALL_FUNCTION);
 	EnableHook(zeek::plugin::HOOK_QUEUE_EVENT);
 	EnableHook(zeek::plugin::HOOK_DRAIN_EVENTS);
@@ -56,6 +57,13 @@ int Plugin::HookLoadFile(const LoadType type, const std::string& file, const std
 	return -1;
 	}
 
+std::pair<int, std::optional<std::string>> Plugin::HookLoadFileExtended(const LoadType type, const std::string& file, const std::string& resolved)
+	{
+	fprintf(stderr, "%.6f %-15s %s %s\n", zeek::run_state::network_time, "| HookLoadFileExtended",
+		file.c_str(), resolved.c_str());
+	return std::make_pair(-1, std::nullopt);
+	}
+
 std::pair<bool, zeek::ValPtr> Plugin::HookFunctionCall(const zeek::Func* func, zeek::detail::Frame* frame,
                                                        zeek::Args* args)
 	{

testing/btest/plugins/hooks-plugin/src/Plugin.h

@@ -9,6 +9,7 @@ class Plugin : public zeek::plugin::Plugin
 {
 protected:
 	int HookLoadFile(const LoadType type, const std::string& file, const std::string& resolved) override;
+	std::pair<int, std::optional<std::string>> HookLoadFileExtended(const LoadType type, const std::string& file, const std::string& resolved) override;
 	std::pair<bool, zeek::ValPtr> HookFunctionCall(const zeek::Func* func, zeek::detail::Frame* parent,
 	                                               zeek::Args* args) override;
 	bool HookQueueEvent(zeek::Event* event) override;

testing/btest/plugins/hooks.zeek

@@ -1,8 +1,19 @@
 # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks
 # @TEST-EXEC: cp -r %DIR/hooks-plugin/* .
 # @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
-# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq  >output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT s1.sig 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq  >output
 # @TEST-EXEC: btest-diff output
 
 @unload base/misc/version
 @load base/init-default
+
+@load-sigs s2
+
+@TEST-START-FILE s1.sig
+# Just empty.
+@TEST-END-FILE
+
+@TEST-START-FILE s2.sig
+# Just empty.
+@TEST-END-FILE
+

testing/btest/plugins/plugin-load-file-extended.zeek

@@ -0,0 +1,17 @@
+# @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Testing LoadFileExtended
+# @TEST-EXEC: cp -r %DIR/plugin-load-file-extended/* .
+# @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
+# @TEST-EXEC: ZEEK_PLUGIN_PATH=$(pwd) zeek -r $TRACES/wikipedia.trace -b Testing::LoadFileExtended xxx yyy -s abc.sig >> output
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE xxx.zeek
+
+event zeek_init() {
+    print "original script";
+}
+
+# @TEST-END-FILE
+
+# @TEST-START-FILE abc.sig
+# empty
+# @TEST-END-FILE

testing/btest/plugins/plugin-load-file-extended/src/Plugin.cc

@@ -0,0 +1,70 @@
+
+#include "Plugin.h"
+
+namespace btest::plugin::Testing_LoadFileExtended
+	{
+Plugin plugin;
+	}
+
+using namespace btest::plugin::Testing_LoadFileExtended;
+
+zeek::plugin::Configuration Plugin::Configure()
+	{
+	EnableHook(zeek::plugin::HOOK_LOAD_FILE_EXT);
+
+	zeek::plugin::Configuration config;
+	config.name = "Testing::LoadFileExtended";
+	config.version.major = 0;
+	config.version.minor = 1;
+	config.version.patch = 4;
+	return config;
+	}
+
+#include <iostream>
+
+std::pair<int, std::optional<std::string>> Plugin::HookLoadFileExtended(const LoadType type,
+                                                                        const std::string& file,
+                                                                        const std::string& resolved)
+	{
+	if ( type == LoadType::SCRIPT && file == "xxx" )
+		{
+		printf("HookLoadExtended/script: file=|%s| resolved=|%s|\n", file.c_str(), resolved.c_str());
+
+		return std::make_pair(1, R"(
+			event zeek_init() {
+				print "new zeek_init(): script has been replaced";
+			}
+
+			event signature_match(state: signature_state, msg: string, data: string) {
+				print msg;
+			}
+		)");
+		}
+
+	if ( type == LoadType::SCRIPT && file == "yyy" )
+		{
+		printf("HookLoadExtended/script: file=|%s| resolved=|%s|\n", file.c_str(), resolved.c_str());
+
+		return std::make_pair(1, R"(
+			event zeek_init() {
+				print "new zeek_init(): script has been added";
+			}
+		)");
+		}
+
+	if ( type == LoadType::SIGNATURES && file == "abc.sig" )
+		{
+		printf("HookLoadExtended/signature: file=|%s| resolved=|%s|\n", file.c_str(), resolved.c_str());
+
+		return std::make_pair(1, R"(
+		signature my-sig {
+			ip-proto == tcp
+			payload /GET \/images/
+			event "signature works!"
+			}
+		)");
+		}
+
+	return std::make_pair(-1, std::nullopt);
+	}
+

testing/btest/plugins/plugin-load-file-extended/src/Plugin.h

@@ -0,0 +1,18 @@
+
+#pragma once
+
+#include <zeek/plugin/Plugin.h>
+
+namespace btest::plugin::Testing_LoadFileExtended {
+
+class Plugin : public zeek::plugin::Plugin
+{
+protected:
+	// Overridden from zeek::plugin::Plugin.
+	zeek::plugin::Configuration Configure() override;
+    std::pair<int, std::optional<std::string>> HookLoadFileExtended(const Plugin::LoadType type, const std::string& file, const std::string& resolved) override;
+};
+
+extern Plugin plugin;
+
+}