Reimplementation of the @prefixes statement.

Any added prefixes are now used *after* all input files have been
parsed to look for a prefixed, flattened version of the input file
somewhere in BROPATH and, if found, load it.

For example, if "lcl" is in @prefixes, and site.bro is loaded, then
a file named "lcl.site.bro" that's in BROPATH would end up being
automatically loaded as well.  Packages work similarly, e.g. loading
"protocols/http" means a file named "lcl.protocols.http.bro" in BROPATH
gets loaded automatically.

src/Net.h

@@ -96,9 +96,12 @@ struct ScannedFile {
     ino_t inode;
     int include_level;
     string name;
+    string subpath;        // path in BROPATH's policy/ containing the file
+    bool skipped;          // This ScannedFile was @unload'd
+    bool prefixes_checked; // if loading prefixes for this file has been tried
 
-    ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name)
+    ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name, string arg_subpath = "", bool arg_skipped = false, bool arg_prefixes_checked = false)
-        : inode(arg_inode), include_level(arg_include_level), name(arg_name)
+        : inode(arg_inode), include_level(arg_include_level), name(arg_name), subpath(arg_subpath), skipped(arg_skipped), prefixes_checked(arg_prefixes_checked)
 	{ }
 };
 

src/scan.l

@@ -119,7 +119,7 @@ static PList(FileInfo) file_stack;
 	}
 
 // Returns true if the file is new, false if it's already been scanned.
-static int load_files_with_prefix(const char* file);
+static int load_files(const char* file);
 
 // ### TODO: columns too - use yyless with '.' action?
 %}
@@ -329,7 +329,7 @@ when	return TOK_WHEN;
 			clear_reST_doc_comments();
 			}
 		}
-	(void) load_files_with_prefix(new_file);
+	(void) load_files(new_file);
 	}
 
 @unload{WS}{FILE}	{
@@ -342,7 +342,7 @@ when	return TOK_WHEN;
 
 	if ( f )
 		{
-		ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename);
+		ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename, "", true);
 		files_scanned.push_back(sf);
 
 		fclose(f);
@@ -515,138 +515,118 @@ YYLTYPE GetCurrentLocation()
 	return currloc;
 	}
 
-static int load_files_with_prefix(const char* orig_file)
+static int load_files(const char* orig_file)
 	{
 	// Whether we pushed on a FileInfo that will restore the
 	// current module after the final file has been scanned. 
 	bool did_module_restore = false;
 
-	// Note, we need to loop through the prefixes backwards, since
+	const char* full_filename = "<internal error>";
-	// we push them onto a stack, with the last one we push on the
+	const char* bropath_subpath = "<internal error>";
-	// stack being the first one we will scan.
+	const char* bropath_subpath_delete = 0;
-	for ( int i = prefixes.length() - 1; i >= 0; --i )
+	FILE* f;
+
+	if ( streq(orig_file, "-") )
 		{
-		const char* prefix = prefixes[i];
+		f = stdin;
+		full_filename = "<stdin>";
 
-		const char* full_filename = "<internal error>";
+		if ( g_policy_debug )
-		const char* bropath_subpath = "<internal error>";
-		const char* bropath_subpath_delete = 0;
-		FILE* f;
-
-		if ( streq(orig_file, "-") )
 			{
-			f = stdin;
+			debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n");
-			full_filename = "<stdin>";
+			g_policy_debug = false;
+			}
+		}
 
-			if ( g_policy_debug )
+	else
+		{
+		f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath);
+		bropath_subpath_delete = bropath_subpath; // This will be deleted.
+		}
+
+	if ( f )
+		{
+		ino_t i = get_inode_num(f, full_filename);
+		std::list<ScannedFile>::const_iterator it;
+
+		for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
+			{
+			if ( it->inode == i )
 				{
-				debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n");
+				fclose(f);
-				g_policy_debug = false;
+				delete [] full_filename;
+				delete [] bropath_subpath_delete;
+				return 0;
 				}
 			}
 
-		else
+		ScannedFile sf(i, file_stack.length(), full_filename, bropath_subpath);
+		files_scanned.push_back(sf);
+
+		if ( g_policy_debug )
 			{
-			int n = strlen(prefix) + strlen(orig_file) + 2;
+			// Add the filename to the file mapping
-			char* new_filename = new char[n];
+			// table (Debug.h).
+			Filemap* map = new Filemap;
 
-			if ( prefix[0] )
+			// Make sure it wasn't already read in.
-				sprintf(new_filename, "%s.%s", prefix, orig_file);
+			HashKey* key = new HashKey(full_filename);
-			else
+			if ( g_dbgfilemaps.Lookup(key) )
-				strcpy(new_filename, orig_file);
-
-			f = search_for_file(new_filename, "bro", &full_filename, true, &bropath_subpath);
-			bropath_subpath_delete = bropath_subpath; // This will be deleted.
-			delete [] new_filename;
-			}
-
-		if ( f )
-			{
-			ino_t i = get_inode_num(f, full_filename);
-			std::list<ScannedFile>::const_iterator it;
-
-			for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
 				{
-				if ( it->inode == i )
+				// reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename);
-					{
+				fclose(f);
-					fclose(f);
+				delete key;
-					delete [] full_filename;
+				return 0;
-					delete [] bropath_subpath_delete;
-					return 0;
-					}
-				}
-
-			ScannedFile sf(i, file_stack.length(), full_filename);
-			files_scanned.push_back(sf);
-
-			if ( g_policy_debug )
-				{
-				// Add the filename to the file mapping
-				// table (Debug.h).
-				Filemap* map = new Filemap;
-
-				// Make sure it wasn't already read in.
-				HashKey* key = new HashKey(full_filename);
-				if ( g_dbgfilemaps.Lookup(key) )
-					{
-					// reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename);
-					fclose(f);
-					delete key;
-					continue;
-					}
-				else
-					{
-					g_dbgfilemaps.Insert(key, map);
-					}
-
-				if ( full_filename )
-					LoadPolicyFileText(full_filename);
-				}
-
-			// Remember where we were.  If this is the first
-			// file being pushed on the stack, i.e., the *last*
-			// one that will be processed, then we want to
-			// restore the module scope in which this @load
-			// was done when we're finished processing it.
-			if ( ! did_module_restore )
-				{
-				file_stack.append(new FileInfo(current_module));
-				did_module_restore = true;
 				}
 			else
-				file_stack.append(new FileInfo);
-
-			char* tmp = copy_string(full_filename);
-			current_scanned_file_path = dirname(tmp);
-			delete [] tmp;
-
-			if ( generate_documentation )
 				{
-				current_reST_doc = new BroDoc(bropath_subpath, full_filename);
+				g_dbgfilemaps.Insert(key, map);
-				docs_generated.push_back(current_reST_doc);
 				}
 
-			delete [] bropath_subpath_delete;
+			if ( full_filename )
-
+				LoadPolicyFileText(full_filename);
-			// "orig_file", could be an alias for yytext, which is ephemeral
-			//  and will be zapped after the yy_switch_to_buffer() below.
-			yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE));
-
-			yylloc.first_line = yylloc.last_line = line_number = 1;
-
-			// Don't delete the old filename - it's pointed to by
-			// every BroObj created when parsing it.
-			yylloc.filename = filename = full_filename;
 			}
 
-		else
+		// Remember where we were.  If this is the first
+		// file being pushed on the stack, i.e., the *last*
+		// one that will be processed, then we want to
+		// restore the module scope in which this @load
+		// was done when we're finished processing it.
+		if ( ! did_module_restore )
 			{
-			if ( streq(prefixes[i], "") )
+			file_stack.append(new FileInfo(current_module));
-				{
+			did_module_restore = true;
-				reporter->Error("can't open %s", full_filename);
-				exit(1);
-				}
 			}
+		else
+			file_stack.append(new FileInfo);
+
+		char* tmp = copy_string(full_filename);
+		current_scanned_file_path = dirname(tmp);
+		delete [] tmp;
+
+		if ( generate_documentation )
+			{
+			current_reST_doc = new BroDoc(bropath_subpath, full_filename);
+			docs_generated.push_back(current_reST_doc);
+			}
+
+		delete [] bropath_subpath_delete;
+
+		// "orig_file", could be an alias for yytext, which is ephemeral
+		//  and will be zapped after the yy_switch_to_buffer() below.
+		yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE));
+
+		yylloc.first_line = yylloc.last_line = line_number = 1;
+
+		// Don't delete the old filename - it's pointed to by
+		// every BroObj created when parsing it.
+		yylloc.filename = filename = full_filename;
+		}
+
+	else
+		{
+		reporter->Error("can't open %s", full_filename);
+		exit(1);
 		}
 
 	return 1;
@@ -758,7 +738,7 @@ void add_input_file(const char* file)
 		reporter->InternalError("empty filename");
 
 	if ( ! filename )
-		(void) load_files_with_prefix(file);
+		(void) load_files(file);
 	else
 		input_files.append(copy_string(file));
 	}
@@ -809,7 +789,7 @@ int yywrap()
 		check_capture_filter_changes();
 		check_dpd_config_changes();
 
-		if ( load_files_with_prefix(input_files[0]) )
+		if ( load_files(input_files[0]) )
 			{
 			// Don't delete the filename - it's pointed to by
 			// every BroObj created when parsing it.
@@ -825,6 +805,44 @@ int yywrap()
 	check_capture_filter_changes();
 	check_dpd_config_changes();
 
+	// For each file scanned so far, and for each @prefix, look for
+	// a prefixed and flattened version of the loaded file in BROPATH.
+	// The flattening involves taking the path in BROPATH in which
+	// the scanned file lives and replacing '/' path separators with a '.'
+	// If the scanned file is "__load__.bro", that part of the flattened file
+	// name is discarded.
+	// If the prefix is non-empty, it gets placed in front of the flattened
+	// path, separated with another '.'
+	std::list<ScannedFile>::iterator it;
+	bool found_prefixed_files = false;
+	for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
+		{
+		if ( it->skipped || it->prefixes_checked ) continue;
+		it->prefixes_checked = true;
+		// prefixes are pushed onto a stack, so iterate backwards
+		for ( int i = prefixes.length() - 1; i >= 0; --i )
+			{
+			// don't look at empty prefixes
+			if ( ! prefixes[i][0] ) continue;
+			string s;
+			s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]);
+			FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0);
+			//printf("====== prefix search ======\n");
+			//printf("File  : %s\n", it->name.c_str());
+			//printf("Path  : %s\n", it->subpath.c_str());
+			//printf("Dotted: %s\n", s.c_str());
+			//printf("Found : %s\n", f ? "T" : "F");
+			//printf("===========================\n");
+			if ( f )
+				{
+				add_input_file(s.c_str());
+				found_prefixed_files = true;
+				fclose(f);
+				}
+			}
+		}
+	if ( found_prefixed_files ) return 0;
+
 	// Add redef statements for any X=Y command line parameters.
 	if ( params.size() > 0 )
 		{

src/util.cc

@@ -15,6 +15,8 @@
 # endif
 #endif
 
+#include <string>
+#include <vector>
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -815,6 +817,79 @@ FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs
 	return f;
 	}
 
+// Canonicalizes a given 'file' that lives in 'path' into a flattened,
+// dotted format.  If the optional 'prefix' argument is given, it is
+// prepended to the dotted-format, separated by another dot.
+// If 'file' is __load__.bro, that part is discarded when constructing
+// the final dotted-format.
+string dot_canon(string path, string file, string prefix)
+	{
+	string dottedform(prefix);
+	if ( prefix != "" )
+		dottedform.append(".");
+	dottedform.append(path);
+	char* tmp = copy_string(file.c_str());
+	char* bname = basename(tmp);
+	if ( ! streq(bname, PACKAGE_LOADER) )
+		{
+		if ( path != "" )
+			dottedform.append(".");
+		dottedform.append(bname);
+		}
+	delete [] tmp;
+	size_t n;
+	while ( (n = dottedform.find("/")) != string::npos ) 
+		dottedform.replace(n, 1, ".");
+	return dottedform;
+	}
+
+// returns a normalized version of a path, removing duplicate slashes,
+// extraneous dots that refer to the current directory, and pops as many
+// parent directories referred to by "../" as possible
+const char* normalize_path(const char* path)
+	{
+	size_t n;
+	string p(path);
+	vector<string> components, final_components;
+	string new_path;
+
+	if ( p[0] == '/' )
+		new_path = "/";
+
+	while ( (n = p.find("/")) != string::npos )
+		{
+		components.push_back(p.substr(0, n));
+		p.erase(0, n + 1);
+		}
+	components.push_back(p);
+
+	vector<string>::const_iterator it;
+	for ( it = components.begin(); it != components.end(); ++it )
+		{
+		if ( *it == "" ) continue;
+		final_components.push_back(*it);
+
+		if ( *it == "." && it != components.begin() )
+			final_components.pop_back();
+		else if ( *it == ".." && final_components[0] != ".." )
+			{
+			final_components.pop_back();
+			final_components.pop_back();
+			}
+		}
+
+	for ( it = final_components.begin(); it != final_components.end(); ++it )
+		{
+		new_path.append(*it);
+		new_path.append("/");
+		}
+
+	if ( new_path.size() > 1 && new_path[new_path.size() - 1] == '/' )
+		new_path.erase(new_path.size() - 1);
+
+	return copy_string(new_path.c_str());
+	}
+
 // Returns the subpath of BROPATH's policy/ directory in which the loaded
 // file in located.  If it's not under a subpath of policy/ then the full
 // path is returned, else the subpath of policy/ concatentated with any
@@ -865,7 +940,10 @@ void get_policy_subpath(const char* dir, const char* file, const char** subpath)
 		*subpath = full_subpath;
 		}
 
+	const char* normalized_subpath = normalize_path(*subpath);
 	delete [] tmp;
+	delete [] *subpath;
+	*subpath = normalized_subpath;
 	}
 
 extern string current_scanned_file_path;

src/util.h

@@ -5,6 +5,7 @@
 #ifndef util_h
 #define util_h
 
+#include <string>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -176,6 +177,8 @@ extern int int_list_cmp(const void* v1, const void* v2);
 
 extern const char* bro_path();
 extern const char* bro_prefixes();
+std::string dot_canon(std::string path, std::string file, std::string prefix = "");
+const char* normalize_path(const char* path);
 void get_policy_subpath(const char* dir, const char* file, const char** subpath);
 extern FILE* search_for_file(const char* filename, const char* ext,
 	const char** full_filename, bool load_pkgs, const char** bropath_subpath);

testing/btest/Baseline/core.load-prefixes/output

@@ -0,0 +1,4 @@
+loaded lcl2.site.bro
+loaded lcl.site.bro
+loaded lcl2.protocols.http.bro
+loaded lcl.protocols.http.bro

testing/btest/core/load-prefixes.bro

@@ -0,0 +1,25 @@
+# A test of prefix-based @load'ing
+
+# @TEST-EXEC: bro addprefixes site protocols/http >output
+# @TEST-EXEC: btest-diff output
+
+@TEST-START-FILE addprefixes.bro
+@prefixes += lcl
+@prefixes += lcl2
+@TEST-END-FILE
+
+@TEST-START-FILE lcl.site.bro
+print "loaded lcl.site.bro";
+@TEST-END-FILE
+
+@TEST-START-FILE lcl2.site.bro
+print "loaded lcl2.site.bro";
+@TEST-END-FILE
+
+@TEST-START-FILE lcl.protocols.http.bro
+print "loaded lcl.protocols.http.bro";
+@TEST-END-FILE
+
+@TEST-START-FILE lcl2.protocols.http.bro
+print "loaded lcl2.protocols.http.bro";
+@TEST-END-FILE