Reimplementation of the @prefixes statement.

Any added prefixes are now used *after* all input files have been
parsed to look for a prefixed, flattened version of the input file
somewhere in BROPATH and, if found, load it.

For example, if "lcl" is in @prefixes, and site.bro is loaded, then
a file named "lcl.site.bro" that's in BROPATH would end up being
automatically loaded as well.  Packages work similarly, e.g. loading
"protocols/http" means a file named "lcl.protocols.http.bro" in BROPATH
gets loaded automatically.

src/Net.h

@@ -96,9 +96,12 @@ struct ScannedFile {
     ino_t inode;
     int include_level;
     string name;
+    string subpath;        // path in BROPATH's policy/ containing the file
+    bool skipped;          // This ScannedFile was @unload'd
+    bool prefixes_checked; // if loading prefixes for this file has been tried
 
-    ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name)
-        : inode(arg_inode), include_level(arg_include_level), name(arg_name)
+    ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name, string arg_subpath = "", bool arg_skipped = false, bool arg_prefixes_checked = false)
+        : inode(arg_inode), include_level(arg_include_level), name(arg_name), subpath(arg_subpath), skipped(arg_skipped), prefixes_checked(arg_prefixes_checked)
 	{ }
 };
 

src/scan.l

@@ -119,7 +119,7 @@ static PList(FileInfo) file_stack;
 	}
 
 // Returns true if the file is new, false if it's already been scanned.
-static int load_files_with_prefix(const char* file);
+static int load_files(const char* file);
 
 // ### TODO: columns too - use yyless with '.' action?
 %}
@@ -329,7 +329,7 @@ when	return TOK_WHEN;
 			clear_reST_doc_comments();
 			}
 		}
-	(void) load_files_with_prefix(new_file);
+	(void) load_files(new_file);
 	}
 
 @unload{WS}{FILE}	{
@@ -342,7 +342,7 @@ when	return TOK_WHEN;
 
 	if ( f )
 		{
-		ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename);
+		ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename, "", true);
 		files_scanned.push_back(sf);
 
 		fclose(f);
@@ -515,19 +515,12 @@ YYLTYPE GetCurrentLocation()
 	return currloc;
 	}
 
-static int load_files_with_prefix(const char* orig_file)
+static int load_files(const char* orig_file)
 	{
 	// Whether we pushed on a FileInfo that will restore the
 	// current module after the final file has been scanned. 
 	bool did_module_restore = false;
 
-	// Note, we need to loop through the prefixes backwards, since
-	// we push them onto a stack, with the last one we push on the
-	// stack being the first one we will scan.
-	for ( int i = prefixes.length() - 1; i >= 0; --i )
-		{
-		const char* prefix = prefixes[i];
-
 	const char* full_filename = "<internal error>";
 	const char* bropath_subpath = "<internal error>";
 	const char* bropath_subpath_delete = 0;
@@ -547,17 +540,8 @@ static int load_files_with_prefix(const char* orig_file)
 
 	else
 		{
-			int n = strlen(prefix) + strlen(orig_file) + 2;
-			char* new_filename = new char[n];
-
-			if ( prefix[0] )
-				sprintf(new_filename, "%s.%s", prefix, orig_file);
-			else
-				strcpy(new_filename, orig_file);
-
-			f = search_for_file(new_filename, "bro", &full_filename, true, &bropath_subpath);
+		f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath);
 		bropath_subpath_delete = bropath_subpath; // This will be deleted.
-			delete [] new_filename;
 		}
 
 	if ( f )
@@ -576,7 +560,7 @@ static int load_files_with_prefix(const char* orig_file)
 				}
 			}
 
-			ScannedFile sf(i, file_stack.length(), full_filename);
+		ScannedFile sf(i, file_stack.length(), full_filename, bropath_subpath);
 		files_scanned.push_back(sf);
 
 		if ( g_policy_debug )
@@ -592,7 +576,7 @@ static int load_files_with_prefix(const char* orig_file)
 				// reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename);
 				fclose(f);
 				delete key;
-					continue;
+				return 0;
 				}
 			else
 				{
@@ -640,14 +624,10 @@ static int load_files_with_prefix(const char* orig_file)
 		}
 
 	else
-			{
-			if ( streq(prefixes[i], "") )
 		{
 		reporter->Error("can't open %s", full_filename);
 		exit(1);
 		}
-			}
-		}
 
 	return 1;
 	}
@@ -758,7 +738,7 @@ void add_input_file(const char* file)
 		reporter->InternalError("empty filename");
 
 	if ( ! filename )
-		(void) load_files_with_prefix(file);
+		(void) load_files(file);
 	else
 		input_files.append(copy_string(file));
 	}
@@ -809,7 +789,7 @@ int yywrap()
 		check_capture_filter_changes();
 		check_dpd_config_changes();
 
-		if ( load_files_with_prefix(input_files[0]) )
+		if ( load_files(input_files[0]) )
 			{
 			// Don't delete the filename - it's pointed to by
 			// every BroObj created when parsing it.
@@ -825,6 +805,44 @@ int yywrap()
 	check_capture_filter_changes();
 	check_dpd_config_changes();
 
+	// For each file scanned so far, and for each @prefix, look for
+	// a prefixed and flattened version of the loaded file in BROPATH.
+	// The flattening involves taking the path in BROPATH in which
+	// the scanned file lives and replacing '/' path separators with a '.'
+	// If the scanned file is "__load__.bro", that part of the flattened file
+	// name is discarded.
+	// If the prefix is non-empty, it gets placed in front of the flattened
+	// path, separated with another '.'
+	std::list<ScannedFile>::iterator it;
+	bool found_prefixed_files = false;
+	for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
+		{
+		if ( it->skipped || it->prefixes_checked ) continue;
+		it->prefixes_checked = true;
+		// prefixes are pushed onto a stack, so iterate backwards
+		for ( int i = prefixes.length() - 1; i >= 0; --i )
+			{
+			// don't look at empty prefixes
+			if ( ! prefixes[i][0] ) continue;
+			string s;
+			s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]);
+			FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0);
+			//printf("====== prefix search ======\n");
+			//printf("File  : %s\n", it->name.c_str());
+			//printf("Path  : %s\n", it->subpath.c_str());
+			//printf("Dotted: %s\n", s.c_str());
+			//printf("Found : %s\n", f ? "T" : "F");
+			//printf("===========================\n");
+			if ( f )
+				{
+				add_input_file(s.c_str());
+				found_prefixed_files = true;
+				fclose(f);
+				}
+			}
+		}
+	if ( found_prefixed_files ) return 0;
+
 	// Add redef statements for any X=Y command line parameters.
 	if ( params.size() > 0 )
 		{

src/util.cc

@@ -15,6 +15,8 @@
 # endif
 #endif
 
+#include <string>
+#include <vector>
 #include <ctype.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -815,6 +817,79 @@ FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs
 	return f;
 	}
 
+// Canonicalizes a given 'file' that lives in 'path' into a flattened,
+// dotted format.  If the optional 'prefix' argument is given, it is
+// prepended to the dotted-format, separated by another dot.
+// If 'file' is __load__.bro, that part is discarded when constructing
+// the final dotted-format.
+string dot_canon(string path, string file, string prefix)
+	{
+	string dottedform(prefix);
+	if ( prefix != "" )
+		dottedform.append(".");
+	dottedform.append(path);
+	char* tmp = copy_string(file.c_str());
+	char* bname = basename(tmp);
+	if ( ! streq(bname, PACKAGE_LOADER) )
+		{
+		if ( path != "" )
+			dottedform.append(".");
+		dottedform.append(bname);
+		}
+	delete [] tmp;
+	size_t n;
+	while ( (n = dottedform.find("/")) != string::npos ) 
+		dottedform.replace(n, 1, ".");
+	return dottedform;
+	}
+
+// returns a normalized version of a path, removing duplicate slashes,
+// extraneous dots that refer to the current directory, and pops as many
+// parent directories referred to by "../" as possible
+const char* normalize_path(const char* path)
+	{
+	size_t n;
+	string p(path);
+	vector<string> components, final_components;
+	string new_path;
+
+	if ( p[0] == '/' )
+		new_path = "/";
+
+	while ( (n = p.find("/")) != string::npos )
+		{
+		components.push_back(p.substr(0, n));
+		p.erase(0, n + 1);
+		}
+	components.push_back(p);
+
+	vector<string>::const_iterator it;
+	for ( it = components.begin(); it != components.end(); ++it )
+		{
+		if ( *it == "" ) continue;
+		final_components.push_back(*it);
+
+		if ( *it == "." && it != components.begin() )
+			final_components.pop_back();
+		else if ( *it == ".." && final_components[0] != ".." )
+			{
+			final_components.pop_back();
+			final_components.pop_back();
+			}
+		}
+
+	for ( it = final_components.begin(); it != final_components.end(); ++it )
+		{
+		new_path.append(*it);
+		new_path.append("/");
+		}
+
+	if ( new_path.size() > 1 && new_path[new_path.size() - 1] == '/' )
+		new_path.erase(new_path.size() - 1);
+
+	return copy_string(new_path.c_str());
+	}
+
 // Returns the subpath of BROPATH's policy/ directory in which the loaded
 // file in located.  If it's not under a subpath of policy/ then the full
 // path is returned, else the subpath of policy/ concatentated with any
@@ -865,7 +940,10 @@ void get_policy_subpath(const char* dir, const char* file, const char** subpath)
 		*subpath = full_subpath;
 		}
 
+	const char* normalized_subpath = normalize_path(*subpath);
 	delete [] tmp;
+	delete [] *subpath;
+	*subpath = normalized_subpath;
 	}
 
 extern string current_scanned_file_path;

src/util.h

@@ -5,6 +5,7 @@
 #ifndef util_h
 #define util_h
 
+#include <string>
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -176,6 +177,8 @@ extern int int_list_cmp(const void* v1, const void* v2);
 
 extern const char* bro_path();
 extern const char* bro_prefixes();
+std::string dot_canon(std::string path, std::string file, std::string prefix = "");
+const char* normalize_path(const char* path);
 void get_policy_subpath(const char* dir, const char* file, const char** subpath);
 extern FILE* search_for_file(const char* filename, const char* ext,
 	const char** full_filename, bool load_pkgs, const char** bropath_subpath);

testing/btest/Baseline/core.load-prefixes/output

@@ -0,0 +1,4 @@
+loaded lcl2.site.bro
+loaded lcl.site.bro
+loaded lcl2.protocols.http.bro
+loaded lcl.protocols.http.bro

testing/btest/core/load-prefixes.bro

@@ -0,0 +1,25 @@
+# A test of prefix-based @load'ing
+
+# @TEST-EXEC: bro addprefixes site protocols/http >output
+# @TEST-EXEC: btest-diff output
+
+@TEST-START-FILE addprefixes.bro
+@prefixes += lcl
+@prefixes += lcl2
+@TEST-END-FILE
+
+@TEST-START-FILE lcl.site.bro
+print "loaded lcl.site.bro";
+@TEST-END-FILE
+
+@TEST-START-FILE lcl2.site.bro
+print "loaded lcl2.site.bro";
+@TEST-END-FILE
+
+@TEST-START-FILE lcl.protocols.http.bro
+print "loaded lcl.protocols.http.bro";
+@TEST-END-FILE
+
+@TEST-START-FILE lcl2.protocols.http.bro
+print "loaded lcl2.protocols.http.bro";
+@TEST-END-FILE