diff --git a/src/Net.h b/src/Net.h index 3544dd7f3d..5a27fd434a 100644 --- a/src/Net.h +++ b/src/Net.h @@ -96,9 +96,12 @@ struct ScannedFile { ino_t inode; int include_level; string name; + string subpath; // path in BROPATH's policy/ containing the file + bool skipped; // This ScannedFile was @unload'd + bool prefixes_checked; // if loading prefixes for this file has been tried - ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name) - : inode(arg_inode), include_level(arg_include_level), name(arg_name) + ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name, string arg_subpath = "", bool arg_skipped = false, bool arg_prefixes_checked = false) + : inode(arg_inode), include_level(arg_include_level), name(arg_name), subpath(arg_subpath), skipped(arg_skipped), prefixes_checked(arg_prefixes_checked) { } }; diff --git a/src/scan.l b/src/scan.l index 29105eae7e..484d54cbfe 100644 --- a/src/scan.l +++ b/src/scan.l @@ -119,7 +119,7 @@ static PList(FileInfo) file_stack; } // Returns true if the file is new, false if it's already been scanned. -static int load_files_with_prefix(const char* file); +static int load_files(const char* file); // ### TODO: columns too - use yyless with '.' action? %} @@ -329,7 +329,7 @@ when return TOK_WHEN; clear_reST_doc_comments(); } } - (void) load_files_with_prefix(new_file); + (void) load_files(new_file); } @unload{WS}{FILE} { @@ -342,7 +342,7 @@ when return TOK_WHEN; if ( f ) { - ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename); + ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename, "", true); files_scanned.push_back(sf); fclose(f); @@ -515,138 +515,118 @@ YYLTYPE GetCurrentLocation() return currloc; } -static int load_files_with_prefix(const char* orig_file) +static int load_files(const char* orig_file) { // Whether we pushed on a FileInfo that will restore the // current module after the final file has been scanned. bool did_module_restore = false; - // Note, we need to loop through the prefixes backwards, since - // we push them onto a stack, with the last one we push on the - // stack being the first one we will scan. - for ( int i = prefixes.length() - 1; i >= 0; --i ) + const char* full_filename = ""; + const char* bropath_subpath = ""; + const char* bropath_subpath_delete = 0; + FILE* f; + + if ( streq(orig_file, "-") ) { - const char* prefix = prefixes[i]; + f = stdin; + full_filename = ""; - const char* full_filename = ""; - const char* bropath_subpath = ""; - const char* bropath_subpath_delete = 0; - FILE* f; - - if ( streq(orig_file, "-") ) + if ( g_policy_debug ) { - f = stdin; - full_filename = ""; + debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n"); + g_policy_debug = false; + } + } - if ( g_policy_debug ) + else + { + f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath); + bropath_subpath_delete = bropath_subpath; // This will be deleted. + } + + if ( f ) + { + ino_t i = get_inode_num(f, full_filename); + std::list::const_iterator it; + + for ( it = files_scanned.begin(); it != files_scanned.end(); ++it ) + { + if ( it->inode == i ) { - debug_msg("Warning: can't use debugger while reading policy from stdin; turning off debugging.\n"); - g_policy_debug = false; + fclose(f); + delete [] full_filename; + delete [] bropath_subpath_delete; + return 0; } } - else + ScannedFile sf(i, file_stack.length(), full_filename, bropath_subpath); + files_scanned.push_back(sf); + + if ( g_policy_debug ) { - int n = strlen(prefix) + strlen(orig_file) + 2; - char* new_filename = new char[n]; + // Add the filename to the file mapping + // table (Debug.h). + Filemap* map = new Filemap; - if ( prefix[0] ) - sprintf(new_filename, "%s.%s", prefix, orig_file); - else - strcpy(new_filename, orig_file); - - f = search_for_file(new_filename, "bro", &full_filename, true, &bropath_subpath); - bropath_subpath_delete = bropath_subpath; // This will be deleted. - delete [] new_filename; - } - - if ( f ) - { - ino_t i = get_inode_num(f, full_filename); - std::list::const_iterator it; - - for ( it = files_scanned.begin(); it != files_scanned.end(); ++it ) + // Make sure it wasn't already read in. + HashKey* key = new HashKey(full_filename); + if ( g_dbgfilemaps.Lookup(key) ) { - if ( it->inode == i ) - { - fclose(f); - delete [] full_filename; - delete [] bropath_subpath_delete; - return 0; - } - } - - ScannedFile sf(i, file_stack.length(), full_filename); - files_scanned.push_back(sf); - - if ( g_policy_debug ) - { - // Add the filename to the file mapping - // table (Debug.h). - Filemap* map = new Filemap; - - // Make sure it wasn't already read in. - HashKey* key = new HashKey(full_filename); - if ( g_dbgfilemaps.Lookup(key) ) - { - // reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename); - fclose(f); - delete key; - continue; - } - else - { - g_dbgfilemaps.Insert(key, map); - } - - if ( full_filename ) - LoadPolicyFileText(full_filename); - } - - // Remember where we were. If this is the first - // file being pushed on the stack, i.e., the *last* - // one that will be processed, then we want to - // restore the module scope in which this @load - // was done when we're finished processing it. - if ( ! did_module_restore ) - { - file_stack.append(new FileInfo(current_module)); - did_module_restore = true; + // reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename); + fclose(f); + delete key; + return 0; } else - file_stack.append(new FileInfo); - - char* tmp = copy_string(full_filename); - current_scanned_file_path = dirname(tmp); - delete [] tmp; - - if ( generate_documentation ) { - current_reST_doc = new BroDoc(bropath_subpath, full_filename); - docs_generated.push_back(current_reST_doc); + g_dbgfilemaps.Insert(key, map); } - delete [] bropath_subpath_delete; - - // "orig_file", could be an alias for yytext, which is ephemeral - // and will be zapped after the yy_switch_to_buffer() below. - yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE)); - - yylloc.first_line = yylloc.last_line = line_number = 1; - - // Don't delete the old filename - it's pointed to by - // every BroObj created when parsing it. - yylloc.filename = filename = full_filename; + if ( full_filename ) + LoadPolicyFileText(full_filename); } - else + // Remember where we were. If this is the first + // file being pushed on the stack, i.e., the *last* + // one that will be processed, then we want to + // restore the module scope in which this @load + // was done when we're finished processing it. + if ( ! did_module_restore ) { - if ( streq(prefixes[i], "") ) - { - reporter->Error("can't open %s", full_filename); - exit(1); - } + file_stack.append(new FileInfo(current_module)); + did_module_restore = true; } + else + file_stack.append(new FileInfo); + + char* tmp = copy_string(full_filename); + current_scanned_file_path = dirname(tmp); + delete [] tmp; + + if ( generate_documentation ) + { + current_reST_doc = new BroDoc(bropath_subpath, full_filename); + docs_generated.push_back(current_reST_doc); + } + + delete [] bropath_subpath_delete; + + // "orig_file", could be an alias for yytext, which is ephemeral + // and will be zapped after the yy_switch_to_buffer() below. + yy_switch_to_buffer(yy_create_buffer(f, YY_BUF_SIZE)); + + yylloc.first_line = yylloc.last_line = line_number = 1; + + // Don't delete the old filename - it's pointed to by + // every BroObj created when parsing it. + yylloc.filename = filename = full_filename; + } + + else + { + reporter->Error("can't open %s", full_filename); + exit(1); } return 1; @@ -758,7 +738,7 @@ void add_input_file(const char* file) reporter->InternalError("empty filename"); if ( ! filename ) - (void) load_files_with_prefix(file); + (void) load_files(file); else input_files.append(copy_string(file)); } @@ -809,7 +789,7 @@ int yywrap() check_capture_filter_changes(); check_dpd_config_changes(); - if ( load_files_with_prefix(input_files[0]) ) + if ( load_files(input_files[0]) ) { // Don't delete the filename - it's pointed to by // every BroObj created when parsing it. @@ -825,6 +805,44 @@ int yywrap() check_capture_filter_changes(); check_dpd_config_changes(); + // For each file scanned so far, and for each @prefix, look for + // a prefixed and flattened version of the loaded file in BROPATH. + // The flattening involves taking the path in BROPATH in which + // the scanned file lives and replacing '/' path separators with a '.' + // If the scanned file is "__load__.bro", that part of the flattened file + // name is discarded. + // If the prefix is non-empty, it gets placed in front of the flattened + // path, separated with another '.' + std::list::iterator it; + bool found_prefixed_files = false; + for ( it = files_scanned.begin(); it != files_scanned.end(); ++it ) + { + if ( it->skipped || it->prefixes_checked ) continue; + it->prefixes_checked = true; + // prefixes are pushed onto a stack, so iterate backwards + for ( int i = prefixes.length() - 1; i >= 0; --i ) + { + // don't look at empty prefixes + if ( ! prefixes[i][0] ) continue; + string s; + s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]); + FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0); + //printf("====== prefix search ======\n"); + //printf("File : %s\n", it->name.c_str()); + //printf("Path : %s\n", it->subpath.c_str()); + //printf("Dotted: %s\n", s.c_str()); + //printf("Found : %s\n", f ? "T" : "F"); + //printf("===========================\n"); + if ( f ) + { + add_input_file(s.c_str()); + found_prefixed_files = true; + fclose(f); + } + } + } + if ( found_prefixed_files ) return 0; + // Add redef statements for any X=Y command line parameters. if ( params.size() > 0 ) { diff --git a/src/util.cc b/src/util.cc index e3b9b632fd..f11befe0e5 100644 --- a/src/util.cc +++ b/src/util.cc @@ -15,6 +15,8 @@ # endif #endif +#include +#include #include #include #include @@ -815,6 +817,79 @@ FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs return f; } +// Canonicalizes a given 'file' that lives in 'path' into a flattened, +// dotted format. If the optional 'prefix' argument is given, it is +// prepended to the dotted-format, separated by another dot. +// If 'file' is __load__.bro, that part is discarded when constructing +// the final dotted-format. +string dot_canon(string path, string file, string prefix) + { + string dottedform(prefix); + if ( prefix != "" ) + dottedform.append("."); + dottedform.append(path); + char* tmp = copy_string(file.c_str()); + char* bname = basename(tmp); + if ( ! streq(bname, PACKAGE_LOADER) ) + { + if ( path != "" ) + dottedform.append("."); + dottedform.append(bname); + } + delete [] tmp; + size_t n; + while ( (n = dottedform.find("/")) != string::npos ) + dottedform.replace(n, 1, "."); + return dottedform; + } + +// returns a normalized version of a path, removing duplicate slashes, +// extraneous dots that refer to the current directory, and pops as many +// parent directories referred to by "../" as possible +const char* normalize_path(const char* path) + { + size_t n; + string p(path); + vector components, final_components; + string new_path; + + if ( p[0] == '/' ) + new_path = "/"; + + while ( (n = p.find("/")) != string::npos ) + { + components.push_back(p.substr(0, n)); + p.erase(0, n + 1); + } + components.push_back(p); + + vector::const_iterator it; + for ( it = components.begin(); it != components.end(); ++it ) + { + if ( *it == "" ) continue; + final_components.push_back(*it); + + if ( *it == "." && it != components.begin() ) + final_components.pop_back(); + else if ( *it == ".." && final_components[0] != ".." ) + { + final_components.pop_back(); + final_components.pop_back(); + } + } + + for ( it = final_components.begin(); it != final_components.end(); ++it ) + { + new_path.append(*it); + new_path.append("/"); + } + + if ( new_path.size() > 1 && new_path[new_path.size() - 1] == '/' ) + new_path.erase(new_path.size() - 1); + + return copy_string(new_path.c_str()); + } + // Returns the subpath of BROPATH's policy/ directory in which the loaded // file in located. If it's not under a subpath of policy/ then the full // path is returned, else the subpath of policy/ concatentated with any @@ -865,7 +940,10 @@ void get_policy_subpath(const char* dir, const char* file, const char** subpath) *subpath = full_subpath; } + const char* normalized_subpath = normalize_path(*subpath); delete [] tmp; + delete [] *subpath; + *subpath = normalized_subpath; } extern string current_scanned_file_path; diff --git a/src/util.h b/src/util.h index a77f284ea0..82c86da950 100644 --- a/src/util.h +++ b/src/util.h @@ -5,6 +5,7 @@ #ifndef util_h #define util_h +#include #include #include #include @@ -176,6 +177,8 @@ extern int int_list_cmp(const void* v1, const void* v2); extern const char* bro_path(); extern const char* bro_prefixes(); +std::string dot_canon(std::string path, std::string file, std::string prefix = ""); +const char* normalize_path(const char* path); void get_policy_subpath(const char* dir, const char* file, const char** subpath); extern FILE* search_for_file(const char* filename, const char* ext, const char** full_filename, bool load_pkgs, const char** bropath_subpath); diff --git a/testing/btest/Baseline/core.load-prefixes/output b/testing/btest/Baseline/core.load-prefixes/output new file mode 100644 index 0000000000..733bc9a594 --- /dev/null +++ b/testing/btest/Baseline/core.load-prefixes/output @@ -0,0 +1,4 @@ +loaded lcl2.site.bro +loaded lcl.site.bro +loaded lcl2.protocols.http.bro +loaded lcl.protocols.http.bro diff --git a/testing/btest/core/load-prefixes.bro b/testing/btest/core/load-prefixes.bro new file mode 100644 index 0000000000..4e27801825 --- /dev/null +++ b/testing/btest/core/load-prefixes.bro @@ -0,0 +1,25 @@ +# A test of prefix-based @load'ing + +# @TEST-EXEC: bro addprefixes site protocols/http >output +# @TEST-EXEC: btest-diff output + +@TEST-START-FILE addprefixes.bro +@prefixes += lcl +@prefixes += lcl2 +@TEST-END-FILE + +@TEST-START-FILE lcl.site.bro +print "loaded lcl.site.bro"; +@TEST-END-FILE + +@TEST-START-FILE lcl2.site.bro +print "loaded lcl2.site.bro"; +@TEST-END-FILE + +@TEST-START-FILE lcl.protocols.http.bro +print "loaded lcl.protocols.http.bro"; +@TEST-END-FILE + +@TEST-START-FILE lcl2.protocols.http.bro +print "loaded lcl2.protocols.http.bro"; +@TEST-END-FILE