Refactor search_for_file() util function.

It was getting too bloated and allocated memory in ways that were
difficult to understand how to manage.  Separated out primarily in to
new find_file() and open_file()/open_package() functions.

Also renamed other util functions for path-related things.
This commit is contained in:
Jon Siwek 2013-10-07 15:01:03 -05:00
parent 68227f112d
commit 90477df973
9 changed files with 331 additions and 304 deletions

View file

@ -85,6 +85,7 @@ BroDoc::~BroDoc()
void BroDoc::AddImport(const std::string& s)
{
/*
std::string lname(s);
// First strip any .bro extension.
size_t ext_pos = lname.find(".bro");
@ -142,6 +143,7 @@ void BroDoc::AddImport(const std::string& s)
delete [] full_filename;
delete [] subpath;
*/
}
void BroDoc::SetPacketFilter(const std::string& s)

View file

@ -342,18 +342,16 @@ vector<ParseLocationRec> parse_location_string(const string& s)
if ( ! sscanf(line_string.c_str(), "%d", &plr.line) )
plr.type = plrUnknown;
FILE* throwaway = search_for_file(filename.c_str(), "bro",
&full_filename, true, 0);
if ( ! throwaway )
string path(find_file(filename, bro_path(), "bro"));
if ( path.empty() )
{
debug_msg("No such policy file: %s.\n", filename.c_str());
plr.type = plrUnknown;
return result;
}
fclose(throwaway);
loc_filename = full_filename;
loc_filename = copy_string(path.c_str());
plr.type = plrFileAndLine;
}
}

View file

@ -4,6 +4,7 @@
#define net_h
#include "net_util.h"
#include "util.h"
#include "BPF_Program.h"
#include "List.h"
#include "PktSrc.h"
@ -97,15 +98,14 @@ struct ScannedFile {
ino_t inode;
int include_level;
string name;
string subpath; // Path in BROPATH's policy/ containing the file.
bool skipped; // This ScannedFile was @unload'd.
bool prefixes_checked; // If loading prefixes for this file has been tried.
ScannedFile(ino_t arg_inode, int arg_include_level, string arg_name,
string arg_subpath = "", bool arg_skipped = false,
ScannedFile(ino_t arg_inode, int arg_include_level, const string& arg_name,
bool arg_skipped = false,
bool arg_prefixes_checked = false)
: inode(arg_inode), include_level(arg_include_level),
name(arg_name), subpath(arg_subpath), skipped(arg_skipped),
name(arg_name), skipped(arg_skipped),
prefixes_checked(arg_prefixes_checked)
{ }
};

View file

@ -294,7 +294,8 @@ void OSFingerprint::load_config(const char* file)
uint32 ln=0;
char buf[MAXLINE];
char* p;
FILE* c = search_for_file(file, "osf", 0, false, 0);
FILE* c = open_file(find_file(file, bro_path(), "osf"));
if (!c)
{

View file

@ -226,7 +226,7 @@ bool RuleMatcher::ReadFiles(const name_list& files)
for ( int i = 0; i < files.length(); ++i )
{
rules_in = search_for_file(files[i], "sig", 0, false, 0);
rules_in = open_file(find_file(files[i], bro_path(), "sig"));
if ( ! rules_in )
{
reporter->Error("Can't open signature file %s", files[i]);

View file

@ -20,8 +20,8 @@ void Manager::GenerateDocs() const
void Manager::File(const std::string& path)
{
// TODO
// determine bropath subpath
// can be a file or directory?
// determine path within BROPATH
}
void Manager::ScriptDependency(const std::string& path, const std::string& dep)

View file

@ -31,7 +31,6 @@
#include "broxygen/Manager.h"
extern YYLTYPE yylloc; // holds start line and column of token
extern int print_loaded_scripts;
// Track the @if... depth.
ptr_compat_int current_depth = 0;
@ -52,31 +51,38 @@ char last_tok[128];
if ( ((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin) ) \
reporter->Error("read failed with \"%s\"", strerror(errno));
static string get_dirname(const char* path)
static string find_relative_file(const string& filename, const string& ext)
{
if ( ! path )
return "";
if ( filename.empty() )
return string();
char* tmp = copy_string(path);
string rval = dirname(tmp);
delete[] tmp;
return rval;
if ( filename[0] == '.' )
return find_file(filename, safe_dirname(::filename), ext);
else
return find_file(filename, bro_path(), ext);
}
static ino_t get_inode_num(FILE* f, const char* filename)
static ino_t get_inode_num(FILE* f, const string& path)
{
struct stat b;
if ( fstat(fileno(f), &b) )
{
reporter->Error("failed to fstat fd of %s: %s\n", filename,
reporter->FatalError("fstat of %s failed: %s\n", path.c_str(),
strerror(errno));
exit(1);
}
return b.st_ino;
}
static ino_t get_inode_num(const string& path)
{
FILE* f = open_file(path);
if ( ! f )
reporter->FatalError("failed to open %s\n", path.c_str());
return get_inode_num(f, path);
}
class FileInfo {
public:
FileInfo(string restore_module = "");
@ -265,7 +271,7 @@ when return TOK_WHEN;
@DEBUG return TOK_DEBUG; // marks input for debugger
@DIR {
string rval = get_dirname(::filename);
string rval(safe_dirname(::filename));
if ( ! rval.empty() && rval[0] == '.' )
{
@ -299,42 +305,29 @@ when return TOK_WHEN;
}
@load-sigs{WS}{FILE} {
const char* new_sig_file = skip_whitespace(yytext + 10);
const char* full_filename = 0;
FILE* f = search_for_file(new_sig_file, "sig", &full_filename, false, 0,
get_dirname(::filename));
const char* file = skip_whitespace(yytext + 10);
string path(find_relative_file(file, "sig"));
if ( f )
{
sig_files.push_back(full_filename);
fclose(f);
delete [] full_filename;
}
else
if ( path.empty() )
reporter->Error("failed to find file associated with @load-sigs %s",
new_sig_file);
file);
else
sig_files.push_back(copy_string(path.c_str()));
}
@unload{WS}{FILE} {
// Skip "@unload".
const char* new_file = skip_whitespace(yytext + 7);
// All we have to do is pretend we've already scanned it.
const char* full_filename;
FILE* f = search_for_file(new_file, "bro", &full_filename, true, 0,
get_dirname(::filename));
if ( f )
{
ScannedFile sf(get_inode_num(f, full_filename), file_stack.length(), full_filename, "", true);
files_scanned.push_back(sf);
fclose(f);
delete [] full_filename;
}
const char* file = skip_whitespace(yytext + 7);
string path(find_relative_file(file, "bro"));
if ( path.empty() )
reporter->Error("failed find file associated with @unload %s", file);
else
reporter->Error("failed find file associated with @unload %s", new_file);
{
// All we have to do is pretend we've already scanned it.
ScannedFile sf(get_inode_num(path), file_stack.length(), path, true);
files_scanned.push_back(sf);
}
}
@prefixes{WS}("+"?)={WS}{PREFIX} {
@ -488,22 +481,35 @@ YYLTYPE GetCurrentLocation()
return currloc;
}
static bool already_scanned(ino_t i)
{
list<ScannedFile>::const_iterator it;
for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
if ( it->inode == i )
return true;
return false;
}
static bool already_scanned(const string& path)
{
return already_scanned(get_inode_num(path));
}
static int load_files(const char* orig_file)
{
// Whether we pushed on a FileInfo that will restore the
// current module after the final file has been scanned.
bool did_module_restore = false;
const char* full_filename = "<internal error>";
const char* bropath_subpath = "<internal error>";
const char* bropath_subpath_delete = 0;
FILE* f;
string file_path;
FILE* f = 0;
if ( streq(orig_file, "-") )
{
f = stdin;
full_filename = "<stdin>";
bropath_subpath = "";
file_path = "<stdin>";
if ( g_policy_debug )
{
@ -514,54 +520,35 @@ static int load_files(const char* orig_file)
else
{
f = search_for_file(orig_file, "bro", &full_filename, true, &bropath_subpath, get_dirname(::filename));
bropath_subpath_delete = bropath_subpath; // This will be deleted.
file_path = find_relative_file(orig_file, "bro");
if ( file_path.empty() )
reporter->FatalError("can't find %s", orig_file);
if ( is_dir(file_path.c_str()) )
f = open_package(file_path);
else
f = open_file(file_path);
if ( ! f )
reporter->FatalError("can't open %s", file_path.c_str());
}
if ( f )
{
ino_t i = get_inode_num(f, full_filename);
std::list<ScannedFile>::const_iterator it;
ino_t i = get_inode_num(f, file_path);
for ( it = files_scanned.begin(); it != files_scanned.end(); ++it )
{
if ( it->inode == i )
{
if ( f != stdin )
{
fclose(f);
delete [] full_filename;
delete [] bropath_subpath_delete;
}
if ( already_scanned(i) )
return 0;
}
}
ScannedFile sf(i, file_stack.length(), full_filename, bropath_subpath);
ScannedFile sf(i, file_stack.length(), file_path);
files_scanned.push_back(sf);
if ( g_policy_debug )
if ( g_policy_debug && ! file_path.empty() )
{
// Add the filename to the file mapping
// table (Debug.h).
// Add the filename to the file mapping table (Debug.h).
Filemap* map = new Filemap;
// Make sure it wasn't already read in.
HashKey* key = new HashKey(full_filename);
if ( g_dbgfilemaps.Lookup(key) )
{
// reporter->Warning("Not re-reading policy file; check BRO_PREFIXES:", full_filename);
fclose(f);
delete key;
return 0;
}
else
{
HashKey* key = new HashKey(file_path.c_str());
g_dbgfilemaps.Insert(key, map);
}
if ( full_filename )
LoadPolicyFileText(full_filename);
LoadPolicyFileText(file_path.c_str());
}
// Remember where we were. If this is the first
@ -577,9 +564,7 @@ static int load_files(const char* orig_file)
else
file_stack.append(new FileInfo);
broxygen_mgr->File(full_filename);
delete [] bropath_subpath_delete;
broxygen_mgr->File(file_path);
// "orig_file", could be an alias for yytext, which is ephemeral
// and will be zapped after the yy_switch_to_buffer() below.
@ -589,14 +574,7 @@ static int load_files(const char* orig_file)
// Don't delete the old filename - it's pointed to by
// every BroObj created when parsing it.
yylloc.filename = filename = full_filename;
}
else
{
reporter->Error("can't open %s", full_filename);
exit(1);
}
yylloc.filename = filename = copy_string(file_path.c_str());
return 1;
}
@ -773,24 +751,22 @@ int yywrap()
if ( ! prefixes[i][0] )
continue;
string s;
s = dot_canon(it->subpath.c_str(), it->name.c_str(), prefixes[i]);
FILE* f = search_for_file(s.c_str(), "bro", 0, false, 0,
get_dirname(::filename));
string sub(find_dir_in_bropath(it->name));
string flat(flatten_script_name(sub, it->name, prefixes[i]));
string path(find_relative_file(flat, "bro"));
if ( ! path.empty() )
{
add_input_file(path.c_str());
found_prefixed_files = true;
}
//printf("====== prefix search ======\n");
//printf("File : %s\n", it->name.c_str());
//printf("Path : %s\n", it->subpath.c_str());
//printf("Dotted: %s\n", s.c_str());
//printf("Path : %s\n", sub.c_str());
//printf("Dotted: %s\n", flat.c_str());
//printf("Found : %s\n", f ? "T" : "F");
//printf("===========================\n");
if ( f )
{
add_input_file(s.c_str());
found_prefixed_files = true;
fclose(f);
}
}
}

View file

@ -917,90 +917,127 @@ string bro_prefixes()
const char* PACKAGE_LOADER = "__load__.bro";
// If filename is pointing to a directory that contains a file called
// PACKAGE_LOADER, returns the files path. Otherwise returns filename itself.
// In both cases, the returned string is newly allocated.
static const char* check_for_dir(const char* filename, bool load_pkgs)
FILE* open_file(const string& path, const string& mode)
{
if ( load_pkgs && is_dir(filename) )
{
char init_filename_buf[1024];
safe_snprintf(init_filename_buf, sizeof(init_filename_buf),
"%s/%s", filename, PACKAGE_LOADER);
if ( path.empty() )
return 0;
if ( access(init_filename_buf, R_OK) == 0 )
return copy_string(init_filename_buf);
}
FILE* rval = fopen(path.c_str(), mode.c_str());
return copy_string(filename);
}
static FILE* open_file(const char* filename, const char** full_filename, bool load_pkgs)
{
filename = check_for_dir(filename, load_pkgs);
if ( full_filename )
*full_filename = copy_string(filename);
FILE* f = fopen(filename, "r");
if ( ! f )
if ( ! rval )
{
char buf[256];
strerror_r(errno, buf, sizeof(buf));
reporter->Error("Failed to open file %s: %s", filename, buf);
}
delete [] filename;
return f;
return rval;
}
// Canonicalizes a given 'file' that lives in 'path' into a flattened,
// dotted format. If the optional 'prefix' argument is given, it is
// prepended to the dotted-format, separated by another dot.
// If 'file' is __load__.bro, that part is discarded when constructing
// the final dotted-format.
string dot_canon(string path, string file, string prefix)
static bool can_read(const string& path)
{
return access(path.c_str(), R_OK) == 0;
}
FILE* open_package(string& path, const string& mode)
{
string arg_path(path);
path.append("/").append(PACKAGE_LOADER);
if ( can_read(path) )
return open_file(path, mode);
reporter->Error("Failed to open package '%s': missing '%s' file",
arg_path.c_str(), PACKAGE_LOADER);
return 0;
}
string safe_dirname(const char* path)
{
if ( ! path )
return ".";
return safe_dirname(string(path));
}
string safe_dirname(const string& path)
{
char* tmp = copy_string(path.c_str());
string rval(dirname(tmp));
delete [] tmp;
return rval;
}
string safe_basename(const char* path)
{
if ( ! path )
return ".";
return safe_basename(string(path));
}
string safe_basename(const string& path)
{
char* tmp = copy_string(path.c_str());
string rval(basename(tmp));
delete [] tmp;
return rval;
}
string flatten_script_name(const string& dir, const string& file,
const string& prefix)
{
string dottedform(prefix);
if ( prefix != "" )
dottedform.append(".");
dottedform.append(path);
char* tmp = copy_string(file.c_str());
char* bname = basename(tmp);
if ( ! streq(bname, PACKAGE_LOADER) )
dottedform.append(dir);
string bname(safe_basename(file));
if ( bname != string(PACKAGE_LOADER) )
{
if ( path != "" )
if ( dir != "" )
dottedform.append(".");
dottedform.append(bname);
}
delete [] tmp;
size_t n;
while ( (n = dottedform.find("/")) != string::npos )
dottedform.replace(n, 1, ".");
return dottedform;
}
// returns a normalized version of a path, removing duplicate slashes,
// extraneous dots that refer to the current directory, and pops as many
// parent directories referred to by "../" as possible
const char* normalize_path(const char* path)
static vector<string>* tokenize_string(string input, const string& delim,
vector<string>* rval)
{
if ( ! rval )
rval = new vector<string>();
size_t n;
while ( (n = input.find(delim)) != string::npos )
{
rval->push_back(input.substr(0, n));
input.erase(0, n + 1);
}
rval->push_back(input);
return rval;
}
string normalize_path(const string& path)
{
size_t n;
string p(path);
vector<string> components, final_components;
string new_path;
if ( p[0] == '/' )
if ( path[0] == '/' )
new_path = "/";
while ( (n = p.find("/")) != string::npos )
{
components.push_back(p.substr(0, n));
p.erase(0, n + 1);
}
components.push_back(p);
tokenize_string(path, "/", &components);
vector<string>::const_iterator it;
for ( it = components.begin(); it != components.end(); ++it )
@ -1026,125 +1063,86 @@ const char* normalize_path(const char* path)
if ( new_path.size() > 1 && new_path[new_path.size() - 1] == '/' )
new_path.erase(new_path.size() - 1);
return copy_string(new_path.c_str());
return new_path;
}
// Returns the subpath of the root Bro script install/source/build directory in
// which the loaded file is located. If it's not under a subpath of that
// directory (e.g. cwd or custom path) then the full path is returned.
void get_script_subpath(const std::string& full_filename, const char** subpath)
string find_dir_in_bropath(const string& path)
{
size_t p;
std::string my_subpath(full_filename);
string rval(path);
// get the parent directory of file (if not already a directory)
if ( ! is_dir(full_filename.c_str()) )
{
char* tmp = copy_string(full_filename.c_str());
my_subpath = dirname(tmp);
delete [] tmp;
}
if ( ! is_dir(path.c_str()) )
rval = safe_dirname(path);
// first check if this is some subpath of the installed scripts root path,
// if not check if it's a subpath of the script source root path,
// then check if it's a subpath of the build directory (where BIF scripts
// will get generated).
// If none of those, will just use the given directory.
if ( (p = my_subpath.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
else if ( (p = my_subpath.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
else if ( (p = my_subpath.find(BRO_BUILD_SOURCE_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_BUILD_SOURCE_PATH));
else if ( (p = my_subpath.find(BRO_BUILD_SCRIPTS_PATH)) != std::string::npos )
my_subpath.erase(0, strlen(BRO_BUILD_SCRIPTS_PATH));
if ( (p = rval.find(BRO_SCRIPT_INSTALL_PATH)) != std::string::npos )
rval.erase(0, strlen(BRO_SCRIPT_INSTALL_PATH));
else if ( (p = rval.find(BRO_SCRIPT_SOURCE_PATH)) != std::string::npos )
rval.erase(0, strlen(BRO_SCRIPT_SOURCE_PATH));
else if ( (p = rval.find(BRO_BUILD_SOURCE_PATH)) != std::string::npos )
rval.erase(0, strlen(BRO_BUILD_SOURCE_PATH));
else if ( (p = rval.find(BRO_BUILD_SCRIPTS_PATH)) != std::string::npos )
rval.erase(0, strlen(BRO_BUILD_SCRIPTS_PATH));
// if root path found, remove path separators until next path component
if ( p != std::string::npos )
while ( my_subpath.size() && my_subpath[0] == '/' )
my_subpath.erase(0, 1);
while ( rval.size() && rval[0] == '/' )
rval.erase(0, 1);
*subpath = normalize_path(my_subpath.c_str());
return normalize_path(rval);
}
FILE* search_for_file(const char* filename, const char* ext,
const char** full_filename, bool load_pkgs,
const char** bropath_subpath, string prepend_to_search_path)
static string find_file_in_path(const string& filename, const string& path,
const string& opt_ext = "")
{
// If the file is a literal absolute path we don't have to search,
// just return the result of trying to open it. If the file is
// might be a relative path, check first if it's a real file that
// can be referenced from cwd, else we'll try to search for it based
// on what path the currently-loading script is in as well as the
// standard BROPATH paths.
if ( filename[0] == '/' ||
(filename[0] == '.' && access(filename, R_OK) == 0) )
{
if ( bropath_subpath )
{
char* tmp = copy_string(filename);
*bropath_subpath = copy_string(dirname(tmp));
delete [] tmp;
}
return open_file(filename, full_filename, load_pkgs);
}
if ( filename.empty() )
return string();
char path[1024], full_filename_buf[1024];
// Prepend the currently loading script's path to BROPATH so that
// @loads can be referenced relatively.
if ( ! prepend_to_search_path.empty() && filename[0] == '.' )
safe_snprintf(path, sizeof(path), "%s:%s",
prepend_to_search_path.c_str(), bro_path());
// If file name is an absolute path, searching within *path* is pointless.
if ( filename[0] == '/' )
{
if ( can_read(filename) )
return filename;
else
safe_strncpy(path, bro_path(), sizeof(path));
char* dir_beginning = path;
char* dir_ending = path;
int more = *dir_beginning != '\0';
while ( more )
{
while ( *dir_ending && *dir_ending != ':' )
++dir_ending;
if ( *dir_ending == ':' )
*dir_ending = '\0';
else
more = 0;
safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
"%s/%s.%s", dir_beginning, filename, ext);
if ( access(full_filename_buf, R_OK) == 0 &&
! is_dir(full_filename_buf) )
{
if ( bropath_subpath )
get_script_subpath(full_filename_buf, bropath_subpath);
return open_file(full_filename_buf, full_filename, load_pkgs);
return string();
}
safe_snprintf(full_filename_buf, sizeof(full_filename_buf),
"%s/%s", dir_beginning, filename);
if ( access(full_filename_buf, R_OK) == 0 )
string abs_path(path + '/' + filename);
if ( ! opt_ext.empty() )
{
if ( bropath_subpath )
get_script_subpath(full_filename_buf, bropath_subpath);
return open_file(full_filename_buf, full_filename, load_pkgs);
string with_ext(abs_path + '.' + opt_ext);
if ( can_read(with_ext) )
return with_ext;
}
dir_beginning = ++dir_ending;
if ( can_read(abs_path) )
return abs_path;
return string();
}
if ( full_filename )
*full_filename = copy_string(filename);
if ( bropath_subpath )
string find_file(const string& filename, const string& path_set,
const string& opt_ext)
{
char* tmp = copy_string(filename);
*bropath_subpath = copy_string(dirname(tmp));
delete [] tmp;
vector<string> paths;
tokenize_string(path_set, ":", &paths);
for ( size_t n = 0; n < paths.size(); ++n )
{
string f = find_file_in_path(filename, paths[n], opt_ext);
if ( ! f.empty() )
return f;
}
return 0;
return string();
}
FILE* rotate_file(const char* name, RecordVal* rotate_info)

View file

@ -205,12 +205,64 @@ extern int int_list_cmp(const void* v1, const void* v2);
extern const char* bro_path();
extern const char* bro_magic_path();
extern std::string bro_prefixes();
std::string dot_canon(std::string path, std::string file, std::string prefix = "");
const char* normalize_path(const char* path);
void get_script_subpath(const std::string& full_filename, const char** subpath);
extern FILE* search_for_file(const char* filename, const char* ext,
const char** full_filename, bool load_pkgs, const char** bropath_subpath,
std::string prepend_to_search_path = "");
// Wrappers for dirname(3) that won't modify argument.
std::string safe_dirname(const char* path);
std::string safe_dirname(const std::string& path);
// Wrappers for basename(3) that won't modify argument.
std::string safe_basename(const char* path);
std::string safe_basename(const std::string& path);
/**
* Flatten a script name by replacing '/' path separators with '.'.
* @param dir A directory containing \a file.
* @param file A path to a Bro script. If it is a __load__.bro, that part
* is discarded when constructing the flattened the name.
* @param prefix A string to prepend to the flattened script name.
* @return The flattened script name.
*/
std::string flatten_script_name(const std::string& dir,
const std::string& file,
const std::string& prefix = "");
/**
* Return a canonical/shortened path string by removing superfluous elements
* (path delimiters, dots referring to CWD or parent dir).
* @param path A filesystem path.
* @return A canonical/shortened version of \a path.
*/
std::string normalize_path(const std::string& path);
/**
* Locate a file/direcotry within BROPATH.
* @param path A file/directory to locate within BROPATH.
* @return The directory within BROPATH that \a path located or an absolute
* path to \a path if it couldn't be located in BROPATH
*/
std::string find_dir_in_bropath(const std::string& path);
/**
* Locate a file within a given search path.
* @param filename Name of a file to find.
* @param path_set Colon-delimited set of paths to search for the file.
* @param opt_ext A filename extension/suffix to allow.
* @return Path to the found file, or an empty string if not found.
*/
std::string find_file(const std::string& filename, const std::string& path_set,
const std::string& opt_ext = "");
// Wrapper around fopen(3). Emits an error when failing to open.
FILE* open_file(const std::string& path, const std::string& mode = "r");
/** Opens a Bro script package.
* @param path Location of a Bro script package (a directory). Will be changed
* to the path of the package's loader script.
* @param mode An fopen(3) mode.
* @return The return value of fopen(3) on the loader script or null if one
* doesn't exist.
*/
FILE* open_package(std::string& path, const std::string& mode = "r");
// Renames the given file to a new temporary name, and opens a new file with
// the original name. Returns new file or NULL on error. Inits rotate_info if