diff --git a/policy/bro.init b/policy/bro.init index f9742798c4..12d707d812 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -272,6 +272,60 @@ type entropy_test_result: record { serial_correlation: double; }; +type Log_Writer: enum { # TODO: Move these into bif and use from C++ as well. + WRITER_DEFAULT, # See default_writer below. + WRITER_ASCII, +}; + +# Each stream gets a unique ID. This type will be extended by +# other scripts. +type Log_ID: enum { + Unknown +}; + +# The default writer to use if a filter does not specify +# anything else. +const Log_default_writer = WRITER_ASCII &redef; + +# A filter defining what to log. +type log_filter: record { + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(rec: any): bool &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: string &optional; + path_func: function(id: string): string &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + include: set[string] &optional; + exclude: set[string] &optional; + + # An event that is raised whenever the filter is applied + # to an entry. The event receives the same parameter + # as the predicate. It will always be generated, + # independent of what the predicate returns. + ev: event(rec: any) &optional; + + # The writer to use. + writer: Log_Writer &default=Log_default_writer; +}; + + # Prototypes of Bro built-in functions. @load strings.bif.bro @load bro.bif.bro @@ -1382,10 +1436,11 @@ const suppress_local_output = F &redef; # Holds the filename of the trace file given with -w (empty if none). const trace_output_file = ""; - + # If a trace file is given, dump *all* packets seen by Bro into it. # By default, Bro applies (very few) heuristics to reduce the volume. # A side effect of setting this to true is that we can write the # packets out before we actually process them, which can be helpful # for debugging in case the analysis triggers a crash. const record_all_packets = F &redef; + diff --git a/policy/logging.bro b/policy/logging.bro index 610e9e6f69..f00dcf7fb2 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -1,176 +1,10 @@ -module Logging; -export { - # The set of writers Bro provides. - type Writer: enum { - WRITER_DEFAULT, # See default_writer below. - WRITER_CSV, - WRITER_DATA_SERIES, - WRITER_SYSLOG - }; - - # Each stream gets a unique ID. This type will be extended by - # other scripts. - type ID: enum { - Unknown - }; - - # The default writer to use if a filter does not specify - # anything else. - const default_writer = WRITER_CSV &redef; - - # Type defining a stream. - type Stream: record { - name: string; - columns: string_vec; - }; - - # A filter defining what to record. - type Filter: record { - # A name to reference this filter. - name: string; - - # A predicate returning True if the filter wants a log entry - # to be recorded. If not given, an implicit True is assumed - # for all entries. The predicate receives one parameter: - # an instance of the log's record type with the fields to be - # logged. - pred: function(rec: any): bool &optional; - - # A path for outputting everything matching this - # filter. The path is either a string, or a function - # called with a single ``ID`` argument and returning a string. - # - # The specific interpretation of the string is left to the - # Writer, but if it's refering to a file, it's assumed that no - # extension is given; the writer will add whatever is - # appropiate. - path: string &optional; - dynamic_path: function(id: string): string &optional; - - # A subset of column names to record. If not given, all - # columns are recorded. - #select: set[string] &optional; - - # An event that is raised whenever the filter is applied - # to an entry. The event receives the same parameter - # as the predicate. It will always be generated, - # independent of what the predicate returns. - #ev: event(rec: any) &optional; - - # The writer to use. - writer: Writer &default=default_writer; - - # Internal tracking of header names and order for this filter. - #columns: string_vec &optional; - }; - - # Logs the record "rec" to the stream "id". The type of - # "rec" must match the stream's "columns" field. - global log: function(id: string, rec: any); - #global log_ev: event(id: string, rec: any); - - # Returns an existing filter previously installed for stream - # "id" under the given "name". If no such filter exists, - # the record "NoSuchFilter" is returned. - global get_filter: function(id: string, name: string) : Filter; - - global create_stream: function(id: string, log_record_type: string); - global add_filter: function(id: string, filter: Filter); - global remove_filter: function(id: string, filter: string): bool; - - global add_default_filter: function(id: string); - global remove_default_filter: function(id: string): bool; - - global open_log_files: function(id: string); - - # This is the internal filter store. The outer table is indexed with a string - # representing the stream name that the set of Logging::Filters is applied to. - global filters: table[string] of set[Filter]; - - # This is the internal stream store. The table is indexed by the stream name. - global streams: table[string] of Stream; - - global files: table[string] of file; -} - - -# Sentinel representing an unknown filter.d -const NoSuchFilter: Filter = [$name="", $path="unknown"]; - -function create_stream(id: string, log_record_type: string) +function Log_add_default_filter(id: Log_ID) { - if ( id in streams ) - print fmt("Stream %s already exists!", id); - - streams[id] = [$name=log_record_type, $columns=record_type_to_vector(log_record_type)]; - # Insert this as a separate step because the file_opened event needs - # the stream id to already exist. - #streams[id]$_file = open_log_file(id); + log_add_filter(id, [$name="default"]); } -function add_filter(id: string, filter: Filter) +function Log_remove_default_filter(id: Log_ID): bool { - if ( id !in filters ) - filters[id] = set(); - - add filters[id][filter]; + log_remove_filter(id, "default"); } - -function remove_filter(id: string, filter: string): bool - { - for ( filt in filters[id] ) - { - if ( filt$name == "default" ) - { - delete filters[id][filt]; - return T; - } - } - return F; - } - -function add_default_filter(id: string) - { - add_filter(id, [$name="default", $path=id]); - } - -function remove_default_filter(id: string): bool - { - return remove_filter("ssh", "default"); - } - -event file_opened(f: file) &priority=10 - { - # Only do any of this for files opened locally. - if ( is_remote_event() ) return; - - # TODO: this shouldn't rely on .log being the extension - local filename = gsub(get_file_name(f), /\.log$/, ""); - if ( filename in streams ) - { - enable_raw_output(f); - - if (peer_description == "" || - peer_description == "manager" || - peer_description == "standalone") - { - print f, join_string_vec(streams[filename]$columns, "\t"); - } - } - else - { - print "no raw output", filename; - } - } - -function log(id: string, rec: any) - { - logging_log(id, rec); - } - - -event bro_init() &priority=-10 - { - # TODO: Check for logging streams without filters. - } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro index df57c6d576..e8ccc7c22e 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -4,7 +4,7 @@ module SSH; export { # Create a new ID for our log stream - #redef enum Logging::ID += { LOG_SSH }; + redef enum Log_ID += { LOG_SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -14,10 +14,6 @@ export { status: string &optional; country: string &default="unknown"; }; - - # This is the prototype for the event that the logging framework tries - # to generate if there is a handler for it. - #global log: event(rec: Log); } event bro_init() @@ -25,35 +21,27 @@ event bro_init() # Create the stream. # First argument is the ID for the stream. # Second argument is the log record type. - Logging::create_stream("ssh", "SSH::Log"); + log_create_stream(LOG_SSH, SSH::Log); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - # Log line event generation is autogenerated for now by checking for - # handlers for MODULE_NAME::log (which isn't the right thing to do, but it will be dealt with later) - Logging::add_default_filter("ssh"); - - # There is currently some problem with &optional values in the records - # passed into the predicate. Maybe it's because I'm not really coercing - # the record to the correct record type before passing it as an argument - # to the Call method? - - # There is also a problem with using &optional sets in the filter records. - # It was found when trying to include the "select" variable. - + Log_add_default_filter(LOG_SSH); + # Printing headers for the filters doesn't work yet either and needs to # be considered in the final design. (based on the "select" set). - #Logging::add_filter("ssh", [$name="successful logins", + #Log::add_filter("ssh", [$name="successful logins", # #$pred(rec: Log) = { print rec$status; return T; }, # $path="ssh-logins", # #$select=set("t"), - # $writer=Logging::WRITER_CSV]); - + # $writer=Log::WRITER_CSV]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + # Log something. - Logging::log("ssh", [$t=network_time(),$status="success"]); - Logging::log("ssh", [$t=network_time(),$status="failure", $country="US"]); - Logging::log("ssh", [$t=network_time(),$status="failure", $country="UK"]); - Logging::log("ssh", [$t=network_time(),$status="success", $country="BR"]); - Logging::log("ssh", [$t=network_time(),$status="failure", $country="MX"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); } @@ -68,4 +56,4 @@ event bro_init() #event SSH::log(rec: SSH::Log) # { # print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); -# } \ No newline at end of file +# } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f67dc173e..03b6079df5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -297,6 +297,9 @@ set(bro_SRCS IRC.cc List.cc Logger.cc + LogMgr.cc + LogWriter.cc + LogWriterAscii.cc Login.cc MIME.cc NCP.cc diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 9ab0bde380..3c9dbad363 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -17,6 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, + { "logging", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 3d8c1ac83f..4614597bfc 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -25,6 +25,7 @@ enum DebugStream { DBG_MAINLOOP, // Main IOSource loop DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli + DBG_LOGGING, // Logging streams NUM_DBGS // Has to be last }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc new file mode 100644 index 0000000000..b9675e4e84 --- /dev/null +++ b/src/LogMgr.cc @@ -0,0 +1,499 @@ + +#include "LogMgr.h" +#include "EventHandler.h" +#include "NetVar.h" + +#include "LogWriterAscii.h" + +struct LogWriterDefinition { + LogWriterType::Type type; // The type. + const char *name; // Descriptive name for error messages. + bool (*init)(); // An optional one-time initialization function. + LogWriter* (*factory)(); // A factory function creating instances. +}; + +LogWriterDefinition log_writers[] = { + { LogWriterType::Ascii, "Ascii", 0, LogWriterAscii::Instantiate }, + + // End marker. + { LogWriterType::None, "None", 0, (LogWriter* (*)())0 } +}; + +struct LogMgr::Filter { + string name; + Func* pred; + Func* path_func; + EventHandlerPtr* event; + string path; + LogWriterDefinition* writer; + + int num_fields; + LogField** fields; + vector > indices; // List of record indices per field. + + typedef map WriterMap; + WriterMap writers; // Writers indexed by path. + + ~Filter(); +}; + +struct LogMgr::Stream { + string name; + RecordType* columns; + list filters; + + ~Stream(); + }; + + +LogMgr::Filter::~Filter() + { + for ( int i = 0; i < num_fields; ++i ) + delete fields[i]; + + for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) + delete i->second; + } + +LogMgr::Stream::~Stream() + { + Unref(columns); + for ( list::iterator f = filters.begin(); f != filters.end(); ++f ) + delete *f; + } + +LogMgr::LogMgr() + { + } + +LogMgr::~LogMgr() + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + delete *s; + } + +bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns) + { + // TODO: Should check that the record has only supported types. + + unsigned int idx = stream_id->AsEnum(); + + // Make sure the vector has an entries for all streams up to the one + // given. + while ( idx >= streams.size() ) + streams.push_back(0); + + if ( streams[idx] ) + // We already know this one, delete the previous definition. + delete streams[idx]; + + // Create new stream and record the type for the columns. + streams[idx] = new Stream; + streams[idx]->name = stream_id->Type()->AsEnumType()->Lookup(idx); + streams[idx]->columns = columns; + columns->Ref(); + + DBG_LOG(DBG_LOGGING, "Created new logging stream '%s'", streams[idx]->name.c_str()); + + return true; + } + +// Helper for recursive record field unrolling. +bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) + { + for ( int i = 0; i < rt->NumFields(); ++i ) + { + BroType* t = rt->FieldType(i); + + list new_indices = indices; + new_indices.push_back(i); + + // Build path name. + string new_path; + if ( ! path.size() ) + new_path = rt->FieldName(i); + else + new_path = path + "." + rt->FieldName(i); + + StringVal* new_path_val = new StringVal(path.c_str()); + + if ( t->InternalType() == TYPE_INTERNAL_OTHER ) + { + if ( t->Tag() == TYPE_RECORD ) + { + // Recurse. + if ( ! TraverseRecord(filter, t->AsRecordType(), include, exclude, new_path, new_indices) ) + return false; + } + else + { + run_time("unsupported field type for log column"); + return false; + } + + continue; + } + + // If include fields are specified, only include if explicitly listed. + if ( include ) + { + if ( ! include->Lookup(new_path_val) ) + return true; + } + + // If exclude fields are specified, do not only include if listed. + if ( exclude ) + { + if ( exclude->Lookup(new_path_val) ) + return true; + } + + // Alright, we want this field. + + filter->indices.push_back(new_indices); + filter->fields = (LogField**) realloc(filter->fields, sizeof(LogField) * ++filter->num_fields); + if ( ! filter->fields ) + { + run_time("out of memory in add_filter"); + return false; + } + + LogField* field = new LogField(); + field->name = new_path; + field->type = t->Tag(); + filter->fields[filter->num_fields - 1] = field; + } + + return true; + } + +bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) + { + RecordType* rtype = fval->Type()->AsRecordType(); + + if ( ! same_type(rtype, log_filter, 0) ) + { + run_time("filter argument not of right type"); + return false; + } + + Stream* stream = streams[stream_id->AsEnum()]; + if ( ! stream ) + { + run_time("undefined log stream"); + return false; + } + + // Find the right writer type. + int writer = 0; + int idx = rtype->FieldOffset("writer"); + Val* writer_val = fval->Lookup(idx); + + if ( ! writer_val ) + { + // Use default. + // FIXME: Shouldn't Lookup() already take care if this? + const Attr* def_attr = log_filter->FieldDecl(idx)->FindAttr(ATTR_DEFAULT); + if ( ! def_attr ) + internal_error("log_filter missing &default for writer attribute"); + + writer_val = def_attr->AttrExpr()->Eval(0); + writer = writer_val->AsEnum(); + Unref(writer_val); + } + else + writer = writer_val->AsEnum(); + + LogWriterDefinition* ld; + for ( ld = log_writers; ld->type != LogWriterType::None; ++ld ) + { + if ( ld->type == writer ) + break; + } + + if ( ld->type == LogWriterType::None ) + internal_error("unknow writer in add_filter"); + + if ( ! ld->factory ) + // Oops, we can't instantuate this guy. + return true; // Count as success, as we will have reported it earlier already. + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't call it again later. + ld->init = 0; + else + // Init failed, disable by deleting factory function. + ld->factory = 0; + return false; + } + + // Create a new Filter instance. + + Val* event = fval->Lookup(rtype->FieldOffset("ev")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + Val* path_func = fval->Lookup(rtype->FieldOffset("path_func")); + + Filter* filter = new Filter; + filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->pred = path_func ? path_func->AsFunc() : 0; + filter->writer = ld; + + if ( event ) + { + // TODO: Implement + filter->event = 0; + } + + // Build the list of fields that the filter wants included, including + // potentially rolling out fields. + Val* include = fval->Lookup(rtype->FieldOffset("include")); + Val* exclude = fval->Lookup(rtype->FieldOffset("exclude")); + + filter->num_fields = 0; + filter->fields = 0; + if ( ! TraverseRecord(filter, stream->columns, include ? include->AsTableVal() : 0, exclude ? exclude->AsTableVal() : 0, "", list()) ) + return false; + + // Get the path for the filter. + Val* path_val = fval->Lookup(rtype->FieldOffset("path")); + + if ( path_val ) + filter->path = path_val->AsString()->CheckString(); + + else + { + // If no path is given, use the Stream ID as the default. + const char* n = stream->name.c_str(); + char* lower = new char[strlen(n) + 1]; + for ( char* s = lower; *n; ++n, ++s ) + { + if ( strncmp(n, "::", 2) == 0 ) + { + // Remove the scope operator. TODO: We need ab better way to + // generate the default here, but let's wait until we have + // everything in the right namespace. + *s = '_'; + ++n; + } + + else + *s = tolower(*n); + } + + filter->path = string(lower); + free(lower); + } + + stream->filters.push_back(filter); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Created new filter '%s' for stream '%s'", filter->name.c_str(), stream->name.c_str()); + DBG_LOG(DBG_LOGGING, " writer : %s", ld->name); + DBG_LOG(DBG_LOGGING, " path : %s", filter->path.c_str()); + DBG_LOG(DBG_LOGGING, " path_func : %s", (filter->path_func ? "set" : "not set")); + DBG_LOG(DBG_LOGGING, " event : %s", (filter->event ? "set" : "not set")); + DBG_LOG(DBG_LOGGING, " pred : %s", (filter->pred ? "set" : "not set")); + + for ( int i = 0; i < filter->num_fields; i++ ) + { + LogField* field = filter->fields[i]; + DBG_LOG(DBG_LOGGING, " field %10s: %s", field->name.c_str(), type_name(field->type)); + } +#endif + + return true; + } + +bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) + { +#if 0 + int idx = stream_id->AsEnum(); + + if ( idx >= streams.size() || ! streams[idx] ) + { + run_time("unknown log stream"); + return false; + } + + +#endif + return true; + } + +bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) + { + unsigned int idx = stream_id->AsEnum(); + + if ( idx >= streams.size() || ! streams[idx] ) + { + run_time("unknown log stream"); + return false; + } + + Stream* stream = streams[idx]; + + columns = columns->CoerceTo(stream->columns); + + if ( ! columns ) + { + run_time("imcompatible log record type"); + return false; + } + + // Send to each of our filters. + for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + { + Filter* filter = *i; + + string path = filter->path; + + if ( filter->event ) + { + // XXX Raise event here. + // TODO: Actually, the filter should be an attribute of the stream, right? + } + + if ( filter->pred ) + { + // XXX Check predicate here. + } + + if ( filter->path_func ) + { + // XXX Do dynamic path here. + } + + // See if we already have a writer for this path. + Filter::WriterMap::iterator w = filter->writers.find(path); + + LogWriter* writer = 0; + if ( w == filter->writers.end() ) + { + // No, need to create one. + assert(filter->writer->factory); + writer = (*filter->writer->factory)(); + + // Copy the fields for LogWriter::Init() as it will take + // ownership. + LogField** arg_fields = new LogField*[filter->num_fields]; + for ( int j = 0; j < filter->num_fields; ++j ) + arg_fields[j] = new LogField(*filter->fields[j]); + + if ( ! writer->Init(path, filter->num_fields, arg_fields) ) + { + Unref(columns); + return false; + } + + filter->writers.insert(Filter::WriterMap::value_type(path, writer)); + } + + else + // We have a writer already. + writer = w->second; + + // Alright, can do the write now. + LogVal** vals = RecordToFilterVals(filter, columns); + writer->Write(vals); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", filter->name.c_str(), stream->name.c_str()); +#endif + } + + Unref(columns); + return true; + } + +LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) + { + LogVal** vals = new LogVal*[filter->num_fields]; + + for ( int i = 0; i < filter->num_fields; ++i ) + { + Val* val = columns; + + // For each field, first find the right value, which can potentially + // be nested inside other records. + list& indices = filter->indices[i]; + + for ( list::iterator j = indices.begin(); j != indices.end(); ++j ) + { + val = val->AsRecordVal()->Lookup(*j); + + if ( ! val ) + { + // Value, or any of its parents, is not set. + vals[i] = new LogVal(false); + break; + } + } + + if ( ! val ) + continue; + + switch ( val->Type()->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + vals[i] = new LogVal(); + vals[i]->val.int_val = val->InternalInt(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + vals[i] = new LogVal(); + vals[i]->val.uint_val = val->InternalUnsigned(); + break; + + case TYPE_SUBNET: + vals[i] = new LogVal(); + vals[i]->val.subnet_val = *val->AsSubNet(); + break; + + case TYPE_NET: + case TYPE_ADDR: + { + vals[i] = new LogVal(); + addr_type t = val->AsAddr(); + copy_addr(&t, &vals[i]->val.addr_val); + break; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + vals[i] = new LogVal(); + vals[i]->val.double_val = val->InternalDouble(); + break; + + case TYPE_STRING: + { + const BroString* s = val->AsString(); + LogVal* lval = (LogVal*) new char[sizeof(LogVal) + sizeof(log_string_type) + s->Len()]; + new (lval) LogVal(); // Run ctor. + lval->val.string_val.len = s->Len(); + memcpy(&lval->val.string_val.string, s->Bytes(), s->Len()); + vals[i] = lval; + break; + } + + default: + internal_error("unsupported type for log_write"); + } + } + + return vals; + } + + +void LogMgr::Error(LogWriter* writer, const char* msg) + { +#if 0 +#endif + } diff --git a/src/LogMgr.h b/src/LogMgr.h new file mode 100644 index 0000000000..b62dd5ff35 --- /dev/null +++ b/src/LogMgr.h @@ -0,0 +1,82 @@ +// +// A class managing log writers and filters. + +#ifndef LOGMGR_H +#define LOGMGR_H + +#include "Val.h" + +// One value per writer type we have. +namespace LogWriterType { + enum Type { + None, + Ascii + }; +}; + +struct LogField { + LogField() { } + LogField(const LogField& other) : name(other.name), type(other.type) { } + string name; + TypeTag type; +}; + +// A string that we can directly include as part of the value union below. +struct log_string_type { + int len; + char string[]; // The string starts right here. +}; + +// All values that can be directly logged by a Writer. +struct LogVal { + LogVal(bool arg_present = true) : present(arg_present) {} + + bool present; // If false, the field is unset (i.e., &optional and not initialzed). + + // The following union is a subset of BroValUnion, including only the + // atomic types. + union { + bro_int_t int_val; + bro_uint_t uint_val; + addr_type addr_val; + subnet_type subnet_val; + double double_val; + log_string_type string_val; + } val; +}; + +class LogWriter; + +class LogMgr { +public: + LogMgr(); + ~LogMgr(); + + // These correspond to the BiFs visible on the scripting layer. The + // actual BiFs just forward here. + bool CreateStream(EnumVal* stream_id, RecordType* columns); + bool AddFilter(EnumVal* stream_id, RecordVal* filter); + bool RemoveFilter(EnumVal* stream_id, StringVal* filter); + bool Write(EnumVal* stream_id, RecordVal* columns); + +protected: + friend class LogWriter; + + /// Functions also used by the writers. + + // Reports an error for the given writer. + void Error(LogWriter* writer, const char* msg); + +private: + struct Filter; + struct Stream; + + bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); + LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); + + vector streams; // Indexed by stream enum. +}; + +extern LogMgr* log_mgr; + +#endif diff --git a/src/LogWriter.cc b/src/LogWriter.cc new file mode 100644 index 0000000000..903ac46f8b --- /dev/null +++ b/src/LogWriter.cc @@ -0,0 +1,76 @@ + +#include "util.h" +#include "LogWriter.h" + +LogWriter::LogWriter() + { + buf = 0; + buf_len = 1024; + } + +LogWriter::~LogWriter() + { + if ( buf ) + free(buf); + + delete [] fields; + } + +bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields) + { + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + DoInit(arg_path, arg_num_fields, arg_fields); + return true; + } + +bool LogWriter::Write(LogVal** vals) + { + bool result = DoWrite(num_fields, fields, vals); + DeleteVals(vals); + return result; + } + +void LogWriter::Finish() + { + DoFinish(); + } + +const char* LogWriter::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + + +void LogWriter::Error(const char *msg) + { + run_time(msg); + } + +void LogWriter::DeleteVals(LogVal** vals) + { + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + } + + diff --git a/src/LogWriter.h b/src/LogWriter.h new file mode 100644 index 0000000000..02a8c03ad5 --- /dev/null +++ b/src/LogWriter.h @@ -0,0 +1,82 @@ +// +// Interface API for a log writer backend. +// +// Note than classes derived from LogWriter must be fully thread-safe and not +// use any non-safe Bro functionality (which is almost all ...). In +// particular, do not use fmt() but LogWriter::Fmt()!. + +#ifndef LOGWRITER_H +#define LOGWRITER_H + +#include "LogMgr.h" +#include "BroString.h" + +class LogWriter { +public: + LogWriter(); + virtual ~LogWriter(); + + // One-time initialization of the writer, defining the logged fields. + // Interpretation of "path" is left to the writer, and will be the value + // configured on the script-level. Returns false if an error occured, in + // which case the writer must not be used futher. + // + // The new instance takes ownership of "fields", and will delete them + // when done. + bool Init(string path, int num_fields, LogField** fields); + + // Writes one log entry. The method takes ownership of "vals" and will + // return immediately after queueing the write request, potentially + // before the output has actually taken place. Returns false if an error + // occured, in which case the writer must not be used further. + bool Write(LogVal** vals); + + // Finished writing to this logger. Will not be called if an error has + // been indicated earlier. After calling this, no more writing must be + // performed. + void Finish(); + +protected: + + //// Methods for Writers to override. + + // Called once for initialization of the Writer. Must return false if an + // error occured, in which case the writer will be disabled. The error + // reason should be reported via Error(). + virtual bool DoInit(string path, int num_fields, LogField** fields) = 0; + + // Called once per entry to record. Must return false if an error + // occured, in which case the writer will be disabled. The error reason + // should be reported via Error(). + virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; + + // Called once on termination. Not called when any of the other methods + // has previously signaled an error, i.e., executing this method signals + // a regular shutdown. + virtual void DoFinish() = 0; + + //// Methods for Writers to use. These are thread-safe. + + // A thread-safe version of fmt(). + const char* Fmt(const char* format, ...); + + // Reports an error. + void Error(const char *msg); + + // Returns the path as passed to Init(). + const string Path() const { return path; } + +private: + // Delete values as passed into Write(). + void DeleteVals(LogVal** vals); + + string path; + int num_fields; + LogField** fields; + + // For Fmt(). + char* buf; + unsigned int buf_len; +}; + +#endif diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc new file mode 100644 index 0000000000..e06ba4f9dd --- /dev/null +++ b/src/LogWriterAscii.cc @@ -0,0 +1,129 @@ + +#include +#include + +#include "LogWriterAscii.h" + +LogWriterAscii::LogWriterAscii() + { + fname = 0; + file = 0; + } + +LogWriterAscii::~LogWriterAscii() + { + if ( fname ) + free(fname); + + if ( file ) + fclose(file); + } + +bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) + { + fname = strdup(Fmt("%s.log", path.c_str())); + + if ( ! (file = fopen(fname, "w")) ) + { + Error(Fmt("cannot open %s: %s", fname, strerror(errno))); + return false; + } + + if ( fputs("# ", file) == EOF ) + goto write_error; + + for ( int i = 0; i < num_fields; i++ ) + { + LogField* field = fields[i]; + if ( fputs(field->name.c_str(), file) == EOF ) + goto write_error; + + if ( fputc('\t', file) == EOF ) + goto write_error; + } + + if ( fputc('\n', file) == EOF ) + goto write_error; + + return true; + +write_error: + Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + return false; + } + +void LogWriterAscii::DoFinish() + { + } + +bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) + { + ODesc desc(DESC_READABLE); + + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 ) + desc.Add("\t"); + + LogVal* val = vals[i]; + LogField* field = fields[i]; + + if ( ! val->present ) + { + desc.Add("-"); // TODO: Probably want to get rid of the "-". + continue; + } + + switch ( field->type ) { + case TYPE_BOOL: + desc.Add(val->val.int_val ? "T" : "F"); + break; + + case TYPE_INT: + case TYPE_ENUM: + desc.Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + desc.Add(val->val.uint_val); + break; + + case TYPE_SUBNET: + desc.Add(dotted_addr(val->val.subnet_val.net)); + desc.Add("/"); + desc.Add(val->val.subnet_val.width); + break; + + case TYPE_NET: + case TYPE_ADDR: + desc.Add(dotted_addr(val->val.addr_val)); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + desc.Add(val->val.double_val); + break; + + case TYPE_STRING: + desc.AddN((const char*)&val->val.string_val.string, val->val.string_val.len); + break; + + default: + Error(Fmt("unsupported field format %d for %s", field->type, field->name.c_str())); + return false; + } + } + + desc.Add("\n"); + + if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 ) + { + Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + return false; + } + + return true; + } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h new file mode 100644 index 0000000000..4672d2ac54 --- /dev/null +++ b/src/LogWriterAscii.h @@ -0,0 +1,27 @@ +// +// Log writer for tab-separated ASCII logs. +// + +#ifndef LOGWRITERASCII_H +#define LOGWRITERASCII_H + +#include "LogWriter.h" + +class LogWriterAscii : public LogWriter { +public: + LogWriterAscii(); + ~LogWriterAscii(); + + static LogWriter* Instantiate() { return new LogWriterAscii; } + +protected: + bool DoInit(string path, int num_fields, LogField** fields); + bool DoWrite(int num_fields, LogField** fields, LogVal** vals); + void DoFinish(); + +private: + FILE* file; + char* fname; +}; + +#endif diff --git a/src/NetVar.cc b/src/NetVar.cc index 0af742ef3e..231d2efe78 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -260,6 +260,8 @@ int record_all_packets; RecordType* script_id; TableType* id_table; +RecordType* log_filter; + #include "const.bif.netvar_def" #include "event.bif.netvar_def" @@ -564,4 +566,6 @@ void init_net_var() script_id = internal_type("script_id")->AsRecordType(); id_table = internal_type("id_table")->AsTableType(); + + log_filter = internal_type("log_filter")->AsRecordType(); } diff --git a/src/NetVar.h b/src/NetVar.h index 7461ec8be0..ccd8294882 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,6 +264,8 @@ extern int record_all_packets; extern RecordType* script_id; extern TableType* id_table; +extern RecordType* log_filter; + // Initializes globals that don't pertain to network/event analysis. extern void init_general_global_var(); diff --git a/src/bro.bif b/src/bro.bif index 278c14500e..52c54706c6 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -359,7 +359,7 @@ function cat%(...%): string return new StringVal(s); %} - + function logging_log%(index: string, rec: any%): any %{ // TODO: Verify that rec is a record @@ -384,7 +384,7 @@ function logging_log%(index: string, rec: any%): any printf("Logging framework is dead (Logging::streams not found).\n"); return false; } - + // Lookup all filters for stream TableVal *filters = opt_internal_table("Logging::filters"); TableVal *stream_filters; @@ -399,8 +399,8 @@ function logging_log%(index: string, rec: any%): any printf("Logging framework is dead (Logging::filters not found).\n"); return false; } - - // Generate the event for the log stream + + // Generate the event for the log stream // This happens regardless of all filters. int name_field = stream_record->Type()->AsRecordType()->FieldOffset("name"); StringVal *log_type = stream_record->AsRecordVal()->Lookup(name_field)->AsStringVal(); @@ -414,7 +414,7 @@ function logging_log%(index: string, rec: any%): any vl->append(recval->Ref()); mgr.QueueEvent(ev_ptr, vl, SOURCE_LOCAL); } - + // Iterate over the stream_filters ListVal* filter_recs = stream_filters->ConvertToList(TYPE_ANY); for ( int i = 0; i < filter_recs->Length(); ++i ) @@ -432,10 +432,10 @@ function logging_log%(index: string, rec: any%): any if ( !pred_val ) continue; } - - // Format and print the line + + // Format and print the line // (send line onward to the filter's WRITER in the future) - + // Get a path name for this filter int path_field = rv->Type()->AsRecordType()->FieldOffset("path"); Val *path_val = rv->Lookup(path_field); @@ -445,7 +445,7 @@ function logging_log%(index: string, rec: any%): any path = path_val->AsStringVal(); else path = index; - + // Get the file with the "path" name found above for this filter. // Open a new file is one does not exist yet. TableVal *logging_files = opt_internal_table("Logging::files"); @@ -474,20 +474,61 @@ function logging_log%(index: string, rec: any%): any } f->Write(d.Description(), 0); f->Write("\n",0); - - + + } Unref(filter_recs); - + return 0; %} - + + +%%{ +#include "LogMgr.h" +%%} + +function log_create_stream%(id: Log_ID, columns: any%) : bool + %{ + if ( columns->Type()->Tag() != TYPE_TYPE ) + { + run_time("log columns must be a type"); + return new Val(0, TYPE_BOOL); + } + + if ( columns->Type()->AsTypeType()->Type()->Tag() != TYPE_RECORD ) + { + run_time("log columns must be a record type"); + return new Val(0, TYPE_BOOL); + } + + bool result = log_mgr->CreateStream(id->AsEnumVal(), columns->Type()->AsTypeType()->Type()->AsRecordType()); + return new Val(result, TYPE_BOOL); + %} + +function log_add_filter%(id: Log_ID, filter: log_filter%) : bool + %{ + bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + +function log_remove_filter%(id: Log_ID, name: string%) : bool + %{ + bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name); + return new Val(result, TYPE_BOOL); + %} + +function log_write%(id: Log_ID, columns: any%) : bool + %{ + bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + function record_type_to_vector%(rt: string%): string_vec %{ VectorVal* result = new VectorVal(internal_type("string_vec")->AsVectorType()); - + RecordType *type = internal_type(rt->CheckString())->AsRecordType(); if ( type ) { @@ -499,8 +540,8 @@ function record_type_to_vector%(rt: string%): string_vec } return result; %} - - + + function cat_sep%(sep: string, def: string, ...%): string %{ ODesc d; @@ -3164,7 +3205,7 @@ function lookup_location%(a: addr%) : geo_location #ifdef GEOIP_COUNTRY_EDITION_V6 if ( geoip_v6 && ! is_v4_addr(a) ) gir = GeoIP_record_by_ipnum_v6(geoip_v6, geoipv6_t(a)); - else + else #endif if ( geoip && is_v4_addr(a) ) { @@ -3252,7 +3293,7 @@ function lookup_asn%(a: addr%) : count gir = GeoIP_name_by_ipnum_v6(geoip_asn, geoipv6_t(a)); else #endif - if ( is_v4_addr(a) ) + if ( is_v4_addr(a) ) { uint32 addr = to_v4_addr(a); gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(addr)); @@ -3264,7 +3305,7 @@ function lookup_asn%(a: addr%) : count if ( gir ) { - // Move the pointer +2 so we don't return + // Move the pointer +2 so we don't return // the first two characters: "AS". return new Val(atoi(gir+2), TYPE_COUNT); } diff --git a/src/logging.bif b/src/logging.bif new file mode 100644 index 0000000000..955b3c293f --- /dev/null +++ b/src/logging.bif @@ -0,0 +1,23 @@ + +enum log_writer %{ + WRITER_DEFAULT, + WRITER_TSV, + WRITER_SYSLOG, +}% + +void log_create%(id: Logging::Stream, columns: any%) + %{ + %} + +void log_add_filter%(id: Logging::Stream, filter: any%) + %{ + %} + +void log_remove_filter%(id: Logging::Stream, name: string%) + %{ + %} + +void log_write%(id: Logging::Stream, columns: any%) + %{ + %} + diff --git a/src/logging.bro b/src/logging.bro new file mode 100644 index 0000000000..6031826351 --- /dev/null +++ b/src/logging.bro @@ -0,0 +1,52 @@ + +module Log; + +export { + # Each stream gets a unique ID. This type will be extended by + # other scripts. + type Stream: enum { + Unknown, + Info, + Debug, + }; + + # The default writer to use if a filter does not specify + # anything else. + const default_writer = WRITER_CSV &redef; + + # A filter defining what to record. + type Filter: record { + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(log: any) &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: any &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + select: set[string] &optional; + + # An event that is raised whenever the filter is applied + # to an entry. The event receives the same parameter + # as the predicate. It will always be generated, + # independent of what the predicate returns. + ev: event(l: any) &optional; + + # The writer to use. + writer: Writer &default=default_writer; + }; +} diff --git a/src/main.cc b/src/main.cc index 82866302fd..141eec7ab3 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Event.h" #include "File.h" #include "Logger.h" +#include "LogMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -71,6 +72,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; Logger* bro_logger; +LogMgr* log_mgr; Func* alarm_hook = 0; Stmt* stmts; EventHandlerPtr bro_signal = 0; @@ -289,6 +291,7 @@ void terminate_bro() delete conn_compressor; delete remote_serializer; delete dpm; + delete log_mgr; } void termination_signal() @@ -724,7 +727,8 @@ int main(int argc, char** argv) persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); - event_registry = new EventRegistry; + event_registry = new EventRegistry(); + log_mgr = new LogMgr(); if ( events_file ) event_player = new EventPlayer(events_file);