From f3b148b019ee3db1ce82c92161e3aa05e45073a8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 19 Jan 2011 09:36:06 -0500 Subject: [PATCH 01/67] Checkpoint for logging framework --- policy/logging.bro | 93 +++++++++++++++++++++++++++++++++++++++++ policy/test-logging.bro | 27 ++++++++++++ src/Type.cc | 4 +- src/Val.cc | 4 ++ src/bro.bif | 12 ++++++ 5 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 policy/logging.bro create mode 100644 policy/test-logging.bro diff --git a/policy/logging.bro b/policy/logging.bro new file mode 100644 index 0000000000..2a61a12a63 --- /dev/null +++ b/policy/logging.bro @@ -0,0 +1,93 @@ +module Logging; + +export { + # The set of writers Bro provides. + type Writer: enum { + WRITER_DEFAULT, # See default_writer below. + WRITER_CSV, + WRITER_DATA_SERIES, + WRITER_SYSLOG + }; + + # Each stream gets a unique ID. This type will be extended by + # other scripts. + type ID: enum { + Unknown + }; + + # The default writer to use if a filter does not specify + # anything else. + const default_writer = WRITER_CSV &redef; + + # Type defining a stream. + type Stream: record { + id : ID; # The ID of the stream. + columns : any; # A record type defining the stream's output columns. + }; + + # A filter defining what to record. + type Filter: record { + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(log: any) &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: any &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + select: set[string] &optional; + + # An event that is raised whenever the filter is applied + # to an entry. The event receives the same parameter + # as the predicate. It will always be generated, + # independent of what the predicate returns. + ev: event(c: connection, log: any) &optional; + + # The writer to use. + writer: Writer &default=default_writer; + }; + + global filters: table[ID] of set[Filter]; + + # Logs the record "rec" to the stream "id". The type of + # "rec" must match the stream's "columns" field. + global log: function(id: ID, rec: any); + + # Returns an existing filter previously installed for stream + # "id" under the given "name". If no such filter exists, + # the record "NoSuchFilter" is returned. + global get_filter: function(id: ID, name: string) : Filter; + + global create: function(stream: Stream); + global add_filter: function(id: ID, filter: Filter); +} + +# Sentinel representing an unknown filter. +const NoSuchFilter: Filter = [$name=""]; + +function add_filter(id: ID, filter: Filter) + { + if ( id !in filters ) + filters[id] = set(); + + add filters[id][filter]; + } + +function log(id: ID, rec: any) + { + logging_log(id, rec); + } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro new file mode 100644 index 0000000000..267f9c36a5 --- /dev/null +++ b/policy/test-logging.bro @@ -0,0 +1,27 @@ +module SSH; + +@load logging + +# Create a new ID for our log stream +redef enum Logging::ID += { SSH }; + +# Define a record with all the columns the log file can have. +# (I'm using a subset of fields from ssh-ext for demonstration.) +type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; +}; + +event bro_init() +{ + # Create the stream. + Logging::create([$id=SSH, $columns=Log]); + + # Add a default filter that simply logs everything to "ssh.log" using the default writer. + #Logging::add_filter(SSH, [$name="default", $path="ssh"]); +} + +# Log something. +Logging::log(SSH, [$t=network_time(), $status="ok"]); diff --git a/src/Type.cc b/src/Type.cc index 1f5c22d58b..7e08c4b140 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1336,7 +1336,9 @@ static int is_init_compat(const BroType* t1, const BroType* t2) int same_type(const BroType* t1, const BroType* t2, int is_init) { - if ( t1 == t2 ) + if ( t1 == t2 || + t1->Tag() == TYPE_ANY || + t2->Tag() == TYPE_ANY ) return 1; t1 = flatten_type(t1); diff --git a/src/Val.cc b/src/Val.cc index f43bafe4d7..307bad61d1 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -3300,6 +3300,10 @@ Val* check_and_promote(Val* v, const BroType* t, int is_init) TypeTag t_tag = t->Tag(); TypeTag v_tag = vt->Tag(); + + // More thought definitely needs to go into this. + if ( t_tag == TYPE_ANY || v_tag == TYPE_ANY ) + return v; if ( ! EitherArithmetic(t_tag, v_tag) || /* allow sets as initializers */ diff --git a/src/bro.bif b/src/bro.bif index 0de77bfc49..78e83b920b 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -359,6 +359,18 @@ function cat%(...%): string return new StringVal(s); %} + +function logging_log%(id: Logging_ID, rec: any%): bool + %{ + // Generate the log line event + // Lookup the log file + TableVal *f = opt_internal_table("Logging::filters"); + TableVal *s = f->Lookup(id, 0)->AsTableVal(); + + // For each filter on 'id' + // Format the output + // Print the line + %} function cat_sep%(sep: string, def: string, ...%): string %{ From 4df961aa60296214af90232f236a4e2587e7e880 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 21 Jan 2011 16:46:06 -0500 Subject: [PATCH 02/67] Log specific event and debug log printing is working! This is mostly a code checkpoint though. --- policy/logging.bro | 55 +++++++++++++++++++------- policy/test-logging.bro | 37 +++++++++++------- src/bro.bif | 87 +++++++++++++++++++++++++++++++++++++---- 3 files changed, 142 insertions(+), 37 deletions(-) diff --git a/policy/logging.bro b/policy/logging.bro index 2a61a12a63..10690a8fbd 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -20,10 +20,10 @@ export { const default_writer = WRITER_CSV &redef; # Type defining a stream. - type Stream: record { - id : ID; # The ID of the stream. - columns : any; # A record type defining the stream's output columns. - }; + #type Stream: record { + # id : string; # The ID of the stream. + # columns : string_vec; # A record type defining the stream's output columns. + #}; # A filter defining what to record. type Filter: record { @@ -61,33 +61,58 @@ export { writer: Writer &default=default_writer; }; - global filters: table[ID] of set[Filter]; + global filters: table[string] of set[Filter]; + global streams: table[string] of string_vec; # Logs the record "rec" to the stream "id". The type of # "rec" must match the stream's "columns" field. - global log: function(id: ID, rec: any); + global log: function(id: string, rec: any); + global log_ev: event(id: string, rec: any); # Returns an existing filter previously installed for stream # "id" under the given "name". If no such filter exists, # the record "NoSuchFilter" is returned. - global get_filter: function(id: ID, name: string) : Filter; + global get_filter: function(id: string, name: string) : Filter; + + global create_stream: function(id: string, columns: string); + global add_filter: function(id: string, filter: Filter); + + global open_log_files: function(id: string); - global create: function(stream: Stream); - global add_filter: function(id: ID, filter: Filter); } # Sentinel representing an unknown filter. const NoSuchFilter: Filter = [$name=""]; -function add_filter(id: ID, filter: Filter) +function create_stream(id: string, columns: string) { - if ( id !in filters ) - filters[id] = set(); + if ( id in streams ) + print fmt("Stream %s already exists!", id); - add filters[id][filter]; + streams[id] = record_type_to_vector(columns); } - -function log(id: ID, rec: any) + +function add_filter(id: string, filter: Filter) + { + #if ( id !in filters ) + # filters[id] = set(); + # + #add filters[id][filter]; + } + +function log(id: string, rec: any) { logging_log(id, rec); + } + + +# THIS IS ONLY FOR THE PROTOTYPE. +# It will be implemented in the core later +function open_log_files(id: string) + { + # Open default log + #open_log_file(id); + + # Find all second names from filters + # Open log for each secondary name } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro index 267f9c36a5..0bb97b962f 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -2,26 +2,35 @@ module SSH; @load logging -# Create a new ID for our log stream -redef enum Logging::ID += { SSH }; +export { + # Create a new ID for our log stream + redef enum Logging::ID += { LOG_SSH }; -# Define a record with all the columns the log file can have. -# (I'm using a subset of fields from ssh-ext for demonstration.) -type Log: record { - t: time; - id: conn_id; # Will be rolled out into individual columns. - status: string &optional; - country: string &default="unknown"; -}; + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; + + global ssh_log: event(rec: Log); +} event bro_init() { # Create the stream. - Logging::create([$id=SSH, $columns=Log]); + Logging::create_stream("ssh", "SSH::Log"); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - #Logging::add_filter(SSH, [$name="default", $path="ssh"]); + #Logging::add_filter("SSH", [$name="default", $path="ssh"]); + + # Log something. + Logging::log("ssh", [$t=network_time(), $country="US", $status="ok"]); } -# Log something. -Logging::log(SSH, [$t=network_time(), $status="ok"]); +event ssh_log(rec: Log) + { + print "Ran the ssh_log handler! kick ass"; + } \ No newline at end of file diff --git a/src/bro.bif b/src/bro.bif index c9e40f6278..c1eeb4145e 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -360,18 +360,89 @@ function cat%(...%): string return new StringVal(s); %} -function logging_log%(id: Logging_ID, rec: any%): bool +function logging_log%(index: string, rec: any%): bool %{ - // Generate the log line event - // Lookup the log file - TableVal *f = opt_internal_table("Logging::filters"); - TableVal *s = f->Lookup(id, 0)->AsTableVal(); + // Verify that rec is a record + // Lookup the stream + TableVal *streams = opt_internal_table("Logging::streams"); + VectorVal *columns; + if ( streams ) + { + Val *lookup_v = streams->Lookup(index); + if ( lookup_v ) + columns = lookup_v->AsVectorVal(); + } + else + { + printf("Logging framework is dead (Logging::streams not found).\n"); + return false; + } + // Generate the event for the log stream + // TODO: make it actually figure out the right handler name. + EventHandlerPtr ev_ptr = internal_handler("SSH::ssh_log"); + if ( ev_ptr ) + { + val_list* vl = new val_list; + vl->append(rec->Ref()); + mgr.QueueEvent(, vl, SOURCE_LOCAL); + } + + // Lookup all filters for stream + TableVal *filters = opt_internal_table("Logging::filters"); + RecordVal *stream_filters; + if ( filters ) + { + Val *lookup_v = filters->Lookup(index); + if ( lookup_v ) + stream_filters = lookup_v->AsRecordVal(); + } + else + { + printf("Logging framework is dead (Logging::filters not found).\n"); + return false; + } + + ODesc d; + const char *field_name; + int field = 0; + RecordType *rt = rec->Type()->AsRecordType(); + for ( unsigned i = 1; i <= columns->Size(); ++i ) + { + field_name = columns->Lookup(i)->AsStringVal()->CheckString(); + field = rec->Type()->AsRecordType()->FieldOffset(field_name); + if ( field >= 0 ) // or if there is a default value + { + rec->AsRecordVal()->Lookup(field)->Describe(&d); + d.Add("\t",0); + } + //printf("Test: %s\n", field_name); + } + printf("Full line: %s\n", d.TakeBytes()); // For each filter on 'id' - // Format the output - // Print the line + // Format the output (iterate through columns and grab fields from rec as found) + // Print the line (send line onward to WRITER) + return false; %} - + +function record_type_to_vector%(rt: string%): string_vec + %{ + VectorVal* result = + new VectorVal(internal_type("string_vec")->AsVectorType()); + + RecordType *type = internal_type(rt->CheckString())->AsRecordType(); + if ( type ) + { + for ( int i = 0; i < type->NumFields(); ++i ) + { + StringVal* val = new StringVal(type->FieldName(i)); + result->Assign(i+1, val, 0); + } + } + return result; + %} + + function cat_sep%(sep: string, def: string, ...%): string %{ ODesc d; From d2628d30fa3c9ebfd37d2276b5db7d333c6e4142 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 26 Jan 2011 15:29:20 -0500 Subject: [PATCH 03/67] Logging framework core functionality now implemented. --- policy/logging.bro | 118 +++++++++++++++++++--------------------- policy/test-logging.bro | 23 ++++++-- src/bro.bif | 65 +++++++++++++--------- 3 files changed, 112 insertions(+), 94 deletions(-) diff --git a/policy/logging.bro b/policy/logging.bro index 10690a8fbd..26f9c01640 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -3,66 +3,67 @@ module Logging; export { # The set of writers Bro provides. type Writer: enum { - WRITER_DEFAULT, # See default_writer below. - WRITER_CSV, - WRITER_DATA_SERIES, - WRITER_SYSLOG + WRITER_DEFAULT, # See default_writer below. + WRITER_CSV, + WRITER_DATA_SERIES, + WRITER_SYSLOG }; # Each stream gets a unique ID. This type will be extended by # other scripts. type ID: enum { - Unknown - }; - + Unknown + }; + # The default writer to use if a filter does not specify # anything else. const default_writer = WRITER_CSV &redef; # Type defining a stream. - #type Stream: record { - # id : string; # The ID of the stream. - # columns : string_vec; # A record type defining the stream's output columns. - #}; + type Stream: record { + name: string; + columns: string_vec; + }; # A filter defining what to record. type Filter: record { - # A name to reference this filter. - name: string; - - # A predicate returning True if the filter wants a log entry - # to be recorded. If not given, an implicit True is assumed - # for all entries. The predicate receives one parameter: - # an instance of the log's record type with the fields to be - # logged. - pred: function(log: any) &optional; - - # A path for outputting everything matching this - # filter. The path is either a string, or a function - # called with a single ``ID`` argument and returning a string. - # - # The specific interpretation of the string is left to the - # Writer, but if it's refering to a file, it's assumed that no - # extension is given; the writer will add whatever is - # appropiate. - path: any &optional; - - # A subset of column names to record. If not given, all - # columns are recorded. - select: set[string] &optional; - - # An event that is raised whenever the filter is applied - # to an entry. The event receives the same parameter - # as the predicate. It will always be generated, - # independent of what the predicate returns. - ev: event(c: connection, log: any) &optional; - - # The writer to use. - writer: Writer &default=default_writer; - }; + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(log: any) &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: any &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + select: set[string] &optional; + + # An event that is raised whenever the filter is applied + # to an entry. The event receives the same parameter + # as the predicate. It will always be generated, + # independent of what the predicate returns. + ev: event(l: any) &optional; + + # The writer to use. + writer: Writer &default=default_writer; + }; + global filters: table[string] of set[Filter]; - global streams: table[string] of string_vec; + global streams: table[string] of Stream; # Logs the record "rec" to the stream "id". The type of # "rec" must match the stream's "columns" field. @@ -74,7 +75,7 @@ export { # the record "NoSuchFilter" is returned. global get_filter: function(id: string, name: string) : Filter; - global create_stream: function(id: string, columns: string); + global create_stream: function(id: string, log_record_type: string); global add_filter: function(id: string, filter: Filter); global open_log_files: function(id: string); @@ -84,20 +85,22 @@ export { # Sentinel representing an unknown filter. const NoSuchFilter: Filter = [$name=""]; -function create_stream(id: string, columns: string) +function create_stream(id: string, log_record_type: string) { if ( id in streams ) print fmt("Stream %s already exists!", id); - streams[id] = record_type_to_vector(columns); + streams[id] = [$name=log_record_type, $columns=record_type_to_vector(log_record_type)]; } function add_filter(id: string, filter: Filter) { - #if ( id !in filters ) - # filters[id] = set(); - # - #add filters[id][filter]; + if ( id !in filters ) + filters[id] = set(); + + # TODO: This is broken and waiting on a bug fix for &optional fields + # in records being used as indexes. + #add filt[filter]; } function log(id: string, rec: any) @@ -105,14 +108,3 @@ function log(id: string, rec: any) logging_log(id, rec); } - -# THIS IS ONLY FOR THE PROTOTYPE. -# It will be implemented in the core later -function open_log_files(id: string) - { - # Open default log - #open_log_file(id); - - # Find all second names from filters - # Open log for each secondary name - } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro index 0bb97b962f..dde9bb8a2f 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -15,22 +15,37 @@ export { country: string &default="unknown"; }; - global ssh_log: event(rec: Log); + # This is the prototype for the event that the logging framework tries + # to generate if there is a handler for it. + global log: event(rec: Log); } event bro_init() { # Create the stream. + # First argument is the ID for the stream. + # Second argument is the log record type. Logging::create_stream("ssh", "SSH::Log"); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - #Logging::add_filter("SSH", [$name="default", $path="ssh"]); + # Filtering is not implemented yet. Waiting on ticket #366 + # Log line event generation is autogenerated for now by checking for + # handlers for MODULE_NAME::log + #Logging::add_filter("ssh", [$name="default", $path="ssh", $ev=log]); # Log something. Logging::log("ssh", [$t=network_time(), $country="US", $status="ok"]); } -event ssh_log(rec: Log) +event log(rec: Log) { - print "Ran the ssh_log handler! kick ass"; + print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); + } + + +module WHATEVER; + +event SSH::log(rec: SSH::Log) + { + print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); } \ No newline at end of file diff --git a/src/bro.bif b/src/bro.bif index c1eeb4145e..ec14775a69 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -360,17 +360,22 @@ function cat%(...%): string return new StringVal(s); %} -function logging_log%(index: string, rec: any%): bool +function logging_log%(index: string, rec: any%): any %{ // Verify that rec is a record + // Lookup the stream TableVal *streams = opt_internal_table("Logging::streams"); VectorVal *columns; + RecordVal *stream_record; if ( streams ) { - Val *lookup_v = streams->Lookup(index); - if ( lookup_v ) - columns = lookup_v->AsVectorVal(); + stream_record = streams->Lookup(index)->AsRecordVal(); + if ( stream_record ) + { + int columns_field = stream_record->Type()->AsRecordType()->FieldOffset("columns"); + columns = stream_record->Lookup(columns_field)->AsVectorVal(); + } } else { @@ -378,31 +383,40 @@ function logging_log%(index: string, rec: any%): bool return false; } - // Generate the event for the log stream - // TODO: make it actually figure out the right handler name. - EventHandlerPtr ev_ptr = internal_handler("SSH::ssh_log"); + // Generate the event for the log stream + // This happens regardless of all filters. + int name_field = stream_record->Type()->AsRecordType()->FieldOffset("name"); + StringVal *log_type = stream_record->AsRecordVal()->Lookup(name_field)->AsStringVal(); + string ID_module = extract_module_name(log_type->CheckString()); + // The log event that is generated by default is MODULE_NAME::log + string log_event_name = make_full_var_name(ID_module.c_str(), "log"); + EventHandlerPtr ev_ptr = internal_handler(log_event_name.c_str()); if ( ev_ptr ) { val_list* vl = new val_list; vl->append(rec->Ref()); - mgr.QueueEvent(, vl, SOURCE_LOCAL); + mgr.QueueEvent(ev_ptr, vl, SOURCE_LOCAL); } // Lookup all filters for stream - TableVal *filters = opt_internal_table("Logging::filters"); - RecordVal *stream_filters; - if ( filters ) - { - Val *lookup_v = filters->Lookup(index); - if ( lookup_v ) - stream_filters = lookup_v->AsRecordVal(); - } - else - { - printf("Logging framework is dead (Logging::filters not found).\n"); - return false; - } + // (ignore this code, it will probably be done in the logging.bro script + // with the "match" statement) + //TableVal *filters = opt_internal_table("Logging::filters"); + //RecordVal *stream_filters; + //if ( filters ) + // { + // Val *lookup_v = filters->Lookup(index); + // if ( lookup_v ) + // stream_filters = lookup_v->AsRecordVal(); + // } + //else + // { + // printf("Logging framework is dead (Logging::filters not found).\n"); + // return false; + // } + // Print the line + // (send line onward to the filter's WRITER in the future) ODesc d; const char *field_name; int field = 0; @@ -416,13 +430,10 @@ function logging_log%(index: string, rec: any%): bool rec->AsRecordVal()->Lookup(field)->Describe(&d); d.Add("\t",0); } - //printf("Test: %s\n", field_name); } - printf("Full line: %s\n", d.TakeBytes()); - // For each filter on 'id' - // Format the output (iterate through columns and grab fields from rec as found) - // Print the line (send line onward to WRITER) - return false; + + printf("%s: %s\n", ID_module.c_str(), d.TakeBytes()); + return 0; %} function record_type_to_vector%(rt: string%): string_vec From 7abd8f177f1d6a082cf2539266cd53d730f664cf Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 2 Feb 2011 18:06:02 -0800 Subject: [PATCH 04/67] Fixing a problem with records having optional fields when used as table/set indices. This addresses #367. In principle, the fix is quite straightford. However, it turns out that sometimes record fields lost their attributes on assignment, and then the hashing can't decide anymore whether a field is optional or not. So that needed to be fixed as well. --- src/Attr.cc | 35 ++++++++++++++++++++++++ src/Attr.h | 16 +++++++++++ src/CompHash.cc | 62 +++++++++++++++++++++++++++++++++--------- src/CompHash.h | 6 ++--- src/Expr.cc | 71 +++++++++++++++++++++++++++++++++++++------------ src/Type.cc | 11 ++++++++ src/Type.h | 3 +++ 7 files changed, 171 insertions(+), 33 deletions(-) diff --git a/src/Attr.cc b/src/Attr.cc index 5a83d0501b..e03367c41e 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -327,6 +327,41 @@ void Attributes::CheckAttr(Attr* a) } } +bool Attributes::operator==(const Attributes& other) const + { + if ( ! attrs ) + return other.attrs; + + if ( ! other.attrs ) + return false; + + loop_over_list(*attrs, i) + { + Attr* a = (*attrs)[i]; + Attr* o = other.FindAttr(a->Tag()); + + if ( ! o ) + return false; + + if ( ! (*a == *o) ) + return false; + } + + loop_over_list(*other.attrs, j) + { + Attr* o = (*other.attrs)[j]; + Attr* a = FindAttr(o->Tag()); + + if ( ! a ) + return false; + + if ( ! (*a == *o) ) + return false; + } + + return true; + } + bool Attributes::Serialize(SerialInfo* info) const { return SerialObj::Serialize(info); diff --git a/src/Attr.h b/src/Attr.h index 73fb101841..26231baeb4 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -52,6 +52,20 @@ public: void Describe(ODesc* d) const; + bool operator==(const Attr& other) const + { + if ( tag != other.tag ) + return false; + + if ( expr || other.expr ) + // If any has an expression and they aren't the same object, we + // declare them unequal, as we can't really find out if the two + // expressions are equivalent. + return (expr == other.expr); + + return true; + } + protected: void AddTag(ODesc* d) const; @@ -79,6 +93,8 @@ public: bool Serialize(SerialInfo* info) const; static Attributes* Unserialize(UnserialInfo* info); + bool operator==(const Attributes& other) const; + protected: Attributes() { type = 0; attrs = 0; } void CheckAttr(Attr* attr); diff --git a/src/CompHash.cc b/src/CompHash.cc index 2e0870303c..cc4f440e06 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -65,11 +65,22 @@ CompositeHash::~CompositeHash() // Computes the piece of the hash for Val*, returning the new kp. char* CompositeHash::SingleValHash(int type_check, char* kp0, - BroType* bt, Val* v) const + BroType* bt, Val* v, bool optional) const { char* kp1 = 0; InternalTypeTag t = bt->InternalType(); + if ( optional ) + { + // Add a marker saying whether the optional field is set. + char* kp = AlignAndPadType(kp0); + *kp = ( v ? 1 : 0); + kp0 = reinterpret_cast(kp+1); + + if ( ! v ) + return kp0; + } + if ( type_check ) { InternalTypeTag vt = v->Type()->InternalType(); @@ -163,12 +174,16 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0, for ( int i = 0; i < num_fields; ++i ) { Val* rv_i = rv->Lookup(i); - if ( ! rv_i ) + + Attributes* a = rt->FieldDecl(i)->attrs; + bool optional = (a && a->FindAttr(ATTR_OPTIONAL)); + + if ( ! (rv_i || optional) ) return 0; if ( ! (kp = SingleValHash(type_check, kp, rt->FieldType(i), - rv_i)) ) + rv_i, optional)) ) return 0; } @@ -248,7 +263,7 @@ HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const char* kp = k; loop_over_list(*tl, i) { - kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i]); + kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i], false); if ( ! kp ) return 0; } @@ -315,10 +330,13 @@ HashKey* CompositeHash::ComputeSingletonHash(const Val* v, int type_check) const } int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v, - int type_check, int sz) const + int type_check, int sz, bool optional) const { InternalTypeTag t = bt->InternalType(); + if ( optional ) + sz = SizeAlign(sz, sizeof(char)); + if ( type_check && v ) { InternalTypeTag vt = v->Type()->InternalType(); @@ -369,9 +387,12 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v, for ( int i = 0; i < num_fields; ++i ) { + Attributes* a = rt->FieldDecl(i)->attrs; + bool optional = (a && a->FindAttr(ATTR_OPTIONAL)); + sz = SingleTypeKeySize(rt->FieldType(i), rv ? rv->Lookup(i) : 0, - type_check, sz); + type_check, sz, optional); if ( ! sz ) return 0; } @@ -418,7 +439,7 @@ int CompositeHash::ComputeKeySize(const Val* v, int type_check) const loop_over_list(*tl, i) { sz = SingleTypeKeySize((*tl)[i], v ? v->AsListVal()->Index(i) : 0, - type_check, sz); + type_check, sz, false); if ( ! sz ) return 0; } @@ -495,20 +516,20 @@ ListVal* CompositeHash::RecoverVals(const HashKey* k) const loop_over_list(*tl, i) { Val* v; - kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v); + kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v, false); ASSERT(v); l->Append(v); } if ( kp != k_end ) - internal_error("under-ran key in CompositeHash::DescribeKey"); + internal_error("under-ran key in CompositeHash::DescribeKey %ld", k_end - kp); return l; } const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, const char* const k_end, BroType* t, - Val*& pval) const + Val*& pval, bool optional) const { // k->Size() == 0 for a single empty string. if ( kp0 >= k_end && k->Size() > 0 ) @@ -516,9 +537,20 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, TypeTag tag = t->Tag(); InternalTypeTag it = t->InternalType(); - const char* kp1 = 0; + if ( optional ) + { + const char* kp = AlignType(kp0); + kp0 = kp1 = reinterpret_cast(kp+1); + + if ( ! *kp ) + { + pval = 0; + return kp0; + } + } + switch ( it ) { case TYPE_INTERNAL_INT: { @@ -647,9 +679,13 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, for ( i = 0; i < num_fields; ++i ) { Val* v; + + Attributes* a = rt->FieldDecl(i)->attrs; + bool optional = (a && a->FindAttr(ATTR_OPTIONAL)); + kp = RecoverOneVal(k, kp, k_end, - rt->FieldType(i), v); - if ( ! v ) + rt->FieldType(i), v, optional); + if ( ! (v || optional) ) { internal_error("didn't recover expected number of fields from HashKey"); pval = 0; diff --git a/src/CompHash.h b/src/CompHash.h index a0632e1bfe..12ab9f7422 100644 --- a/src/CompHash.h +++ b/src/CompHash.h @@ -30,7 +30,7 @@ protected: // Computes the piece of the hash for Val*, returning the new kp. // Used as a helper for ComputeHash in the non-singleton case. char* SingleValHash(int type_check, char* kp, - BroType* bt, Val* v) const; + BroType* bt, Val* v, bool optional) const; // Recovers just one Val of possibly many; called from RecoverVals. // Upon return, pval will point to the recovered Val of type t. @@ -38,7 +38,7 @@ protected: // upon errors, so there is no return value for invalid input. const char* RecoverOneVal(const HashKey* k, const char* kp, const char* const k_end, - BroType* t, Val*& pval) const; + BroType* t, Val*& pval, bool optional) const; // Rounds the given pointer up to the nearest multiple of the // given size, if not already a multiple. @@ -77,7 +77,7 @@ protected: int ComputeKeySize(const Val* v = 0, int type_check = 1) const; int SingleTypeKeySize(BroType*, const Val*, - int type_check, int sz) const; + int type_check, int sz, bool optional) const; TypeList* type; char* key; // space for composite key diff --git a/src/Expr.cc b/src/Expr.cc index dbfca7c9cb..5788cdeb7b 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2531,16 +2531,35 @@ bool AssignExpr::TypeCheck() return true; } + if ( op1->Type()->Tag() == TYPE_RECORD && + op2->Type()->Tag() == TYPE_RECORD ) + { + if ( same_type(op1->Type(), op2->Type()) ) + { + RecordType* rt1 = op1->Type()->AsRecordType(); + RecordType* rt2 = op2->Type()->AsRecordType(); + + // Make sure the attributes match as well. + for ( int i = 0; i < rt1->NumFields(); ++i ) + { + const TypeDecl* td1 = rt1->FieldDecl(i); + const TypeDecl* td2 = rt2->FieldDecl(i); + + if ( same_attrs(td1->attrs, td2->attrs) ) + // Everything matches. + return true; + } + } + + // Need to coerce. + op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType()); + return true; + } + if ( ! same_type(op1->Type(), op2->Type()) ) { - if ( op1->Type()->Tag() == TYPE_RECORD && - op2->Type()->Tag() == TYPE_RECORD ) - op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType()); - else - { - ExprError("type clash in assignment"); - return false; - } + ExprError("type clash in assignment"); + return false; } return true; @@ -5308,21 +5327,39 @@ int check_and_promote_expr(Expr*& e, BroType* t) return 1; } - else if ( ! same_type(t, et) ) + if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD ) { - if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD ) - { - RecordType* t_r = t->AsRecordType(); - RecordType* et_r = et->AsRecordType(); + RecordType* t_r = t->AsRecordType(); + RecordType* et_r = et->AsRecordType(); - if ( record_promotion_compatible(t_r, et_r) ) + if ( same_type(t, et) ) + { + // Make sure the attributes match as well. + for ( int i = 0; i < t_r->NumFields(); ++i ) { - e = new RecordCoerceExpr(e, t_r); - return 1; + const TypeDecl* td1 = t_r->FieldDecl(i); + const TypeDecl* td2 = et_r->FieldDecl(i); + + if ( same_attrs(td1->attrs, td2->attrs) ) + // Everything matches perfectly. + return 1; } } - else if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE && + if ( record_promotion_compatible(t_r, et_r) ) // Note: This is always true currently. + { + e = new RecordCoerceExpr(e, t_r); + return 1; + } + + t->Error("incompatible record types", e); + return 0; + } + + + if ( ! same_type(t, et) ) + { + if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE && et->AsTableType()->IsUnspecifiedTable() ) { e = new TableCoerceExpr(e, t->AsTableType()); diff --git a/src/Type.cc b/src/Type.cc index 55794dfce5..62e0f195ec 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1430,6 +1430,17 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) return 0; } +int same_attrs(const Attributes* a1, const Attributes* a2) + { + if ( ! a1 ) + return (a2 != 0); + + if ( ! a2 ) + return 0; + + return (*a1 == *a2); + } + int record_promotion_compatible(const RecordType* /* super_rec */, const RecordType* /* sub_rec */) { diff --git a/src/Type.h b/src/Type.h index ff4d3df9e6..7e890d9e07 100644 --- a/src/Type.h +++ b/src/Type.h @@ -509,6 +509,9 @@ inline BroType* error_type() { return base_type(TYPE_ERROR); } // test is done in the context of an initialization. extern int same_type(const BroType* t1, const BroType* t2, int is_init=0); +// True if the two attribute lists are equivalent. +extern int same_attrs(const Attributes* a1, const Attributes* a2); + // Returns true if the record sub_rec can be promoted to the record // super_rec. extern int record_promotion_compatible(const RecordType* super_rec, From 0dbbee46ae47b46bb724b151471ca11c09581fd7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Feb 2011 14:46:28 -0800 Subject: [PATCH 05/67] Teaching bifcl to accept scoped IDs. --- src/builtin-func.l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtin-func.l b/src/builtin-func.l index ca7923b852..a9459ea4a8 100644 --- a/src/builtin-func.l +++ b/src/builtin-func.l @@ -27,7 +27,7 @@ int check_c_mode(int t) %} WS [ \t]+ -ID [A-Za-z_][A-Za-z_0-9]* +ID [A-Za-z_](([A-Za-z_0-9]|::)*[A-Za-z_0-9])? ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) %option nodefault From f43766650f730b9f03bbc800e88066b8197c70a2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Feb 2011 14:46:49 -0800 Subject: [PATCH 06/67] Fixing hashing records with optional strings. --- src/CompHash.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CompHash.cc b/src/CompHash.cc index cc4f440e06..2f1a41e311 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -407,7 +407,7 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v, case TYPE_INTERNAL_STRING: if ( ! v ) - return 0; + return optional ? sz : 0; // Factor in length field. sz = SizeAlign(sz, sizeof(int)); From cdb20e61b753d27bbf8be3550488b3854c6e9758 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Feb 2011 14:46:28 -0800 Subject: [PATCH 07/67] Teaching bifcl to accept scoped IDs. --- src/builtin-func.l | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/builtin-func.l b/src/builtin-func.l index ca7923b852..a9459ea4a8 100644 --- a/src/builtin-func.l +++ b/src/builtin-func.l @@ -27,7 +27,7 @@ int check_c_mode(int t) %} WS [ \t]+ -ID [A-Za-z_][A-Za-z_0-9]* +ID [A-Za-z_](([A-Za-z_0-9]|::)*[A-Za-z_0-9])? ESCSEQ (\\([^\n]|[0-7]+|x[[:xdigit:]]+)) %option nodefault From 95069f0993d06913d855ad8d6bc4765ca892ca57 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Feb 2011 16:04:32 -0800 Subject: [PATCH 08/67] Implementing += operator for record types. This is per #375. Record types can now get additional fields later via '+='. The added fields must however either be &optional or have a &default value. Example: type Foo: record { a: count; b: count &optional; }; redef record Foo += { c: count &default=42; d: count &optional; }; global f: Foo = [$a=21]; print f; Output: [a=21, b=, c=42, d=] --- src/Type.cc | 20 ++++++++++++++++++++ src/Type.h | 4 ++++ src/parse.y | 22 ++++++++++++++++++++-- 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/src/Type.cc b/src/Type.cc index ec7c8e510b..cfb7f86c0b 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -898,6 +898,26 @@ void RecordType::Describe(ODesc* d) const } } +const char* RecordType::AddFields(type_decl_list* others) + { + assert(types); + + loop_over_list(*others, i) + { + TypeDecl* td = (*others)[i]; + + if ( ! td->FindAttr(ATTR_DEFAULT) && ! td->FindAttr(ATTR_OPTIONAL) ) + return "extension field must be &optional or have &default"; + + types->append(td); + } + + delete others; + + num_fields = types->length(); + return 0; + } + void RecordType::DescribeFields(ODesc* d) const { if ( d->IsReadable() ) diff --git a/src/Type.h b/src/Type.h index ff4d3df9e6..23cdd0af25 100644 --- a/src/Type.h +++ b/src/Type.h @@ -409,6 +409,10 @@ public: int NumFields() const { return num_fields; } + // Returns 0 if all is ok, otherwise a pointer to an error message. Takes + // ownership of list. + const char* AddFields(type_decl_list* types); + void Describe(ODesc* d) const; void DescribeFields(ODesc* d) const; diff --git a/src/parse.y b/src/parse.y index 3cf2c07b18..b4d01c624c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -799,8 +799,8 @@ decl: | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REDEF); } - | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO - '{' enum_id_list opt_comma '}' ';' + | TOK_REDEF TOK_ENUM global_id TOK_ADD_TO + '{' enum_id_list opt_comma '}' ';' { if ( ! $3->Type() ) $3->Error("unknown identifier"); @@ -815,6 +815,24 @@ decl: } } + | TOK_REDEF TOK_RECORD global_id TOK_ADD_TO + '{' type_decl_list '}' ';' + { + if ( ! $3->Type() ) + $3->Error("unknown identifier"); + else + { + RecordType* add_to = $3->Type()->AsRecordType(); + if ( ! add_to ) + $3->Error("not a record type"); + else { + const char* error = add_to->AddFields($6); + if ( error ) + $3->Error(error); + } + } + } + | TOK_TYPE global_id ':' refined_type opt_attr ';' { add_type($2, $4, $5, 0); From 2f30c3d245f75fbe14012e2d3ab480f3556de922 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 7 Feb 2011 16:10:09 -0800 Subject: [PATCH 09/67] Adding some tests for the record-extension feature. These will go somewhere else eventually, just making sure they don't get lost. --- testing/rec.bro | 17 +++++++++++++++++ testing/rec2.bro | 17 +++++++++++++++++ testing/wrong-rec.bro | 13 +++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 testing/rec.bro create mode 100644 testing/rec2.bro create mode 100644 testing/wrong-rec.bro diff --git a/testing/rec.bro b/testing/rec.bro new file mode 100644 index 0000000000..904edd4e8c --- /dev/null +++ b/testing/rec.bro @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +type Foo: record { + a: count; + b: count &optional; +}; + +redef record Foo += { + c: count &default=42; + d: count &optional; +}; + +global f: Foo = [$a=21]; + +print f; + diff --git a/testing/rec2.bro b/testing/rec2.bro new file mode 100644 index 0000000000..c8324ae577 --- /dev/null +++ b/testing/rec2.bro @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +type Foo: record { + a: count; + b: count &optional; +}; + +redef record Foo += { + c: count &default=42; + d: string &optional; +}; + +global f: Foo = [$a=21, $d="XXX"]; + +print f; + diff --git a/testing/wrong-rec.bro b/testing/wrong-rec.bro new file mode 100644 index 0000000000..e5f553bf45 --- /dev/null +++ b/testing/wrong-rec.bro @@ -0,0 +1,13 @@ +# @TEST-EXEC-FAIL: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type Foo: record { + a: count; + b: count &optional; +}; + +redef record Foo += { + c: count; + d: string &optional; +}; + From d7786a6576e47640a10dcdfc05a5d6c0a1d712d1 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 8 Feb 2011 15:49:06 -0500 Subject: [PATCH 10/67] Checkpoint --- policy/logging.bro | 73 ++++++++++++++++++++++++++++++++++------- policy/test-logging.bro | 45 ++++++++++++++----------- src/bro.bif | 59 +++++++++++++++++---------------- src/strings.bif | 16 +++++++++ 4 files changed, 134 insertions(+), 59 deletions(-) diff --git a/policy/logging.bro b/policy/logging.bro index 26f9c01640..757ace3948 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -23,6 +23,9 @@ export { type Stream: record { name: string; columns: string_vec; + + # This is for tracking files internally if the output is a file. + _file: file &optional; }; # A filter defining what to record. @@ -35,7 +38,7 @@ export { # for all entries. The predicate receives one parameter: # an instance of the log's record type with the fields to be # logged. - pred: function(log: any) &optional; + pred: function(rec: any): bool &optional; # A path for outputting everything matching this # filter. The path is either a string, or a function @@ -55,16 +58,12 @@ export { # to an entry. The event receives the same parameter # as the predicate. It will always be generated, # independent of what the predicate returns. - ev: event(l: any) &optional; - + #ev: event(rec: any) &optional; + # The writer to use. writer: Writer &default=default_writer; }; - - global filters: table[string] of set[Filter]; - global streams: table[string] of Stream; - # Logs the record "rec" to the stream "id". The type of # "rec" must match the stream's "columns" field. global log: function(id: string, rec: any); @@ -80,9 +79,16 @@ export { global open_log_files: function(id: string); + # This is the internal filter store. The outer table is indexed with a string + # representing the stream name that the set of Logging::Filters is applied to. + global filters: table[string] of set[Filter]; + + # This is the internal stream store. The table is indexed by the stream name. + global streams: table[string] of Stream; + } -# Sentinel representing an unknown filter. +# Sentinel representing an unknown filter.d const NoSuchFilter: Filter = [$name=""]; function create_stream(id: string, log_record_type: string) @@ -91,6 +97,9 @@ function create_stream(id: string, log_record_type: string) print fmt("Stream %s already exists!", id); streams[id] = [$name=log_record_type, $columns=record_type_to_vector(log_record_type)]; + # Insert this as a separate step because the file_opened event needs + # the stream id to already exist. + streams[id]$_file = open_log_file(id); } function add_filter(id: string, filter: Filter) @@ -98,13 +107,55 @@ function add_filter(id: string, filter: Filter) if ( id !in filters ) filters[id] = set(); - # TODO: This is broken and waiting on a bug fix for &optional fields - # in records being used as indexes. - #add filt[filter]; + #add filters[id][filter]; + } + + +event file_opened(f: file) &priority=10 + { + # Only do any of this for files opened locally. + if ( is_remote_event() ) return; + + local filename = gsub(get_file_name(f), /\.log$/, ""); + if ( filename in streams ) + { + enable_raw_output(f); + + if (peer_description == "" || + peer_description == "manager" || + peer_description == "standalone") + { + print f, join_string_vec(streams[filename]$columns, "\t"); + } + } + else + { + print "no raw output", filename; + } } function log(id: string, rec: any) { + #if ( id !in streams ) + # { + # print fmt("Unable to log id: %s. Unknown logging stream.", id); + # return; + # } + #local stream = streams[id]; + # + #if ( id !in filters ) + # { + # print fmt("No filters for %s?", id); + # return; + # } + # + #local filter = match rec using filters[id]; + logging_log(id, rec); } + +event bro_init() &priority=-10 + { + # Check for logging streams without filters. + } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro index dde9bb8a2f..df617a9b79 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -4,15 +4,15 @@ module SSH; export { # Create a new ID for our log stream - redef enum Logging::ID += { LOG_SSH }; + #redef enum Logging::ID += { LOG_SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) type Log: record { - t: time; - id: conn_id; # Will be rolled out into individual columns. - status: string &optional; - country: string &default="unknown"; + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; }; # This is the prototype for the event that the logging framework tries @@ -28,24 +28,29 @@ event bro_init() Logging::create_stream("ssh", "SSH::Log"); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - # Filtering is not implemented yet. Waiting on ticket #366 + # Filtering is not implemented yet. Waiting on ticket #367 # Log line event generation is autogenerated for now by checking for # handlers for MODULE_NAME::log - #Logging::add_filter("ssh", [$name="default", $path="ssh", $ev=log]); + #Logging::add_filter("ssh", [$name="default", $path="ssh"]); # Log something. - Logging::log("ssh", [$t=network_time(), $country="US", $status="ok"]); + Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >"]); + Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="US"]); + Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="RU"]); + Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="RU"]); + Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="RU"]); + } -event log(rec: Log) - { - print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); - } - - -module WHATEVER; - -event SSH::log(rec: SSH::Log) - { - print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); - } \ No newline at end of file +#event log(rec: Log) +# { +# print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); +# } +# +# +#module WHATEVER; +# +#event SSH::log(rec: SSH::Log) +# { +# print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); +# } \ No newline at end of file diff --git a/src/bro.bif b/src/bro.bif index a573e6a716..88e3e6df51 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -362,7 +362,8 @@ function cat%(...%): string function logging_log%(index: string, rec: any%): any %{ - // Verify that rec is a record + // TODO: Verify that rec is a record + // TODO: Coerce to the correct record type to fill in defaults. // Lookup the stream TableVal *streams = opt_internal_table("Logging::streams"); @@ -382,7 +383,22 @@ function logging_log%(index: string, rec: any%): any printf("Logging framework is dead (Logging::streams not found).\n"); return false; } - + + // Lookup all filters for stream + TableVal *filters = opt_internal_table("Logging::filters"); + RecordVal *stream_filters; + if ( filters ) + { + Val *lookup_v = filters->Lookup(index); + if ( lookup_v ) + stream_filters = lookup_v->AsRecordVal(); + } + else + { + printf("Logging framework is dead (Logging::filters not found).\n"); + return false; + } + // Generate the event for the log stream // This happens regardless of all filters. int name_field = stream_record->Type()->AsRecordType()->FieldOffset("name"); @@ -397,42 +413,29 @@ function logging_log%(index: string, rec: any%): any vl->append(rec->Ref()); mgr.QueueEvent(ev_ptr, vl, SOURCE_LOCAL); } - - // Lookup all filters for stream - // (ignore this code, it will probably be done in the logging.bro script - // with the "match" statement) - //TableVal *filters = opt_internal_table("Logging::filters"); - //RecordVal *stream_filters; - //if ( filters ) - // { - // Val *lookup_v = filters->Lookup(index); - // if ( lookup_v ) - // stream_filters = lookup_v->AsRecordVal(); - // } - //else - // { - // printf("Logging framework is dead (Logging::filters not found).\n"); - // return false; - // } - // Print the line + // Format and print the line // (send line onward to the filter's WRITER in the future) - ODesc d; - const char *field_name; - int field = 0; + int file_field = stream_record->Type()->AsRecordType()->FieldOffset("_file"); + BroFile *f = stream_record->AsRecordVal()->Lookup(file_field)->AsFile(); + if ( ! f->IsOpen() ) + return 0; + + ODesc d; RecordType *rt = rec->Type()->AsRecordType(); for ( unsigned i = 1; i <= columns->Size(); ++i ) { - field_name = columns->Lookup(i)->AsStringVal()->CheckString(); - field = rec->Type()->AsRecordType()->FieldOffset(field_name); + const char *field_name = columns->Lookup(i)->AsStringVal()->CheckString(); + int field = rec->Type()->AsRecordType()->FieldOffset(field_name); if ( field >= 0 ) // or if there is a default value { rec->AsRecordVal()->Lookup(field)->Describe(&d); - d.Add("\t",0); + d.Add("\t"); } } - - printf("%s: %s\n", ID_module.c_str(), d.TakeBytes()); + f->Write(d.Description(), 0); + f->Write("\n",0); + return 0; %} diff --git a/src/strings.bif b/src/strings.bif index 44b0c57eb6..e83b4666a2 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -135,6 +135,22 @@ function sort_string_array%(a: string_array%): string_array vs_to_string_array(vs, b, 1, n); return b; %} + +function join_string_vec%(vec: string_vec, sep: string%): string + %{ + ODesc d; + VectorVal *v = vec->AsVectorVal(); + for ( unsigned i = 1; i < v->Size(); ++i ) + { + if ( i > 1 ) + d.Add(sep->CheckString(), 0); + v->Lookup(i+1)->Describe(&d); + } + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + return new StringVal(s); + %} + function edit%(arg_s: string, arg_edit_char: string%): string %{ From e0c65cc8f9395fca3c5e856f0691cb3947ce2bdf Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 9 Feb 2011 12:22:26 -0500 Subject: [PATCH 11/67] Initial functional implementation of logging framework. The policy/test-logging.bro script has the first usage of the logging code and is commented to indicate current problems and limitations. --- policy/logging.bro | 77 +++++++++++++++++++++--------------- policy/test-logging.bro | 33 +++++++++++----- src/bro.bif | 86 +++++++++++++++++++++++++++++++---------- 3 files changed, 135 insertions(+), 61 deletions(-) diff --git a/policy/logging.bro b/policy/logging.bro index 757ace3948..610e9e6f69 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -23,23 +23,20 @@ export { type Stream: record { name: string; columns: string_vec; - - # This is for tracking files internally if the output is a file. - _file: file &optional; }; # A filter defining what to record. type Filter: record { # A name to reference this filter. name: string; - + # A predicate returning True if the filter wants a log entry # to be recorded. If not given, an implicit True is assumed # for all entries. The predicate receives one parameter: # an instance of the log's record type with the fields to be # logged. pred: function(rec: any): bool &optional; - + # A path for outputting everything matching this # filter. The path is either a string, or a function # called with a single ``ID`` argument and returning a string. @@ -48,12 +45,13 @@ export { # Writer, but if it's refering to a file, it's assumed that no # extension is given; the writer will add whatever is # appropiate. - path: any &optional; - + path: string &optional; + dynamic_path: function(id: string): string &optional; + # A subset of column names to record. If not given, all # columns are recorded. - select: set[string] &optional; - + #select: set[string] &optional; + # An event that is raised whenever the filter is applied # to an entry. The event receives the same parameter # as the predicate. It will always be generated, @@ -62,12 +60,15 @@ export { # The writer to use. writer: Writer &default=default_writer; + + # Internal tracking of header names and order for this filter. + #columns: string_vec &optional; }; # Logs the record "rec" to the stream "id". The type of # "rec" must match the stream's "columns" field. global log: function(id: string, rec: any); - global log_ev: event(id: string, rec: any); + #global log_ev: event(id: string, rec: any); # Returns an existing filter previously installed for stream # "id" under the given "name". If no such filter exists, @@ -76,20 +77,26 @@ export { global create_stream: function(id: string, log_record_type: string); global add_filter: function(id: string, filter: Filter); + global remove_filter: function(id: string, filter: string): bool; + + global add_default_filter: function(id: string); + global remove_default_filter: function(id: string): bool; global open_log_files: function(id: string); # This is the internal filter store. The outer table is indexed with a string # representing the stream name that the set of Logging::Filters is applied to. global filters: table[string] of set[Filter]; - + # This is the internal stream store. The table is indexed by the stream name. global streams: table[string] of Stream; - + + global files: table[string] of file; } + # Sentinel representing an unknown filter.d -const NoSuchFilter: Filter = [$name=""]; +const NoSuchFilter: Filter = [$name="", $path="unknown"]; function create_stream(id: string, log_record_type: string) { @@ -99,7 +106,7 @@ function create_stream(id: string, log_record_type: string) streams[id] = [$name=log_record_type, $columns=record_type_to_vector(log_record_type)]; # Insert this as a separate step because the file_opened event needs # the stream id to already exist. - streams[id]$_file = open_log_file(id); + #streams[id]$_file = open_log_file(id); } function add_filter(id: string, filter: Filter) @@ -107,15 +114,38 @@ function add_filter(id: string, filter: Filter) if ( id !in filters ) filters[id] = set(); - #add filters[id][filter]; + add filters[id][filter]; } +function remove_filter(id: string, filter: string): bool + { + for ( filt in filters[id] ) + { + if ( filt$name == "default" ) + { + delete filters[id][filt]; + return T; + } + } + return F; + } + +function add_default_filter(id: string) + { + add_filter(id, [$name="default", $path=id]); + } + +function remove_default_filter(id: string): bool + { + return remove_filter("ssh", "default"); + } event file_opened(f: file) &priority=10 { # Only do any of this for files opened locally. if ( is_remote_event() ) return; + # TODO: this shouldn't rely on .log being the extension local filename = gsub(get_file_name(f), /\.log$/, ""); if ( filename in streams ) { @@ -136,26 +166,11 @@ event file_opened(f: file) &priority=10 function log(id: string, rec: any) { - #if ( id !in streams ) - # { - # print fmt("Unable to log id: %s. Unknown logging stream.", id); - # return; - # } - #local stream = streams[id]; - # - #if ( id !in filters ) - # { - # print fmt("No filters for %s?", id); - # return; - # } - # - #local filter = match rec using filters[id]; - logging_log(id, rec); } event bro_init() &priority=-10 { - # Check for logging streams without filters. + # TODO: Check for logging streams without filters. } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro index df617a9b79..df57c6d576 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -17,7 +17,7 @@ export { # This is the prototype for the event that the logging framework tries # to generate if there is a handler for it. - global log: event(rec: Log); + #global log: event(rec: Log); } event bro_init() @@ -28,17 +28,32 @@ event bro_init() Logging::create_stream("ssh", "SSH::Log"); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - # Filtering is not implemented yet. Waiting on ticket #367 # Log line event generation is autogenerated for now by checking for - # handlers for MODULE_NAME::log - #Logging::add_filter("ssh", [$name="default", $path="ssh"]); + # handlers for MODULE_NAME::log (which isn't the right thing to do, but it will be dealt with later) + Logging::add_default_filter("ssh"); + + # There is currently some problem with &optional values in the records + # passed into the predicate. Maybe it's because I'm not really coercing + # the record to the correct record type before passing it as an argument + # to the Call method? + + # There is also a problem with using &optional sets in the filter records. + # It was found when trying to include the "select" variable. + + # Printing headers for the filters doesn't work yet either and needs to + # be considered in the final design. (based on the "select" set). + #Logging::add_filter("ssh", [$name="successful logins", + # #$pred(rec: Log) = { print rec$status; return T; }, + # $path="ssh-logins", + # #$select=set("t"), + # $writer=Logging::WRITER_CSV]); # Log something. - Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >"]); - Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="US"]); - Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="RU"]); - Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="RU"]); - Logging::log("ssh", [$t=network_time(),$status="ok\x00ok < >", $country="RU"]); + Logging::log("ssh", [$t=network_time(),$status="success"]); + Logging::log("ssh", [$t=network_time(),$status="failure", $country="US"]); + Logging::log("ssh", [$t=network_time(),$status="failure", $country="UK"]); + Logging::log("ssh", [$t=network_time(),$status="success", $country="BR"]); + Logging::log("ssh", [$t=network_time(),$status="failure", $country="MX"]); } diff --git a/src/bro.bif b/src/bro.bif index 88e3e6df51..278c14500e 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -364,7 +364,8 @@ function logging_log%(index: string, rec: any%): any %{ // TODO: Verify that rec is a record // TODO: Coerce to the correct record type to fill in defaults. - + RecordVal *recval = rec->AsRecordVal(); + // Lookup the stream TableVal *streams = opt_internal_table("Logging::streams"); VectorVal *columns; @@ -386,12 +387,12 @@ function logging_log%(index: string, rec: any%): any // Lookup all filters for stream TableVal *filters = opt_internal_table("Logging::filters"); - RecordVal *stream_filters; + TableVal *stream_filters; if ( filters ) { Val *lookup_v = filters->Lookup(index); if ( lookup_v ) - stream_filters = lookup_v->AsRecordVal(); + stream_filters = lookup_v->AsTableVal(); } else { @@ -410,31 +411,74 @@ function logging_log%(index: string, rec: any%): any if ( ev_ptr ) { val_list* vl = new val_list; - vl->append(rec->Ref()); + vl->append(recval->Ref()); mgr.QueueEvent(ev_ptr, vl, SOURCE_LOCAL); } - // Format and print the line - // (send line onward to the filter's WRITER in the future) - int file_field = stream_record->Type()->AsRecordType()->FieldOffset("_file"); - BroFile *f = stream_record->AsRecordVal()->Lookup(file_field)->AsFile(); - if ( ! f->IsOpen() ) - return 0; - - ODesc d; - RecordType *rt = rec->Type()->AsRecordType(); - for ( unsigned i = 1; i <= columns->Size(); ++i ) + // Iterate over the stream_filters + ListVal* filter_recs = stream_filters->ConvertToList(TYPE_ANY); + for ( int i = 0; i < filter_recs->Length(); ++i ) { - const char *field_name = columns->Lookup(i)->AsStringVal()->CheckString(); - int field = rec->Type()->AsRecordType()->FieldOffset(field_name); - if ( field >= 0 ) // or if there is a default value + RecordVal* rv = filter_recs->Index(i)->AsListVal()->Index(0)->AsRecordVal(); + int pred_field_index = rv->Type()->AsRecordType()->FieldOffset("pred"); + Val *pred_func = rv->Lookup(pred_field_index); + if ( pred_func ) { - rec->AsRecordVal()->Lookup(field)->Describe(&d); - d.Add("\t"); + val_list args(1); + args.append(recval->Ref()); + int pred_val = pred_func->AsFunc()->Call(&args)->AsBool(); + //printf("predicate val: %d\n", pred_val); + // If the predicate returns false, don't send this record on to a writer. + if ( !pred_val ) + continue; } + + // Format and print the line + // (send line onward to the filter's WRITER in the future) + + // Get a path name for this filter + int path_field = rv->Type()->AsRecordType()->FieldOffset("path"); + Val *path_val = rv->Lookup(path_field); + StringVal *path; + // If no path is given, use the Stream ID as the path + if ( path_val ) + path = path_val->AsStringVal(); + else + path = index; + + // Get the file with the "path" name found above for this filter. + // Open a new file is one does not exist yet. + TableVal *logging_files = opt_internal_table("Logging::files"); + Val *ff = logging_files->Lookup(path); + if ( !ff ) + { + logging_files->Assign(path, new Val(new BroFile(path->CheckString(), "w"))); + ff = logging_files->Lookup(path); + } + BroFile *f = ff->AsFile(); + if ( ! f->IsOpen() ) + continue; + + // Actually format the log line and print it to the file found above. + ODesc d; + RecordType *rt = rec->Type()->AsRecordType(); + for ( unsigned i = 1; i <= columns->Size(); ++i ) + { + const char *field_name = columns->Lookup(i)->AsStringVal()->CheckString(); + int field = rt->FieldOffset(field_name); + if ( field >= 0 ) // or if there is a default value + { + rec->AsRecordVal()->Lookup(field)->Describe(&d); + d.Add("\t"); + } + } + f->Write(d.Description(), 0); + f->Write("\n",0); + + } - f->Write(d.Description(), 0); - f->Write("\n",0); + + Unref(filter_recs); return 0; %} From ffa494e428216eba024a20c6a15af42e0f5a147b Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 9 Feb 2011 15:18:12 -0500 Subject: [PATCH 12/67] Fixed an off-by-one error in join_string_vec --- src/strings.bif | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strings.bif b/src/strings.bif index e83b4666a2..7b473f7aaa 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -140,9 +140,9 @@ function join_string_vec%(vec: string_vec, sep: string%): string %{ ODesc d; VectorVal *v = vec->AsVectorVal(); - for ( unsigned i = 1; i < v->Size(); ++i ) + for ( unsigned i = 0; i < v->Size(); ++i ) { - if ( i > 1 ) + if ( i > 0 ) d.Add(sep->CheckString(), 0); v->Lookup(i+1)->Describe(&d); } From af3267acc3fa7b5e7b933bf8de7124202853bfc8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 10 Feb 2011 19:35:40 -0800 Subject: [PATCH 13/67] Enabling automatic coercion from record type A to be B as long as A has all the types that B has. --- src/Expr.cc | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index dbfca7c9cb..3be0bdb592 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3994,15 +3994,8 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r) { int t_i = t_r->FieldOffset(sub_r->FieldName(i)); if ( t_i < 0 ) - { - // Same as in RecordConstructorExpr::InitVal. - char buf[512]; - safe_snprintf(buf, sizeof(buf), - "orphan record field \"%s\"", - sub_r->FieldName(i)); - Error(buf); + // Orphane field in rhs, that's ok. continue; - } BroType* sub_t_i = sub_r->FieldType(i); BroType* sup_t_i = t_r->FieldType(t_i); @@ -4047,7 +4040,21 @@ Val* RecordCoerceExpr::Fold(Val* v) const for ( int i = 0; i < map_size; ++i ) { if ( map[i] >= 0 ) - val->Assign(i, rv->Lookup(map[i])->Ref()); + { + Val* rhs = rv->Lookup(map[i]); + if ( ! rhs ) + { + const Attr* def = rv->Type()->AsRecordType()->FieldDecl(map[i])->FindAttr(ATTR_DEFAULT); + if ( def ) + rhs = def->AttrExpr()->Eval(0); + } + + if ( rhs ) + rhs = rhs->Ref(); + + assert(rhs || Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)); + val->Assign(i, rhs); + } else val->Assign(i, 0); } From 9d407d882c67be22e979f35e9b44efcd609f6153 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Feb 2011 13:01:34 -0800 Subject: [PATCH 14/67] Two small infrastructure extensions for passing information into the logging framework. - To enable passing a type into a bif, there's now a new BroType-derived class TypeType and a corresponding TYPE_TYPE tag. With that, a Val can now have a type as its value. This is experimental for now. - RecordVal's get a new method CoerceTo() to coerce their value into a another record type with the usual semantics. Most of the code in there was previously in RecordContructorExpr::InitVal(), which is now calling the new CoerceTo() method. --- src/Expr.cc | 44 ++++------------------------------------- src/Type.cc | 5 +++++ src/Type.h | 28 +++++++++++++++++++++++++++ src/Val.cc | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++ src/Val.h | 26 +++++++++++++++++++++++++ 5 files changed, 119 insertions(+), 40 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index 5788cdeb7b..0f186b1f99 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -284,7 +284,7 @@ Val* NameExpr::Eval(Frame* f) const Val* v; if ( id->AsType() ) - RunTime("cannot evaluate type name"); + return new Val(id->AsType(), true); if ( id->IsGlobal() ) v = id->ID_Val(); @@ -3309,48 +3309,12 @@ RecordConstructorExpr::RecordConstructorExpr(ListExpr* constructor_list) Val* RecordConstructorExpr::InitVal(const BroType* t, Val* aggr) const { - if ( ! aggr ) - aggr = new RecordVal(const_cast(t->AsRecordType())); + RecordVal* rv = Eval(0)->AsRecordVal(); + RecordVal* ar = rv->CoerceTo(t->AsRecordType(), aggr); - if ( record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) ) + if ( ar ) { - RecordVal* ar = aggr->AsRecordVal(); - RecordType* ar_t = aggr->Type()->AsRecordType(); - - RecordVal* rv = Eval(0)->AsRecordVal(); - RecordType* rv_t = rv->Type()->AsRecordType(); - - int i; - for ( i = 0; i < rv_t->NumFields(); ++i ) - { - int t_i = ar_t->FieldOffset(rv_t->FieldName(i)); - - if ( t_i < 0 ) - { - char buf[512]; - safe_snprintf(buf, sizeof(buf), - "orphan field \"%s\" in initialization", - rv_t->FieldName(i)); - Error(buf); - break; - } - - else - ar->Assign(t_i, rv->Lookup(i)->Ref()); - } - - for ( i = 0; i < ar_t->NumFields(); ++i ) - if ( ! ar->Lookup(i) && - ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) - { - char buf[512]; - safe_snprintf(buf, sizeof(buf), - "non-optional field \"%s\" missing in initialization", ar_t->FieldName(i)); - Error(buf); - } - Unref(rv); - return ar; } diff --git a/src/Type.cc b/src/Type.cc index db74781072..31a04bfeb3 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -95,6 +95,7 @@ BroType::BroType(TypeTag t, bool arg_base_type) case TYPE_FUNC: case TYPE_FILE: case TYPE_VECTOR: + case TYPE_TYPE: internal_tag = TYPE_INTERNAL_OTHER; break; @@ -1426,6 +1427,9 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) case TYPE_FILE: return same_type(t1->YieldType(), t2->YieldType(), is_init); + case TYPE_TYPE: + return same_type(t1, t2, is_init); + case TYPE_UNION: error("union type in same_type()"); } @@ -1514,6 +1518,7 @@ int is_assignable(BroType* t) case TYPE_VECTOR: case TYPE_FILE: case TYPE_TABLE: + case TYPE_TYPE: return 1; case TYPE_VOID: diff --git a/src/Type.h b/src/Type.h index 7e890d9e07..8901380ab7 100644 --- a/src/Type.h +++ b/src/Type.h @@ -31,6 +31,7 @@ typedef enum { TYPE_FUNC, TYPE_FILE, TYPE_VECTOR, + TYPE_TYPE, TYPE_ERROR #define NUM_TYPES (int(TYPE_ERROR) + 1) } TypeTag; @@ -59,6 +60,7 @@ class ListExpr; class EnumType; class Serializer; class VectorType; +class TypeType; const int DOES_NOT_MATCH_INDEX = 0; const int MATCHES_INDEX_SCALAR = 1; @@ -151,6 +153,7 @@ public: CHECK_TYPE_TAG(TYPE_SUBNET, "BroType::AsSubNetType"); return (const SubNetType*) this; } + SubNetType* AsSubNetType() { CHECK_TYPE_TAG(TYPE_SUBNET, "BroType::AsSubNetType"); @@ -192,6 +195,18 @@ public: return (VectorType*) this; } + const TypeType* AsTypeType() const + { + CHECK_TYPE_TAG(TYPE_TYPE, "BroType::AsTypeType"); + return (TypeType*) this; + } + + TypeType* AsTypeType() + { + CHECK_TYPE_TAG(TYPE_TYPE, "BroType::AsTypeType"); + return (TypeType*) this; + } + int IsSet() const { return tag == TYPE_TABLE && (YieldType() == 0); @@ -359,6 +374,19 @@ protected: ID* return_value; }; +class TypeType : public BroType { +public: + TypeType(BroType* t) : BroType(TYPE_TYPE) { type = t->Ref(); } + ~TypeType() { Unref(type); } + + BroType* Type() { return type; } + +protected: + TypeType() {} + + BroType* type; +}; + class TypeDecl { public: TypeDecl(BroType* t, const char* i, attr_list* attrs = 0); diff --git a/src/Val.cc b/src/Val.cc index e7f601210a..dbc55856db 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2892,6 +2892,62 @@ Val* RecordVal::Lookup(int field) const return (*AsRecord())[field]; } +RecordVal* RecordVal::CoerceTo(const RecordType* t, Val* aggr) const + { + if ( ! record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) ) + return 0; + + if ( ! aggr ) + aggr = new RecordVal(const_cast(t->AsRecordType())); + + RecordVal* ar = aggr->AsRecordVal(); + RecordType* ar_t = aggr->Type()->AsRecordType(); + + const RecordType* rv_t = Type()->AsRecordType(); + + int i; + for ( i = 0; i < rv_t->NumFields(); ++i ) + { + int t_i = ar_t->FieldOffset(rv_t->FieldName(i)); + + if ( t_i < 0 ) + { + char buf[512]; + safe_snprintf(buf, sizeof(buf), + "orphan field \"%s\" in initialization", + rv_t->FieldName(i)); + Error(buf); + break; + } + + else + ar->Assign(t_i, Lookup(i)->Ref()); + } + + for ( i = 0; i < ar_t->NumFields(); ++i ) + if ( ! ar->Lookup(i) && + ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) + { + char buf[512]; + safe_snprintf(buf, sizeof(buf), + "non-optional field \"%s\" missing in initialization", ar_t->FieldName(i)); + Error(buf); + } + + return ar; + } + +RecordVal* RecordVal::CoerceTo(RecordType* t) + { + if ( same_type(Type(), t) ) + { + this->Ref(); + return this; + } + + return CoerceTo(t, 0); + } + void RecordVal::Describe(ODesc* d) const { const val_list* vl = AsRecord(); diff --git a/src/Val.h b/src/Val.h index d21562c907..c056c4e630 100644 --- a/src/Val.h +++ b/src/Val.h @@ -39,6 +39,7 @@ class TableVal; class RecordVal; class ListVal; class StringVal; +class EnumVal; class MutableVal; class StateAccess; @@ -163,6 +164,15 @@ public: // class has ref'd it. Val(BroFile* f); + Val(BroType* t, bool type_type) // Extra arg to differentiate from protected version. + { + type = new TypeType(t->Ref()); + attribs = 0; +#ifdef DEBUG + bound_id = 0; +#endif + } + Val() { val.int_val = 0; @@ -245,6 +255,12 @@ public: return &val.subnet_val; } + const BroType* AsType() const + { + CHECK_TAG(type->Tag(), TYPE_TYPE, "Val::Type", type_name) + return type; + } + // ... in network byte order const addr_type AsAddr() const { @@ -295,6 +311,7 @@ public: CONVERTER(TYPE_LIST, ListVal*, AsListVal) CONVERTER(TYPE_STRING, StringVal*, AsStringVal) CONVERTER(TYPE_VECTOR, VectorVal*, AsVectorVal) + CONVERTER(TYPE_ENUM, EnumVal*, AsEnumVal) #define CONST_CONVERTER(tag, ctype, name) \ const ctype name() const \ @@ -921,6 +938,15 @@ public: void SetOrigin(BroObj* o) { origin = o; } BroObj* GetOrigin() const { return origin; } + // Returns a new value representing the value coerced to the given type. + // If coercion is not possible, returns 0. The non-const version may + // return the current value ref'ed if its type matches directly. + // + // *aggr* is optional; if non-zero, we add to it. See Expr::InitVal(). We + // leave it out in the non-const version to make the choice unambigious. + RecordVal* CoerceTo(const RecordType* other, Val* aggr) const; + RecordVal* CoerceTo(RecordType* other); + unsigned int MemoryAllocation() const; protected: From 68062e87f1c51627ac221adabe73f52c43f4af91 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Feb 2011 13:03:46 -0800 Subject: [PATCH 15/67] Lots of infracstructure for the new logging framework. This pretty much follows the proposal on the projects page. It includes: - A new LogMgr, maintaining the set of writers. - The abstract LogWriter API. - An initial implementation in the form of LogWriterAscii producing tab-separated columns. Note that things are only partially working right now, things are subject to change, and it's all not much tested at all. That's why I'm creating separate branch for now. Example: bro -B logging test-logging && cat debug.log 1298063168.409852/1298063168.410368 [logging] Created new logging stream 'SSH::LOG_SSH' 1298063168.409852/1298063168.410547 [logging] Created new filter 'default' for stream 'SSH::LOG_SSH' 1298063168.409852/1298063168.410564 [logging] writer : Ascii 1298063168.409852/1298063168.410574 [logging] path : ssh_log_ssh 1298063168.409852/1298063168.410584 [logging] path_func : not set 1298063168.409852/1298063168.410594 [logging] event : not set 1298063168.409852/1298063168.410604 [logging] pred : not set 1298063168.409852/1298063168.410614 [logging] field t: time 1298063168.409852/1298063168.410625 [logging] field id.orig_h: addr 1298063168.409852/1298063168.410635 [logging] field id.orig_p: port 1298063168.409852/1298063168.410645 [logging] field id.resp_h: addr 1298063168.409852/1298063168.410655 [logging] field id.resp_p: port 1298063168.409852/1298063168.410665 [logging] field status: string 1298063168.409852/1298063168.410675 [logging] field country: string 1298063168.409852/1298063168.410817 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH' 1298063168.409852/1298063168.410865 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH' 1298063168.409852/1298063168.410906 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH' 1298063168.409852/1298063168.410945 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH' 1298063168.409852/1298063168.411044 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH > cat ssh_log_ssh.log 1298063168.40985 1.2.3.4 66770 2.3.4.5 65616 success unknown 1298063168.40985 1.2.3.4 66770 2.3.4.5 65616 failure US 1298063168.40985 1.2.3.4 66770 2.3.4.5 65616 failure UK 1298063168.40985 1.2.3.4 66770 2.3.4.5 65616 success BR 1298063168.40985 1.2.3.4 66770 2.3.4.5 65616 failure MX --- policy/bro.init | 57 ++++- policy/logging.bro | 174 +------------- policy/test-logging.bro | 42 ++-- src/CMakeLists.txt | 3 + src/DebugLogger.cc | 1 + src/DebugLogger.h | 1 + src/LogMgr.cc | 499 ++++++++++++++++++++++++++++++++++++++++ src/LogMgr.h | 82 +++++++ src/LogWriter.cc | 76 ++++++ src/LogWriter.h | 82 +++++++ src/LogWriterAscii.cc | 129 +++++++++++ src/LogWriterAscii.h | 27 +++ src/NetVar.cc | 4 + src/NetVar.h | 2 + src/bro.bif | 79 +++++-- src/logging.bif | 23 ++ src/logging.bro | 52 +++++ src/main.cc | 6 +- 18 files changed, 1121 insertions(+), 218 deletions(-) create mode 100644 src/LogMgr.cc create mode 100644 src/LogMgr.h create mode 100644 src/LogWriter.cc create mode 100644 src/LogWriter.h create mode 100644 src/LogWriterAscii.cc create mode 100644 src/LogWriterAscii.h create mode 100644 src/logging.bif create mode 100644 src/logging.bro diff --git a/policy/bro.init b/policy/bro.init index f9742798c4..12d707d812 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -272,6 +272,60 @@ type entropy_test_result: record { serial_correlation: double; }; +type Log_Writer: enum { # TODO: Move these into bif and use from C++ as well. + WRITER_DEFAULT, # See default_writer below. + WRITER_ASCII, +}; + +# Each stream gets a unique ID. This type will be extended by +# other scripts. +type Log_ID: enum { + Unknown +}; + +# The default writer to use if a filter does not specify +# anything else. +const Log_default_writer = WRITER_ASCII &redef; + +# A filter defining what to log. +type log_filter: record { + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(rec: any): bool &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: string &optional; + path_func: function(id: string): string &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + include: set[string] &optional; + exclude: set[string] &optional; + + # An event that is raised whenever the filter is applied + # to an entry. The event receives the same parameter + # as the predicate. It will always be generated, + # independent of what the predicate returns. + ev: event(rec: any) &optional; + + # The writer to use. + writer: Log_Writer &default=Log_default_writer; +}; + + # Prototypes of Bro built-in functions. @load strings.bif.bro @load bro.bif.bro @@ -1382,10 +1436,11 @@ const suppress_local_output = F &redef; # Holds the filename of the trace file given with -w (empty if none). const trace_output_file = ""; - + # If a trace file is given, dump *all* packets seen by Bro into it. # By default, Bro applies (very few) heuristics to reduce the volume. # A side effect of setting this to true is that we can write the # packets out before we actually process them, which can be helpful # for debugging in case the analysis triggers a crash. const record_all_packets = F &redef; + diff --git a/policy/logging.bro b/policy/logging.bro index 610e9e6f69..f00dcf7fb2 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -1,176 +1,10 @@ -module Logging; -export { - # The set of writers Bro provides. - type Writer: enum { - WRITER_DEFAULT, # See default_writer below. - WRITER_CSV, - WRITER_DATA_SERIES, - WRITER_SYSLOG - }; - - # Each stream gets a unique ID. This type will be extended by - # other scripts. - type ID: enum { - Unknown - }; - - # The default writer to use if a filter does not specify - # anything else. - const default_writer = WRITER_CSV &redef; - - # Type defining a stream. - type Stream: record { - name: string; - columns: string_vec; - }; - - # A filter defining what to record. - type Filter: record { - # A name to reference this filter. - name: string; - - # A predicate returning True if the filter wants a log entry - # to be recorded. If not given, an implicit True is assumed - # for all entries. The predicate receives one parameter: - # an instance of the log's record type with the fields to be - # logged. - pred: function(rec: any): bool &optional; - - # A path for outputting everything matching this - # filter. The path is either a string, or a function - # called with a single ``ID`` argument and returning a string. - # - # The specific interpretation of the string is left to the - # Writer, but if it's refering to a file, it's assumed that no - # extension is given; the writer will add whatever is - # appropiate. - path: string &optional; - dynamic_path: function(id: string): string &optional; - - # A subset of column names to record. If not given, all - # columns are recorded. - #select: set[string] &optional; - - # An event that is raised whenever the filter is applied - # to an entry. The event receives the same parameter - # as the predicate. It will always be generated, - # independent of what the predicate returns. - #ev: event(rec: any) &optional; - - # The writer to use. - writer: Writer &default=default_writer; - - # Internal tracking of header names and order for this filter. - #columns: string_vec &optional; - }; - - # Logs the record "rec" to the stream "id". The type of - # "rec" must match the stream's "columns" field. - global log: function(id: string, rec: any); - #global log_ev: event(id: string, rec: any); - - # Returns an existing filter previously installed for stream - # "id" under the given "name". If no such filter exists, - # the record "NoSuchFilter" is returned. - global get_filter: function(id: string, name: string) : Filter; - - global create_stream: function(id: string, log_record_type: string); - global add_filter: function(id: string, filter: Filter); - global remove_filter: function(id: string, filter: string): bool; - - global add_default_filter: function(id: string); - global remove_default_filter: function(id: string): bool; - - global open_log_files: function(id: string); - - # This is the internal filter store. The outer table is indexed with a string - # representing the stream name that the set of Logging::Filters is applied to. - global filters: table[string] of set[Filter]; - - # This is the internal stream store. The table is indexed by the stream name. - global streams: table[string] of Stream; - - global files: table[string] of file; -} - - -# Sentinel representing an unknown filter.d -const NoSuchFilter: Filter = [$name="", $path="unknown"]; - -function create_stream(id: string, log_record_type: string) +function Log_add_default_filter(id: Log_ID) { - if ( id in streams ) - print fmt("Stream %s already exists!", id); - - streams[id] = [$name=log_record_type, $columns=record_type_to_vector(log_record_type)]; - # Insert this as a separate step because the file_opened event needs - # the stream id to already exist. - #streams[id]$_file = open_log_file(id); + log_add_filter(id, [$name="default"]); } -function add_filter(id: string, filter: Filter) +function Log_remove_default_filter(id: Log_ID): bool { - if ( id !in filters ) - filters[id] = set(); - - add filters[id][filter]; + log_remove_filter(id, "default"); } - -function remove_filter(id: string, filter: string): bool - { - for ( filt in filters[id] ) - { - if ( filt$name == "default" ) - { - delete filters[id][filt]; - return T; - } - } - return F; - } - -function add_default_filter(id: string) - { - add_filter(id, [$name="default", $path=id]); - } - -function remove_default_filter(id: string): bool - { - return remove_filter("ssh", "default"); - } - -event file_opened(f: file) &priority=10 - { - # Only do any of this for files opened locally. - if ( is_remote_event() ) return; - - # TODO: this shouldn't rely on .log being the extension - local filename = gsub(get_file_name(f), /\.log$/, ""); - if ( filename in streams ) - { - enable_raw_output(f); - - if (peer_description == "" || - peer_description == "manager" || - peer_description == "standalone") - { - print f, join_string_vec(streams[filename]$columns, "\t"); - } - } - else - { - print "no raw output", filename; - } - } - -function log(id: string, rec: any) - { - logging_log(id, rec); - } - - -event bro_init() &priority=-10 - { - # TODO: Check for logging streams without filters. - } \ No newline at end of file diff --git a/policy/test-logging.bro b/policy/test-logging.bro index df57c6d576..e8ccc7c22e 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -4,7 +4,7 @@ module SSH; export { # Create a new ID for our log stream - #redef enum Logging::ID += { LOG_SSH }; + redef enum Log_ID += { LOG_SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -14,10 +14,6 @@ export { status: string &optional; country: string &default="unknown"; }; - - # This is the prototype for the event that the logging framework tries - # to generate if there is a handler for it. - #global log: event(rec: Log); } event bro_init() @@ -25,35 +21,27 @@ event bro_init() # Create the stream. # First argument is the ID for the stream. # Second argument is the log record type. - Logging::create_stream("ssh", "SSH::Log"); + log_create_stream(LOG_SSH, SSH::Log); # Add a default filter that simply logs everything to "ssh.log" using the default writer. - # Log line event generation is autogenerated for now by checking for - # handlers for MODULE_NAME::log (which isn't the right thing to do, but it will be dealt with later) - Logging::add_default_filter("ssh"); - - # There is currently some problem with &optional values in the records - # passed into the predicate. Maybe it's because I'm not really coercing - # the record to the correct record type before passing it as an argument - # to the Call method? - - # There is also a problem with using &optional sets in the filter records. - # It was found when trying to include the "select" variable. - + Log_add_default_filter(LOG_SSH); + # Printing headers for the filters doesn't work yet either and needs to # be considered in the final design. (based on the "select" set). - #Logging::add_filter("ssh", [$name="successful logins", + #Log::add_filter("ssh", [$name="successful logins", # #$pred(rec: Log) = { print rec$status; return T; }, # $path="ssh-logins", # #$select=set("t"), - # $writer=Logging::WRITER_CSV]); - + # $writer=Log::WRITER_CSV]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + # Log something. - Logging::log("ssh", [$t=network_time(),$status="success"]); - Logging::log("ssh", [$t=network_time(),$status="failure", $country="US"]); - Logging::log("ssh", [$t=network_time(),$status="failure", $country="UK"]); - Logging::log("ssh", [$t=network_time(),$status="success", $country="BR"]); - Logging::log("ssh", [$t=network_time(),$status="failure", $country="MX"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); } @@ -68,4 +56,4 @@ event bro_init() #event SSH::log(rec: SSH::Log) # { # print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); -# } \ No newline at end of file +# } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0f67dc173e..03b6079df5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -297,6 +297,9 @@ set(bro_SRCS IRC.cc List.cc Logger.cc + LogMgr.cc + LogWriter.cc + LogWriterAscii.cc Login.cc MIME.cc NCP.cc diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 9ab0bde380..3c9dbad363 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -17,6 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, + { "logging", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 3d8c1ac83f..4614597bfc 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -25,6 +25,7 @@ enum DebugStream { DBG_MAINLOOP, // Main IOSource loop DBG_DPD, // Dynamic application detection framework DBG_TM, // Time-machine packet input via Brocolli + DBG_LOGGING, // Logging streams NUM_DBGS // Has to be last }; diff --git a/src/LogMgr.cc b/src/LogMgr.cc new file mode 100644 index 0000000000..b9675e4e84 --- /dev/null +++ b/src/LogMgr.cc @@ -0,0 +1,499 @@ + +#include "LogMgr.h" +#include "EventHandler.h" +#include "NetVar.h" + +#include "LogWriterAscii.h" + +struct LogWriterDefinition { + LogWriterType::Type type; // The type. + const char *name; // Descriptive name for error messages. + bool (*init)(); // An optional one-time initialization function. + LogWriter* (*factory)(); // A factory function creating instances. +}; + +LogWriterDefinition log_writers[] = { + { LogWriterType::Ascii, "Ascii", 0, LogWriterAscii::Instantiate }, + + // End marker. + { LogWriterType::None, "None", 0, (LogWriter* (*)())0 } +}; + +struct LogMgr::Filter { + string name; + Func* pred; + Func* path_func; + EventHandlerPtr* event; + string path; + LogWriterDefinition* writer; + + int num_fields; + LogField** fields; + vector > indices; // List of record indices per field. + + typedef map WriterMap; + WriterMap writers; // Writers indexed by path. + + ~Filter(); +}; + +struct LogMgr::Stream { + string name; + RecordType* columns; + list filters; + + ~Stream(); + }; + + +LogMgr::Filter::~Filter() + { + for ( int i = 0; i < num_fields; ++i ) + delete fields[i]; + + for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) + delete i->second; + } + +LogMgr::Stream::~Stream() + { + Unref(columns); + for ( list::iterator f = filters.begin(); f != filters.end(); ++f ) + delete *f; + } + +LogMgr::LogMgr() + { + } + +LogMgr::~LogMgr() + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + delete *s; + } + +bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns) + { + // TODO: Should check that the record has only supported types. + + unsigned int idx = stream_id->AsEnum(); + + // Make sure the vector has an entries for all streams up to the one + // given. + while ( idx >= streams.size() ) + streams.push_back(0); + + if ( streams[idx] ) + // We already know this one, delete the previous definition. + delete streams[idx]; + + // Create new stream and record the type for the columns. + streams[idx] = new Stream; + streams[idx]->name = stream_id->Type()->AsEnumType()->Lookup(idx); + streams[idx]->columns = columns; + columns->Ref(); + + DBG_LOG(DBG_LOGGING, "Created new logging stream '%s'", streams[idx]->name.c_str()); + + return true; + } + +// Helper for recursive record field unrolling. +bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) + { + for ( int i = 0; i < rt->NumFields(); ++i ) + { + BroType* t = rt->FieldType(i); + + list new_indices = indices; + new_indices.push_back(i); + + // Build path name. + string new_path; + if ( ! path.size() ) + new_path = rt->FieldName(i); + else + new_path = path + "." + rt->FieldName(i); + + StringVal* new_path_val = new StringVal(path.c_str()); + + if ( t->InternalType() == TYPE_INTERNAL_OTHER ) + { + if ( t->Tag() == TYPE_RECORD ) + { + // Recurse. + if ( ! TraverseRecord(filter, t->AsRecordType(), include, exclude, new_path, new_indices) ) + return false; + } + else + { + run_time("unsupported field type for log column"); + return false; + } + + continue; + } + + // If include fields are specified, only include if explicitly listed. + if ( include ) + { + if ( ! include->Lookup(new_path_val) ) + return true; + } + + // If exclude fields are specified, do not only include if listed. + if ( exclude ) + { + if ( exclude->Lookup(new_path_val) ) + return true; + } + + // Alright, we want this field. + + filter->indices.push_back(new_indices); + filter->fields = (LogField**) realloc(filter->fields, sizeof(LogField) * ++filter->num_fields); + if ( ! filter->fields ) + { + run_time("out of memory in add_filter"); + return false; + } + + LogField* field = new LogField(); + field->name = new_path; + field->type = t->Tag(); + filter->fields[filter->num_fields - 1] = field; + } + + return true; + } + +bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) + { + RecordType* rtype = fval->Type()->AsRecordType(); + + if ( ! same_type(rtype, log_filter, 0) ) + { + run_time("filter argument not of right type"); + return false; + } + + Stream* stream = streams[stream_id->AsEnum()]; + if ( ! stream ) + { + run_time("undefined log stream"); + return false; + } + + // Find the right writer type. + int writer = 0; + int idx = rtype->FieldOffset("writer"); + Val* writer_val = fval->Lookup(idx); + + if ( ! writer_val ) + { + // Use default. + // FIXME: Shouldn't Lookup() already take care if this? + const Attr* def_attr = log_filter->FieldDecl(idx)->FindAttr(ATTR_DEFAULT); + if ( ! def_attr ) + internal_error("log_filter missing &default for writer attribute"); + + writer_val = def_attr->AttrExpr()->Eval(0); + writer = writer_val->AsEnum(); + Unref(writer_val); + } + else + writer = writer_val->AsEnum(); + + LogWriterDefinition* ld; + for ( ld = log_writers; ld->type != LogWriterType::None; ++ld ) + { + if ( ld->type == writer ) + break; + } + + if ( ld->type == LogWriterType::None ) + internal_error("unknow writer in add_filter"); + + if ( ! ld->factory ) + // Oops, we can't instantuate this guy. + return true; // Count as success, as we will have reported it earlier already. + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't call it again later. + ld->init = 0; + else + // Init failed, disable by deleting factory function. + ld->factory = 0; + return false; + } + + // Create a new Filter instance. + + Val* event = fval->Lookup(rtype->FieldOffset("ev")); + Val* pred = fval->Lookup(rtype->FieldOffset("pred")); + Val* path_func = fval->Lookup(rtype->FieldOffset("path_func")); + + Filter* filter = new Filter; + filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->pred = path_func ? path_func->AsFunc() : 0; + filter->writer = ld; + + if ( event ) + { + // TODO: Implement + filter->event = 0; + } + + // Build the list of fields that the filter wants included, including + // potentially rolling out fields. + Val* include = fval->Lookup(rtype->FieldOffset("include")); + Val* exclude = fval->Lookup(rtype->FieldOffset("exclude")); + + filter->num_fields = 0; + filter->fields = 0; + if ( ! TraverseRecord(filter, stream->columns, include ? include->AsTableVal() : 0, exclude ? exclude->AsTableVal() : 0, "", list()) ) + return false; + + // Get the path for the filter. + Val* path_val = fval->Lookup(rtype->FieldOffset("path")); + + if ( path_val ) + filter->path = path_val->AsString()->CheckString(); + + else + { + // If no path is given, use the Stream ID as the default. + const char* n = stream->name.c_str(); + char* lower = new char[strlen(n) + 1]; + for ( char* s = lower; *n; ++n, ++s ) + { + if ( strncmp(n, "::", 2) == 0 ) + { + // Remove the scope operator. TODO: We need ab better way to + // generate the default here, but let's wait until we have + // everything in the right namespace. + *s = '_'; + ++n; + } + + else + *s = tolower(*n); + } + + filter->path = string(lower); + free(lower); + } + + stream->filters.push_back(filter); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Created new filter '%s' for stream '%s'", filter->name.c_str(), stream->name.c_str()); + DBG_LOG(DBG_LOGGING, " writer : %s", ld->name); + DBG_LOG(DBG_LOGGING, " path : %s", filter->path.c_str()); + DBG_LOG(DBG_LOGGING, " path_func : %s", (filter->path_func ? "set" : "not set")); + DBG_LOG(DBG_LOGGING, " event : %s", (filter->event ? "set" : "not set")); + DBG_LOG(DBG_LOGGING, " pred : %s", (filter->pred ? "set" : "not set")); + + for ( int i = 0; i < filter->num_fields; i++ ) + { + LogField* field = filter->fields[i]; + DBG_LOG(DBG_LOGGING, " field %10s: %s", field->name.c_str(), type_name(field->type)); + } +#endif + + return true; + } + +bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) + { +#if 0 + int idx = stream_id->AsEnum(); + + if ( idx >= streams.size() || ! streams[idx] ) + { + run_time("unknown log stream"); + return false; + } + + +#endif + return true; + } + +bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) + { + unsigned int idx = stream_id->AsEnum(); + + if ( idx >= streams.size() || ! streams[idx] ) + { + run_time("unknown log stream"); + return false; + } + + Stream* stream = streams[idx]; + + columns = columns->CoerceTo(stream->columns); + + if ( ! columns ) + { + run_time("imcompatible log record type"); + return false; + } + + // Send to each of our filters. + for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + { + Filter* filter = *i; + + string path = filter->path; + + if ( filter->event ) + { + // XXX Raise event here. + // TODO: Actually, the filter should be an attribute of the stream, right? + } + + if ( filter->pred ) + { + // XXX Check predicate here. + } + + if ( filter->path_func ) + { + // XXX Do dynamic path here. + } + + // See if we already have a writer for this path. + Filter::WriterMap::iterator w = filter->writers.find(path); + + LogWriter* writer = 0; + if ( w == filter->writers.end() ) + { + // No, need to create one. + assert(filter->writer->factory); + writer = (*filter->writer->factory)(); + + // Copy the fields for LogWriter::Init() as it will take + // ownership. + LogField** arg_fields = new LogField*[filter->num_fields]; + for ( int j = 0; j < filter->num_fields; ++j ) + arg_fields[j] = new LogField(*filter->fields[j]); + + if ( ! writer->Init(path, filter->num_fields, arg_fields) ) + { + Unref(columns); + return false; + } + + filter->writers.insert(Filter::WriterMap::value_type(path, writer)); + } + + else + // We have a writer already. + writer = w->second; + + // Alright, can do the write now. + LogVal** vals = RecordToFilterVals(filter, columns); + writer->Write(vals); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", filter->name.c_str(), stream->name.c_str()); +#endif + } + + Unref(columns); + return true; + } + +LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) + { + LogVal** vals = new LogVal*[filter->num_fields]; + + for ( int i = 0; i < filter->num_fields; ++i ) + { + Val* val = columns; + + // For each field, first find the right value, which can potentially + // be nested inside other records. + list& indices = filter->indices[i]; + + for ( list::iterator j = indices.begin(); j != indices.end(); ++j ) + { + val = val->AsRecordVal()->Lookup(*j); + + if ( ! val ) + { + // Value, or any of its parents, is not set. + vals[i] = new LogVal(false); + break; + } + } + + if ( ! val ) + continue; + + switch ( val->Type()->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + vals[i] = new LogVal(); + vals[i]->val.int_val = val->InternalInt(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + vals[i] = new LogVal(); + vals[i]->val.uint_val = val->InternalUnsigned(); + break; + + case TYPE_SUBNET: + vals[i] = new LogVal(); + vals[i]->val.subnet_val = *val->AsSubNet(); + break; + + case TYPE_NET: + case TYPE_ADDR: + { + vals[i] = new LogVal(); + addr_type t = val->AsAddr(); + copy_addr(&t, &vals[i]->val.addr_val); + break; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + vals[i] = new LogVal(); + vals[i]->val.double_val = val->InternalDouble(); + break; + + case TYPE_STRING: + { + const BroString* s = val->AsString(); + LogVal* lval = (LogVal*) new char[sizeof(LogVal) + sizeof(log_string_type) + s->Len()]; + new (lval) LogVal(); // Run ctor. + lval->val.string_val.len = s->Len(); + memcpy(&lval->val.string_val.string, s->Bytes(), s->Len()); + vals[i] = lval; + break; + } + + default: + internal_error("unsupported type for log_write"); + } + } + + return vals; + } + + +void LogMgr::Error(LogWriter* writer, const char* msg) + { +#if 0 +#endif + } diff --git a/src/LogMgr.h b/src/LogMgr.h new file mode 100644 index 0000000000..b62dd5ff35 --- /dev/null +++ b/src/LogMgr.h @@ -0,0 +1,82 @@ +// +// A class managing log writers and filters. + +#ifndef LOGMGR_H +#define LOGMGR_H + +#include "Val.h" + +// One value per writer type we have. +namespace LogWriterType { + enum Type { + None, + Ascii + }; +}; + +struct LogField { + LogField() { } + LogField(const LogField& other) : name(other.name), type(other.type) { } + string name; + TypeTag type; +}; + +// A string that we can directly include as part of the value union below. +struct log_string_type { + int len; + char string[]; // The string starts right here. +}; + +// All values that can be directly logged by a Writer. +struct LogVal { + LogVal(bool arg_present = true) : present(arg_present) {} + + bool present; // If false, the field is unset (i.e., &optional and not initialzed). + + // The following union is a subset of BroValUnion, including only the + // atomic types. + union { + bro_int_t int_val; + bro_uint_t uint_val; + addr_type addr_val; + subnet_type subnet_val; + double double_val; + log_string_type string_val; + } val; +}; + +class LogWriter; + +class LogMgr { +public: + LogMgr(); + ~LogMgr(); + + // These correspond to the BiFs visible on the scripting layer. The + // actual BiFs just forward here. + bool CreateStream(EnumVal* stream_id, RecordType* columns); + bool AddFilter(EnumVal* stream_id, RecordVal* filter); + bool RemoveFilter(EnumVal* stream_id, StringVal* filter); + bool Write(EnumVal* stream_id, RecordVal* columns); + +protected: + friend class LogWriter; + + /// Functions also used by the writers. + + // Reports an error for the given writer. + void Error(LogWriter* writer, const char* msg); + +private: + struct Filter; + struct Stream; + + bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); + LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); + + vector streams; // Indexed by stream enum. +}; + +extern LogMgr* log_mgr; + +#endif diff --git a/src/LogWriter.cc b/src/LogWriter.cc new file mode 100644 index 0000000000..903ac46f8b --- /dev/null +++ b/src/LogWriter.cc @@ -0,0 +1,76 @@ + +#include "util.h" +#include "LogWriter.h" + +LogWriter::LogWriter() + { + buf = 0; + buf_len = 1024; + } + +LogWriter::~LogWriter() + { + if ( buf ) + free(buf); + + delete [] fields; + } + +bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields) + { + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + DoInit(arg_path, arg_num_fields, arg_fields); + return true; + } + +bool LogWriter::Write(LogVal** vals) + { + bool result = DoWrite(num_fields, fields, vals); + DeleteVals(vals); + return result; + } + +void LogWriter::Finish() + { + DoFinish(); + } + +const char* LogWriter::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + + +void LogWriter::Error(const char *msg) + { + run_time(msg); + } + +void LogWriter::DeleteVals(LogVal** vals) + { + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + } + + diff --git a/src/LogWriter.h b/src/LogWriter.h new file mode 100644 index 0000000000..02a8c03ad5 --- /dev/null +++ b/src/LogWriter.h @@ -0,0 +1,82 @@ +// +// Interface API for a log writer backend. +// +// Note than classes derived from LogWriter must be fully thread-safe and not +// use any non-safe Bro functionality (which is almost all ...). In +// particular, do not use fmt() but LogWriter::Fmt()!. + +#ifndef LOGWRITER_H +#define LOGWRITER_H + +#include "LogMgr.h" +#include "BroString.h" + +class LogWriter { +public: + LogWriter(); + virtual ~LogWriter(); + + // One-time initialization of the writer, defining the logged fields. + // Interpretation of "path" is left to the writer, and will be the value + // configured on the script-level. Returns false if an error occured, in + // which case the writer must not be used futher. + // + // The new instance takes ownership of "fields", and will delete them + // when done. + bool Init(string path, int num_fields, LogField** fields); + + // Writes one log entry. The method takes ownership of "vals" and will + // return immediately after queueing the write request, potentially + // before the output has actually taken place. Returns false if an error + // occured, in which case the writer must not be used further. + bool Write(LogVal** vals); + + // Finished writing to this logger. Will not be called if an error has + // been indicated earlier. After calling this, no more writing must be + // performed. + void Finish(); + +protected: + + //// Methods for Writers to override. + + // Called once for initialization of the Writer. Must return false if an + // error occured, in which case the writer will be disabled. The error + // reason should be reported via Error(). + virtual bool DoInit(string path, int num_fields, LogField** fields) = 0; + + // Called once per entry to record. Must return false if an error + // occured, in which case the writer will be disabled. The error reason + // should be reported via Error(). + virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; + + // Called once on termination. Not called when any of the other methods + // has previously signaled an error, i.e., executing this method signals + // a regular shutdown. + virtual void DoFinish() = 0; + + //// Methods for Writers to use. These are thread-safe. + + // A thread-safe version of fmt(). + const char* Fmt(const char* format, ...); + + // Reports an error. + void Error(const char *msg); + + // Returns the path as passed to Init(). + const string Path() const { return path; } + +private: + // Delete values as passed into Write(). + void DeleteVals(LogVal** vals); + + string path; + int num_fields; + LogField** fields; + + // For Fmt(). + char* buf; + unsigned int buf_len; +}; + +#endif diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc new file mode 100644 index 0000000000..e06ba4f9dd --- /dev/null +++ b/src/LogWriterAscii.cc @@ -0,0 +1,129 @@ + +#include +#include + +#include "LogWriterAscii.h" + +LogWriterAscii::LogWriterAscii() + { + fname = 0; + file = 0; + } + +LogWriterAscii::~LogWriterAscii() + { + if ( fname ) + free(fname); + + if ( file ) + fclose(file); + } + +bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) + { + fname = strdup(Fmt("%s.log", path.c_str())); + + if ( ! (file = fopen(fname, "w")) ) + { + Error(Fmt("cannot open %s: %s", fname, strerror(errno))); + return false; + } + + if ( fputs("# ", file) == EOF ) + goto write_error; + + for ( int i = 0; i < num_fields; i++ ) + { + LogField* field = fields[i]; + if ( fputs(field->name.c_str(), file) == EOF ) + goto write_error; + + if ( fputc('\t', file) == EOF ) + goto write_error; + } + + if ( fputc('\n', file) == EOF ) + goto write_error; + + return true; + +write_error: + Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + return false; + } + +void LogWriterAscii::DoFinish() + { + } + +bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) + { + ODesc desc(DESC_READABLE); + + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 ) + desc.Add("\t"); + + LogVal* val = vals[i]; + LogField* field = fields[i]; + + if ( ! val->present ) + { + desc.Add("-"); // TODO: Probably want to get rid of the "-". + continue; + } + + switch ( field->type ) { + case TYPE_BOOL: + desc.Add(val->val.int_val ? "T" : "F"); + break; + + case TYPE_INT: + case TYPE_ENUM: + desc.Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + desc.Add(val->val.uint_val); + break; + + case TYPE_SUBNET: + desc.Add(dotted_addr(val->val.subnet_val.net)); + desc.Add("/"); + desc.Add(val->val.subnet_val.width); + break; + + case TYPE_NET: + case TYPE_ADDR: + desc.Add(dotted_addr(val->val.addr_val)); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + desc.Add(val->val.double_val); + break; + + case TYPE_STRING: + desc.AddN((const char*)&val->val.string_val.string, val->val.string_val.len); + break; + + default: + Error(Fmt("unsupported field format %d for %s", field->type, field->name.c_str())); + return false; + } + } + + desc.Add("\n"); + + if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 ) + { + Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + return false; + } + + return true; + } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h new file mode 100644 index 0000000000..4672d2ac54 --- /dev/null +++ b/src/LogWriterAscii.h @@ -0,0 +1,27 @@ +// +// Log writer for tab-separated ASCII logs. +// + +#ifndef LOGWRITERASCII_H +#define LOGWRITERASCII_H + +#include "LogWriter.h" + +class LogWriterAscii : public LogWriter { +public: + LogWriterAscii(); + ~LogWriterAscii(); + + static LogWriter* Instantiate() { return new LogWriterAscii; } + +protected: + bool DoInit(string path, int num_fields, LogField** fields); + bool DoWrite(int num_fields, LogField** fields, LogVal** vals); + void DoFinish(); + +private: + FILE* file; + char* fname; +}; + +#endif diff --git a/src/NetVar.cc b/src/NetVar.cc index 0af742ef3e..231d2efe78 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -260,6 +260,8 @@ int record_all_packets; RecordType* script_id; TableType* id_table; +RecordType* log_filter; + #include "const.bif.netvar_def" #include "event.bif.netvar_def" @@ -564,4 +566,6 @@ void init_net_var() script_id = internal_type("script_id")->AsRecordType(); id_table = internal_type("id_table")->AsTableType(); + + log_filter = internal_type("log_filter")->AsRecordType(); } diff --git a/src/NetVar.h b/src/NetVar.h index 7461ec8be0..ccd8294882 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,6 +264,8 @@ extern int record_all_packets; extern RecordType* script_id; extern TableType* id_table; +extern RecordType* log_filter; + // Initializes globals that don't pertain to network/event analysis. extern void init_general_global_var(); diff --git a/src/bro.bif b/src/bro.bif index 278c14500e..52c54706c6 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -359,7 +359,7 @@ function cat%(...%): string return new StringVal(s); %} - + function logging_log%(index: string, rec: any%): any %{ // TODO: Verify that rec is a record @@ -384,7 +384,7 @@ function logging_log%(index: string, rec: any%): any printf("Logging framework is dead (Logging::streams not found).\n"); return false; } - + // Lookup all filters for stream TableVal *filters = opt_internal_table("Logging::filters"); TableVal *stream_filters; @@ -399,8 +399,8 @@ function logging_log%(index: string, rec: any%): any printf("Logging framework is dead (Logging::filters not found).\n"); return false; } - - // Generate the event for the log stream + + // Generate the event for the log stream // This happens regardless of all filters. int name_field = stream_record->Type()->AsRecordType()->FieldOffset("name"); StringVal *log_type = stream_record->AsRecordVal()->Lookup(name_field)->AsStringVal(); @@ -414,7 +414,7 @@ function logging_log%(index: string, rec: any%): any vl->append(recval->Ref()); mgr.QueueEvent(ev_ptr, vl, SOURCE_LOCAL); } - + // Iterate over the stream_filters ListVal* filter_recs = stream_filters->ConvertToList(TYPE_ANY); for ( int i = 0; i < filter_recs->Length(); ++i ) @@ -432,10 +432,10 @@ function logging_log%(index: string, rec: any%): any if ( !pred_val ) continue; } - - // Format and print the line + + // Format and print the line // (send line onward to the filter's WRITER in the future) - + // Get a path name for this filter int path_field = rv->Type()->AsRecordType()->FieldOffset("path"); Val *path_val = rv->Lookup(path_field); @@ -445,7 +445,7 @@ function logging_log%(index: string, rec: any%): any path = path_val->AsStringVal(); else path = index; - + // Get the file with the "path" name found above for this filter. // Open a new file is one does not exist yet. TableVal *logging_files = opt_internal_table("Logging::files"); @@ -474,20 +474,61 @@ function logging_log%(index: string, rec: any%): any } f->Write(d.Description(), 0); f->Write("\n",0); - - + + } Unref(filter_recs); - + return 0; %} - + + +%%{ +#include "LogMgr.h" +%%} + +function log_create_stream%(id: Log_ID, columns: any%) : bool + %{ + if ( columns->Type()->Tag() != TYPE_TYPE ) + { + run_time("log columns must be a type"); + return new Val(0, TYPE_BOOL); + } + + if ( columns->Type()->AsTypeType()->Type()->Tag() != TYPE_RECORD ) + { + run_time("log columns must be a record type"); + return new Val(0, TYPE_BOOL); + } + + bool result = log_mgr->CreateStream(id->AsEnumVal(), columns->Type()->AsTypeType()->Type()->AsRecordType()); + return new Val(result, TYPE_BOOL); + %} + +function log_add_filter%(id: Log_ID, filter: log_filter%) : bool + %{ + bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + +function log_remove_filter%(id: Log_ID, name: string%) : bool + %{ + bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name); + return new Val(result, TYPE_BOOL); + %} + +function log_write%(id: Log_ID, columns: any%) : bool + %{ + bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + function record_type_to_vector%(rt: string%): string_vec %{ VectorVal* result = new VectorVal(internal_type("string_vec")->AsVectorType()); - + RecordType *type = internal_type(rt->CheckString())->AsRecordType(); if ( type ) { @@ -499,8 +540,8 @@ function record_type_to_vector%(rt: string%): string_vec } return result; %} - - + + function cat_sep%(sep: string, def: string, ...%): string %{ ODesc d; @@ -3164,7 +3205,7 @@ function lookup_location%(a: addr%) : geo_location #ifdef GEOIP_COUNTRY_EDITION_V6 if ( geoip_v6 && ! is_v4_addr(a) ) gir = GeoIP_record_by_ipnum_v6(geoip_v6, geoipv6_t(a)); - else + else #endif if ( geoip && is_v4_addr(a) ) { @@ -3252,7 +3293,7 @@ function lookup_asn%(a: addr%) : count gir = GeoIP_name_by_ipnum_v6(geoip_asn, geoipv6_t(a)); else #endif - if ( is_v4_addr(a) ) + if ( is_v4_addr(a) ) { uint32 addr = to_v4_addr(a); gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(addr)); @@ -3264,7 +3305,7 @@ function lookup_asn%(a: addr%) : count if ( gir ) { - // Move the pointer +2 so we don't return + // Move the pointer +2 so we don't return // the first two characters: "AS". return new Val(atoi(gir+2), TYPE_COUNT); } diff --git a/src/logging.bif b/src/logging.bif new file mode 100644 index 0000000000..955b3c293f --- /dev/null +++ b/src/logging.bif @@ -0,0 +1,23 @@ + +enum log_writer %{ + WRITER_DEFAULT, + WRITER_TSV, + WRITER_SYSLOG, +}% + +void log_create%(id: Logging::Stream, columns: any%) + %{ + %} + +void log_add_filter%(id: Logging::Stream, filter: any%) + %{ + %} + +void log_remove_filter%(id: Logging::Stream, name: string%) + %{ + %} + +void log_write%(id: Logging::Stream, columns: any%) + %{ + %} + diff --git a/src/logging.bro b/src/logging.bro new file mode 100644 index 0000000000..6031826351 --- /dev/null +++ b/src/logging.bro @@ -0,0 +1,52 @@ + +module Log; + +export { + # Each stream gets a unique ID. This type will be extended by + # other scripts. + type Stream: enum { + Unknown, + Info, + Debug, + }; + + # The default writer to use if a filter does not specify + # anything else. + const default_writer = WRITER_CSV &redef; + + # A filter defining what to record. + type Filter: record { + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(log: any) &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: any &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + select: set[string] &optional; + + # An event that is raised whenever the filter is applied + # to an entry. The event receives the same parameter + # as the predicate. It will always be generated, + # independent of what the predicate returns. + ev: event(l: any) &optional; + + # The writer to use. + writer: Writer &default=default_writer; + }; +} diff --git a/src/main.cc b/src/main.cc index 82866302fd..141eec7ab3 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Event.h" #include "File.h" #include "Logger.h" +#include "LogMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -71,6 +72,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; Logger* bro_logger; +LogMgr* log_mgr; Func* alarm_hook = 0; Stmt* stmts; EventHandlerPtr bro_signal = 0; @@ -289,6 +291,7 @@ void terminate_bro() delete conn_compressor; delete remote_serializer; delete dpm; + delete log_mgr; } void termination_signal() @@ -724,7 +727,8 @@ int main(int argc, char** argv) persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); - event_registry = new EventRegistry; + event_registry = new EventRegistry(); + log_mgr = new LogMgr(); if ( events_file ) event_player = new EventPlayer(events_file); From 56880da77958d87c52a417802842fe3dad1217bd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Feb 2011 13:11:55 -0800 Subject: [PATCH 16/67] Adding a TODO for the internal logging stuff. --- TODO.logging | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 TODO.logging diff --git a/TODO.logging b/TODO.logging new file mode 100644 index 0000000000..886049b5f0 --- /dev/null +++ b/TODO.logging @@ -0,0 +1,13 @@ +List of the things not implemented yet: + + - Removing filters + - Filter predicates + - Filter events + - Dynamic path function. + - Cluster-style remote_print + - Rotation support + - Spawning writers in separate threads (not clear if we want that initially). + - Not sure if the logging does the right thing with &optional and + &default values. Needs testing. + +There's probably more missing. From 3fbb3c0fcd57ec7baa9f602e30de95fd62af9644 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 13:45:07 -0800 Subject: [PATCH 17/67] Missing piece for previous patch adding type vals. --- src/Type.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Type.cc b/src/Type.cc index 31a04bfeb3..4c3d5bba8f 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -27,6 +27,7 @@ const char* type_name(TypeTag t) "func", "file", "vector", + "type", "error", }; From c0cd62a5a5e7270db901d56a1d651df7bd87338f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 13:45:44 -0800 Subject: [PATCH 18/67] Enable passing events into bifs. When an event was globally decleared, previously it did not get assigned a value initially until the first implementation body was added. That then triggered an "not used" error when passing such an event as argument into a bif. Now we always assign a function value immediately, just without any body inititally. When globally declaring an event, i --- src/Func.cc | 12 ++++++++---- src/Var.cc | 10 ++++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Func.cc b/src/Func.cc index 5d71be2b0f..f398425216 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -239,11 +239,15 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits, : Func(BRO_FUNC) { id = arg_id; - Body b; - b.stmts = AddInits(arg_body, aggr_inits); - b.priority = 0; - bodies.push_back(b); frame_size = arg_frame_size; + + if ( arg_body ) + { + Body b; + b.stmts = AddInits(arg_body, aggr_inits); + b.priority = 0; + bodies.push_back(b); + } } BroFunc::~BroFunc() diff --git a/src/Var.cc b/src/Var.cc index b107156b3a..c9e497d8a5 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -170,6 +170,16 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, } id->UpdateValAttrs(); + + if ( t && t->Tag() == TYPE_FUNC && t->AsFuncType()->IsEvent() ) + { + // For events, add a function value (without any body) here so that + // we can later access the ID even if no implementations have been + // defined. + Func* f = new BroFunc(id, 0, 0, 0); + id->SetVal(new Val(f)); + id->SetConst(); + } } From b98d5adcd578ae04f1b58c0b7027796174529b48 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 14:13:06 -0800 Subject: [PATCH 19/67] Fixing case of an event handler having no bodies. To sqash later. --- src/Func.cc | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Func.cc b/src/Func.cc index f398425216..d9c3aa7bdf 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -271,6 +271,13 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const #ifdef PROFILE_BRO_FUNCTIONS DEBUG_MSG("Function: %s\n", id->Name()); #endif + if ( ! bodies.size() ) + { + // Can only happen for events. + assert(IsEvent()); + return 0 ; + } + SegmentProfiler(segment_logger, location); Frame* f = new Frame(frame_size, this, args); From 091547de4f54fe28c1a92507f7477baf765bf2ed Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 14:13:49 -0800 Subject: [PATCH 20/67] Preparing LogWriter API for rotation and flushing. --- src/LogWriter.h | 31 +++++++++++++++++++++++++++++++ src/LogWriterAscii.cc | 11 +++++++++++ src/LogWriterAscii.h | 8 +++++--- 3 files changed, 47 insertions(+), 3 deletions(-) diff --git a/src/LogWriter.h b/src/LogWriter.h index 02a8c03ad5..2b8d321cd2 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -50,6 +50,37 @@ protected: // should be reported via Error(). virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; + // Called when the flushing status for this writer is changed. If + // flushing is enabled, the writer should attempt to write out + // information as quickly as possible even if that may have an + // performance impact. If disabled (which the writer should assume to be + // the default), then it can buffer things up as necessary and write out + // in a way optimized for performance. + // + // A writer may ignore flushing if it doesn't fit with its semantics. + virtual void DoSetFlushing(bool enabled) = 0; + + // Called when a log output is to be rotated. Most directly, this only + // applies to writers outputting files, thoug a writer may also trigger + // other regular actions if that fits a similar model + // + // The string "rotate_path" is interpreted in writer-specific way, yet + // should generally should have similar semantics that the "path" passed + // into DoInit(), except that now it reflects the name to where the + // rotated output is to be moved. After rotation, output should continue + // normally with the standard "path". As an example, for file-based + // output, the rotate_path may be the original filename with an embedded + // timestamp. + // + // The writer must return false if an error occured that prevent the + // writer for continuing operation; it will then be disabled. The error + // reason should be reported via Error(). If a recoverable error occurs, + // still call Error(), but return true. + // + // A writer may ignore rotation requests if it doesn't fit with its + // semantics. In that case, still return true. + virtual bool DoRotate(string rotated_path) = 0; + // Called once on termination. Not called when any of the other methods // has previously signaled an error, i.e., executing this method signals // a regular shutdown. diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index e06ba4f9dd..69b9993778 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -127,3 +127,14 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) return true; } + +bool LogWriterAscii::DoRotate(string rotated_path) + { + return true; + } + +void LogWriterAscii::DoSetFlushing(bool enabled) + { + } + + diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 4672d2ac54..fa014fbc81 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -15,9 +15,11 @@ public: static LogWriter* Instantiate() { return new LogWriterAscii; } protected: - bool DoInit(string path, int num_fields, LogField** fields); - bool DoWrite(int num_fields, LogField** fields, LogVal** vals); - void DoFinish(); + virtual bool DoInit(string path, int num_fields, LogField** fields); + virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals); + virtual void DoSetFlushing(bool enabled); + virtual bool DoRotate(string rotated_path); + virtual void DoFinish(); private: FILE* file; From f6da93992c3023ef53b5e8c72ea363237b034068 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 14:14:33 -0800 Subject: [PATCH 21/67] A log write now raises the corresponding event. The event has moved from the filters to the streams, and must now be specificed when creating the stream. (Not clear yet whether that is a indeed the right interface). --- TODO.logging | 12 +++++++++++- policy/bro.init | 8 +------- policy/test-logging.bro | 18 ++++++++++++------ src/LogMgr.cc | 29 +++++++++++++---------------- src/LogMgr.h | 3 ++- src/bro.bif | 32 ++++++++++++++++++++++++++++---- 6 files changed, 67 insertions(+), 35 deletions(-) diff --git a/TODO.logging b/TODO.logging index 886049b5f0..8d1be59e20 100644 --- a/TODO.logging +++ b/TODO.logging @@ -2,12 +2,22 @@ List of the things not implemented yet: - Removing filters - Filter predicates - - Filter events - Dynamic path function. - Cluster-style remote_print - Rotation support + - Flushing support - Spawning writers in separate threads (not clear if we want that initially). - Not sure if the logging does the right thing with &optional and &default values. Needs testing. + - Seems we could do some of the filter-related type checks + currently done dynamically at startup via a TraversalCallback. There's probably more missing. + +Question: + + * Is giving the record column twice to create_stream too + redundant? (once directly, and once via the event):: + + global ssh_log: event(rec: Log); + log_create_stream(LOG_SSH, SSH::Log, ssh_log); diff --git a/policy/bro.init b/policy/bro.init index 47fa336939..0b727fcdc5 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -308,19 +308,13 @@ type log_filter: record { # extension is given; the writer will add whatever is # appropiate. path: string &optional; - path_func: function(id: string): string &optional; + path_func: function(id: Log_ID): string &optional; # A subset of column names to record. If not given, all # columns are recorded. include: set[string] &optional; exclude: set[string] &optional; - # An event that is raised whenever the filter is applied - # to an entry. The event receives the same parameter - # as the predicate. It will always be generated, - # independent of what the predicate returns. - ev: event(rec: any) &optional; - # The writer to use. writer: Log_Writer &default=Log_default_writer; }; diff --git a/policy/test-logging.bro b/policy/test-logging.bro index e8ccc7c22e..1a95fed7d2 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -16,12 +16,15 @@ export { }; } +global ssh_log: event(rec: Log); + event bro_init() { # Create the stream. # First argument is the ID for the stream. # Second argument is the log record type. - log_create_stream(LOG_SSH, SSH::Log); + # Third argument is the log event, which must receive a single argument of type arg2. + log_create_stream(LOG_SSH, SSH::Log, ssh_log); # Add a default filter that simply logs everything to "ssh.log" using the default writer. Log_add_default_filter(LOG_SSH); @@ -36,8 +39,10 @@ event bro_init() local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + # Log something. - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); + log_write(LOG_SSH, r); log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); @@ -45,10 +50,11 @@ event bro_init() } -#event log(rec: Log) -# { -# print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); -# } +event ssh_log(rec: Log) + { + print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); + print rec; + } # # #module WHATEVER; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index b9675e4e84..0fe40620e1 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -1,5 +1,6 @@ #include "LogMgr.h" +#include "Event.h" #include "EventHandler.h" #include "NetVar.h" @@ -23,7 +24,6 @@ struct LogMgr::Filter { string name; Func* pred; Func* path_func; - EventHandlerPtr* event; string path; LogWriterDefinition* writer; @@ -40,6 +40,7 @@ struct LogMgr::Filter { struct LogMgr::Stream { string name; RecordType* columns; + EventHandlerPtr event; list filters; ~Stream(); @@ -72,7 +73,7 @@ LogMgr::~LogMgr() delete *s; } -bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns) +bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerPtr handler) { // TODO: Should check that the record has only supported types. @@ -91,9 +92,10 @@ bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns) streams[idx] = new Stream; streams[idx]->name = stream_id->Type()->AsEnumType()->Lookup(idx); streams[idx]->columns = columns; + streams[idx]->event = handler; columns->Ref(); - DBG_LOG(DBG_LOGGING, "Created new logging stream '%s'", streams[idx]->name.c_str()); + DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", streams[idx]->name.c_str(), streams[idx]->event->Name()); return true; } @@ -242,12 +244,6 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) filter->pred = path_func ? path_func->AsFunc() : 0; filter->writer = ld; - if ( event ) - { - // TODO: Implement - filter->event = 0; - } - // Build the list of fields that the filter wants included, including // potentially rolling out fields. Val* include = fval->Lookup(rtype->FieldOffset("include")); @@ -295,7 +291,6 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) DBG_LOG(DBG_LOGGING, " writer : %s", ld->name); DBG_LOG(DBG_LOGGING, " path : %s", filter->path.c_str()); DBG_LOG(DBG_LOGGING, " path_func : %s", (filter->path_func ? "set" : "not set")); - DBG_LOG(DBG_LOGGING, " event : %s", (filter->event ? "set" : "not set")); DBG_LOG(DBG_LOGGING, " pred : %s", (filter->pred ? "set" : "not set")); for ( int i = 0; i < filter->num_fields; i++ ) @@ -344,6 +339,14 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) return false; } + // Raise the log event. + if ( stream->event ) + { + val_list* vl = new val_list; + vl->append(columns->Ref()); + mgr.QueueEvent(stream->event, vl, SOURCE_LOCAL); + } + // Send to each of our filters. for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) { @@ -351,12 +354,6 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) string path = filter->path; - if ( filter->event ) - { - // XXX Raise event here. - // TODO: Actually, the filter should be an attribute of the stream, right? - } - if ( filter->pred ) { // XXX Check predicate here. diff --git a/src/LogMgr.h b/src/LogMgr.h index b62dd5ff35..28a06821db 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -5,6 +5,7 @@ #define LOGMGR_H #include "Val.h" +#include "EventHandler.h" // One value per writer type we have. namespace LogWriterType { @@ -54,7 +55,7 @@ public: // These correspond to the BiFs visible on the scripting layer. The // actual BiFs just forward here. - bool CreateStream(EnumVal* stream_id, RecordType* columns); + bool CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerPtr handler); bool AddFilter(EnumVal* stream_id, RecordVal* filter); bool RemoveFilter(EnumVal* stream_id, StringVal* filter); bool Write(EnumVal* stream_id, RecordVal* columns); diff --git a/src/bro.bif b/src/bro.bif index 52c54706c6..e52e30ab1d 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -488,21 +488,45 @@ function logging_log%(index: string, rec: any%): any #include "LogMgr.h" %%} -function log_create_stream%(id: Log_ID, columns: any%) : bool +function log_create_stream%(id: Log_ID, columns: any, ev: any%) : bool %{ if ( columns->Type()->Tag() != TYPE_TYPE ) { run_time("log columns must be a type"); - return new Val(0, TYPE_BOOL); + return new Val(0, TYPE_BOOL); } if ( columns->Type()->AsTypeType()->Type()->Tag() != TYPE_RECORD ) { run_time("log columns must be a record type"); - return new Val(0, TYPE_BOOL); + return new Val(0, TYPE_BOOL); } - bool result = log_mgr->CreateStream(id->AsEnumVal(), columns->Type()->AsTypeType()->Type()->AsRecordType()); + if ( ev->Type()->Tag() != TYPE_FUNC || ! ev->Type()->AsFuncType()->IsEvent() ) + { + run_time("event argument is not of event type"); + return new Val(0, TYPE_BOOL); + } + + FuncType* et = ev->Type()->AsFuncType(); + type_list* args = et->ArgTypes()->Types(); + + if ( args->length() != 1 ) + { + run_time("event must take a single argument"); + return new Val(0, TYPE_BOOL); + } + + if ( ! same_type((*args)[0], columns->Type()->AsTypeType()->Type()) ) + { + run_time("event argument type does not match log record type"); + return new Val(0, TYPE_BOOL); + } + + EventHandler* handler = event_registry->Lookup(ev->AsFunc()->GetID()->Name()); + assert(handler); + + bool result = log_mgr->CreateStream(id->AsEnumVal(), columns->Type()->AsTypeType()->Type()->AsRecordType(), handler); return new Val(result, TYPE_BOOL); %} From a798200dbfb2066bfe84694cce2fa5a2d5d0d72a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 14:30:11 -0800 Subject: [PATCH 22/67] Adding predicate support. Now the filter's $pred field controls what is being logged. --- TODO.logging | 1 - policy/test-logging.bro | 8 ++++++++ src/LogMgr.cc | 16 +++++++++++++--- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/TODO.logging b/TODO.logging index 8d1be59e20..55811683a6 100644 --- a/TODO.logging +++ b/TODO.logging @@ -1,7 +1,6 @@ List of the things not implemented yet: - Removing filters - - Filter predicates - Dynamic path function. - Cluster-style remote_print - Rotation support diff --git a/policy/test-logging.bro b/policy/test-logging.bro index 1a95fed7d2..dea93be80e 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -18,6 +18,11 @@ export { global ssh_log: event(rec: Log); +function fail(rec: Log): bool + { + return rec$status != "success"; + } + event bro_init() { # Create the stream. @@ -29,6 +34,9 @@ event bro_init() # Add a default filter that simply logs everything to "ssh.log" using the default writer. Log_add_default_filter(LOG_SSH); + log_add_filter(LOG_SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); + log_add_filter(LOG_SSH, [$name="f2", $path="ssh.failure", $pred=fail]); + # Printing headers for the filters doesn't work yet either and needs to # be considered in the final design. (based on the "select" set). #Log::add_filter("ssh", [$name="successful logins", diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0fe40620e1..86d80ae5b0 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -241,9 +241,11 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) Filter* filter = new Filter; filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); filter->pred = pred ? pred->AsFunc() : 0; - filter->pred = path_func ? path_func->AsFunc() : 0; + filter->path_func = path_func ? path_func->AsFunc() : 0; filter->writer = ld; + // TODO: Check that the predciate is of the right type. + // Build the list of fields that the filter wants included, including // potentially rolling out fields. Val* include = fval->Lookup(rtype->FieldOffset("include")); @@ -342,7 +344,7 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) // Raise the log event. if ( stream->event ) { - val_list* vl = new val_list; + val_list* vl = new val_list(1); vl->append(columns->Ref()); mgr.QueueEvent(stream->event, vl, SOURCE_LOCAL); } @@ -356,7 +358,15 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) if ( filter->pred ) { - // XXX Check predicate here. + // See whether the predicates indicates that we want to log this record. + val_list vl(1); + vl.append(columns->Ref()); + Val* v = filter->pred->Call(&vl); + int result = v->AsBool(); + Unref(v); + + if ( ! result ) + continue; } if ( filter->path_func ) From 28216c84a26e228bbda9429370951d92c95abe30 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 16:45:35 -0800 Subject: [PATCH 23/67] Implemented removing of log filters. --- TODO.logging | 1 - src/LogMgr.cc | 30 ++++++++++++++++++++++++------ 2 files changed, 24 insertions(+), 7 deletions(-) diff --git a/TODO.logging b/TODO.logging index 55811683a6..88a4e74fba 100644 --- a/TODO.logging +++ b/TODO.logging @@ -1,6 +1,5 @@ List of the things not implemented yet: - - Removing filters - Dynamic path function. - Cluster-style remote_print - Rotation support diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 86d80ae5b0..ab36a26758 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -179,13 +179,16 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) return false; } - Stream* stream = streams[stream_id->AsEnum()]; - if ( ! stream ) + unsigned int i = stream_id->AsEnum(); + + if ( i >= streams.size() || ! streams[i] ) { - run_time("undefined log stream"); + run_time("unknown log stream"); return false; } + Stream* stream = streams[i]; + // Find the right writer type. int writer = 0; int idx = rtype->FieldOffset("writer"); @@ -307,8 +310,7 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) { -#if 0 - int idx = stream_id->AsEnum(); + unsigned int idx = stream_id->AsEnum(); if ( idx >= streams.size() || ! streams[idx] ) { @@ -316,8 +318,24 @@ bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) return false; } + Stream* stream = streams[idx]; -#endif + string name = filter->AsString()->CheckString(); + + for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + { + if ( (*i)->name == name ) + { + Filter* filter = *i; + stream->filters.erase(i); + DBG_LOG(DBG_LOGGING, "Removed filter '%s' from stream '%s'", filter->name.c_str(), stream->name.c_str()); + delete filter; + return true; + } + } + + // If we don't find the filter, we don't treat that as an error. + DBG_LOG(DBG_LOGGING, "Did not find filter '%s' for removing from stream '%s'", name.c_str(), stream->name.c_str()); return true; } From 434f57f85ffec06e32d9e927c5845cde53acc787 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 17:12:58 -0800 Subject: [PATCH 24/67] Implementing dynamic paths via $path_func. One change to original plan: the function also receives the standard $path argument. --- policy/bro.init | 2 +- src/LogMgr.cc | 30 ++++++++++++++++++++++++++---- 2 files changed, 27 insertions(+), 5 deletions(-) diff --git a/policy/bro.init b/policy/bro.init index 0b727fcdc5..16904d91b7 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -308,7 +308,7 @@ type log_filter: record { # extension is given; the writer will add whatever is # appropiate. path: string &optional; - path_func: function(id: Log_ID): string &optional; + path_func: function(id: Log_ID, path: string): string &optional; # A subset of column names to record. If not given, all # columns are recorded. diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ab36a26758..f6e80cee5a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -25,6 +25,7 @@ struct LogMgr::Filter { Func* pred; Func* path_func; string path; + Val* path_val; LogWriterDefinition* writer; int num_fields; @@ -54,6 +55,8 @@ LogMgr::Filter::~Filter() for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) delete i->second; + + Unref(path_val); } LogMgr::Stream::~Stream() @@ -262,8 +265,11 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) // Get the path for the filter. Val* path_val = fval->Lookup(rtype->FieldOffset("path")); - if ( path_val ) + if ( path_val ) + { filter->path = path_val->AsString()->CheckString(); + filter->path_val = path_val->Ref(); + } else { @@ -286,6 +292,7 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) } filter->path = string(lower); + filter->path_val = new StringVal(lower); free(lower); } @@ -371,7 +378,6 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) { Filter* filter = *i; - string path = filter->path; if ( filter->pred ) @@ -386,10 +392,26 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) if ( ! result ) continue; } - + if ( filter->path_func ) { - // XXX Do dynamic path here. + val_list vl(2); + vl.append(stream_id->Ref()); + vl.append(filter->path_val->Ref()); + Val* v = filter->path_func->Call(&vl); + + if ( ! v->Type()->Tag() == TYPE_STRING ) + { + run_time("path_func did not return string"); + Unref(v); + return false; + } + + path = v->AsString()->CheckString(); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Path function for filter '%s' on stream '%s' return '%s'", filter->name.c_str(), stream->name.c_str(), path.c_str()); +#endif } // See if we already have a writer for this path. From cf148c8a25ea6dfa00120726094976c3f3d98b8a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 17:33:29 -0800 Subject: [PATCH 25/67] New bif log_set_buf() to set the buffering state for a stream. --- TODO.logging | 7 ++--- src/LogMgr.cc | 59 ++++++++++++++++++++++++------------------- src/LogMgr.h | 2 ++ src/LogWriter.cc | 11 ++++++++ src/LogWriter.h | 21 ++++++++++----- src/LogWriterAscii.cc | 7 ++++- src/LogWriterAscii.h | 2 +- src/bro.bif | 6 +++++ 8 files changed, 78 insertions(+), 37 deletions(-) diff --git a/TODO.logging b/TODO.logging index 88a4e74fba..f8208b79f8 100644 --- a/TODO.logging +++ b/TODO.logging @@ -1,18 +1,19 @@ List of the things not implemented yet: - - Dynamic path function. - Cluster-style remote_print - Rotation support - - Flushing support - Spawning writers in separate threads (not clear if we want that initially). - Not sure if the logging does the right thing with &optional and &default values. Needs testing. - Seems we could do some of the filter-related type checks currently done dynamically at startup via a TraversalCallback. + - LogMgs' error handling + - Put script function/constants into namespace. + - Cleanup code. There's probably more missing. -Question: +Questions: * Is giving the record column twice to create_stream too redundant? (once directly, and once via the event):: diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f6e80cee5a..fc4b2bd02d 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -76,6 +76,19 @@ LogMgr::~LogMgr() delete *s; } +LogMgr::Stream* LogMgr::FindStream(EnumVal* stream_id) + { + unsigned int idx = stream_id->AsEnum(); + + if ( idx >= streams.size() || ! streams[idx] ) + { + run_time("unknown log stream"); + return 0; + } + + return streams[idx]; + } + bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerPtr handler) { // TODO: Should check that the record has only supported types. @@ -182,15 +195,9 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) return false; } - unsigned int i = stream_id->AsEnum(); - - if ( i >= streams.size() || ! streams[i] ) - { - run_time("unknown log stream"); + Stream* stream = FindStream(stream_id); + if ( ! stream ) return false; - } - - Stream* stream = streams[i]; // Find the right writer type. int writer = 0; @@ -317,15 +324,9 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) { - unsigned int idx = stream_id->AsEnum(); - - if ( idx >= streams.size() || ! streams[idx] ) - { - run_time("unknown log stream"); + Stream* stream = FindStream(stream_id); + if ( ! stream ) return false; - } - - Stream* stream = streams[idx]; string name = filter->AsString()->CheckString(); @@ -348,15 +349,9 @@ bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) { - unsigned int idx = stream_id->AsEnum(); - - if ( idx >= streams.size() || ! streams[idx] ) - { - run_time("unknown log stream"); + Stream* stream = FindStream(stream_id); + if ( ! stream ) return false; - } - - Stream* stream = streams[idx]; columns = columns->CoerceTo(stream->columns); @@ -538,9 +533,21 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) return vals; } +bool LogMgr::SetBuf(EnumVal* stream_id, bool enabled) + { + Stream* stream = FindStream(stream_id); + if ( ! stream ) + return false; + + for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + { + for ( Filter::WriterMap::iterator j = (*i)->writers.begin(); j != (*i)->writers.end(); j++ ) + j->second->SetBuf(enabled); + } + + return true; + } void LogMgr::Error(LogWriter* writer, const char* msg) { -#if 0 -#endif } diff --git a/src/LogMgr.h b/src/LogMgr.h index 28a06821db..3357f871b5 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -59,6 +59,7 @@ public: bool AddFilter(EnumVal* stream_id, RecordVal* filter); bool RemoveFilter(EnumVal* stream_id, StringVal* filter); bool Write(EnumVal* stream_id, RecordVal* columns); + bool SetBuf(EnumVal* stream_id, bool enabled); // Changes the state for all writers for that stream. protected: friend class LogWriter; @@ -74,6 +75,7 @@ private: bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); + Stream* FindStream(EnumVal* stream_id); vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriter.cc b/src/LogWriter.cc index 903ac46f8b..fd9bcf0b44 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -6,6 +6,7 @@ LogWriter::LogWriter() { buf = 0; buf_len = 1024; + buffering = true; } LogWriter::~LogWriter() @@ -37,6 +38,16 @@ void LogWriter::Finish() DoFinish(); } +bool LogWriter::SetBuf(bool enabled) + { + if ( enabled == buffering ) + // No change. + return true; + + buffering = enabled; + return DoSetBuf(enabled); + } + const char* LogWriter::Fmt(const char* format, ...) { if ( ! buf ) diff --git a/src/LogWriter.h b/src/LogWriter.h index 2b8d321cd2..e13c06edff 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -36,6 +36,9 @@ public: // performed. void Finish(); + // Sets the buffering status for the writer, if the writer supports + bool SetBuf(bool enabled); + protected: //// Methods for Writers to override. @@ -50,15 +53,17 @@ protected: // should be reported via Error(). virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; - // Called when the flushing status for this writer is changed. If - // flushing is enabled, the writer should attempt to write out + // Called when the buffering status for this writer is changed. If + // buffering is disabled, the writer should attempt to write out // information as quickly as possible even if that may have an - // performance impact. If disabled (which the writer should assume to be + // performance impact. If enabled (which the writer should assume to be // the default), then it can buffer things up as necessary and write out - // in a way optimized for performance. + // in a way optimized for performance. The current buffering state can + // alse be queried via IsBuf(). // - // A writer may ignore flushing if it doesn't fit with its semantics. - virtual void DoSetFlushing(bool enabled) = 0; + // A writer may ignore buffering if it doesn't fit with its semantics. + // Still return true in that case. + virtual bool DoSetBuf(bool enabled) = 0; // Called when a log output is to be rotated. Most directly, this only // applies to writers outputting files, thoug a writer may also trigger @@ -91,6 +96,9 @@ protected: // A thread-safe version of fmt(). const char* Fmt(const char* format, ...); + // Returns the current buffering state. + bool IsBuf() { return buffering; } + // Reports an error. void Error(const char *msg); @@ -104,6 +112,7 @@ private: string path; int num_fields; LogField** fields; + bool buffering; // For Fmt(). char* buf; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 69b9993778..7218a96dde 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -125,6 +125,9 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) return false; } + if ( IsBuf() ) + fflush(file); + return true; } @@ -133,8 +136,10 @@ bool LogWriterAscii::DoRotate(string rotated_path) return true; } -void LogWriterAscii::DoSetFlushing(bool enabled) +bool LogWriterAscii::DoSetBuf(bool enabled) { + // Nothing to do. + return true; } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index fa014fbc81..7b6b67b758 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -17,7 +17,7 @@ public: protected: virtual bool DoInit(string path, int num_fields, LogField** fields); virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals); - virtual void DoSetFlushing(bool enabled); + virtual bool DoSetBuf(bool enabled); virtual bool DoRotate(string rotated_path); virtual void DoFinish(); diff --git a/src/bro.bif b/src/bro.bif index e52e30ab1d..e6888af3db 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -548,6 +548,12 @@ function log_write%(id: Log_ID, columns: any%) : bool return new Val(result, TYPE_BOOL); %} +function log_set_buf%(id: Log_ID, buffered: bool%): bool + %{ + bool result = log_mgr->SetBuf(id->AsEnumVal(), buffered); + return new Val(result, TYPE_BOOL); + %} + function record_type_to_vector%(rt: string%): string_vec %{ VectorVal* result = From ac936feb9566b0886a0f9a40884003edbe120f96 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 17:56:09 -0800 Subject: [PATCH 26/67] Consistent error handling. --- TODO.logging | 14 +++++--------- src/LogMgr.cc | 38 +++++++++++++++++++++++++++++++++++--- src/LogMgr.h | 1 + src/LogWriter.cc | 34 +++++++++++++++++++++++++--------- src/LogWriter.h | 45 ++++++++++++++++++++++++++------------------- 5 files changed, 92 insertions(+), 40 deletions(-) diff --git a/TODO.logging b/TODO.logging index f8208b79f8..a73f027750 100644 --- a/TODO.logging +++ b/TODO.logging @@ -1,21 +1,17 @@ List of the things not implemented yet: - - Cluster-style remote_print - - Rotation support - - Spawning writers in separate threads (not clear if we want that initially). + - Cluster-style remote_print. + - Rotation support. - Not sure if the logging does the right thing with &optional and &default values. Needs testing. - - Seems we could do some of the filter-related type checks - currently done dynamically at startup via a TraversalCallback. - - LogMgs' error handling - Put script function/constants into namespace. + - Some typchecks are still missing. - Cleanup code. - -There's probably more missing. + - Spawning writers in separate threads (not clear if we want that initially). Questions: - * Is giving the record column twice to create_stream too + * Is giving the record column twice to create_stream() too redundant? (once directly, and once via the event):: global ssh_log: event(rec: Log); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index fc4b2bd02d..f6de4db2c1 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -89,6 +89,28 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* stream_id) return streams[idx]; } +void LogMgr::RemoveDisabledWriters(Stream* stream) + { + for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + { + Filter* filter = (*i); + + list disabled; + + for ( Filter::WriterMap::iterator j = filter->writers.begin(); j != filter->writers.end(); j++ ) + { + if ( j->second->Disabled() ) + { + delete j->second; + disabled.push_back(j->first); + } + } + + for ( list::iterator j = disabled.begin(); j != disabled.end(); j++ ) + filter->writers.erase(*j); + } + } + bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerPtr handler) { // TODO: Should check that the record has only supported types. @@ -272,7 +294,7 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) // Get the path for the filter. Val* path_val = fval->Lookup(rtype->FieldOffset("path")); - if ( path_val ) + if ( path_val ) { filter->path = path_val->AsString()->CheckString(); filter->path_val = path_val->Ref(); @@ -349,6 +371,8 @@ bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) { + bool error = false; + Stream* stream = FindStream(stream_id); if ( ! stream ) return false; @@ -387,7 +411,7 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) if ( ! result ) continue; } - + if ( filter->path_func ) { val_list vl(2); @@ -440,7 +464,8 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) // Alright, can do the write now. LogVal** vals = RecordToFilterVals(filter, columns); - writer->Write(vals); + if ( ! writer->Write(vals) ) + error = true; #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", filter->name.c_str(), stream->name.c_str()); @@ -448,6 +473,10 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) } Unref(columns); + + if ( error ) + RemoveDisabledWriters(stream); + return true; } @@ -545,9 +574,12 @@ bool LogMgr::SetBuf(EnumVal* stream_id, bool enabled) j->second->SetBuf(enabled); } + RemoveDisabledWriters(stream); + return true; } void LogMgr::Error(LogWriter* writer, const char* msg) { + run_time(fmt("error with writer for %s: %s", writer->Path().c_str(), msg)); } diff --git a/src/LogMgr.h b/src/LogMgr.h index 3357f871b5..bd269ccfde 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -76,6 +76,7 @@ private: bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); Stream* FindStream(EnumVal* stream_id); + void RemoveDisabledWriters(Stream* stream); vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriter.cc b/src/LogWriter.cc index fd9bcf0b44..4292849a9a 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -7,6 +7,7 @@ LogWriter::LogWriter() buf = 0; buf_len = 1024; buffering = true; + disabled = false; } LogWriter::~LogWriter() @@ -22,7 +23,13 @@ bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields) path = arg_path; num_fields = arg_num_fields; fields = arg_fields; - DoInit(arg_path, arg_num_fields, arg_fields); + + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + { + disabled = true; + return false; + } + return true; } @@ -30,12 +37,11 @@ bool LogWriter::Write(LogVal** vals) { bool result = DoWrite(num_fields, fields, vals); DeleteVals(vals); - return result; - } -void LogWriter::Finish() - { - DoFinish(); + if ( ! result ) + disabled = true; + + return result; } bool LogWriter::SetBuf(bool enabled) @@ -45,7 +51,18 @@ bool LogWriter::SetBuf(bool enabled) return true; buffering = enabled; - return DoSetBuf(enabled); + if ( ! DoSetBuf(enabled) ) + { + disabled = true; + return false; + } + + return true; + } + +void LogWriter::Finish() + { + DoFinish(); } const char* LogWriter::Fmt(const char* format, ...) @@ -72,10 +89,9 @@ const char* LogWriter::Fmt(const char* format, ...) return buf; } - void LogWriter::Error(const char *msg) { - run_time(msg); + log_mgr->Error(this, msg); } void LogWriter::DeleteVals(LogVal** vals) diff --git a/src/LogWriter.h b/src/LogWriter.h index e13c06edff..780b8b2a4c 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -31,38 +31,38 @@ public: // occured, in which case the writer must not be used further. bool Write(LogVal** vals); + // Sets the buffering status for the writer, if the writer supports + bool SetBuf(bool enabled); + // Finished writing to this logger. Will not be called if an error has // been indicated earlier. After calling this, no more writing must be // performed. void Finish(); - // Sets the buffering status for the writer, if the writer supports - bool SetBuf(bool enabled); - protected: - //// Methods for Writers to override. + // Methods for Writers to override. If any of these returs false, it will + // be assumed that a fatal error has occured that prevents the writer + // from further operation. It will then be disabled and deleted. In that + // case, the writer should also report the error via Error(). If a writer + // does not specifically implement one of the methods, it must still + // always return true. - // Called once for initialization of the Writer. Must return false if an - // error occured, in which case the writer will be disabled. The error - // reason should be reported via Error(). + // Called once for initialization of the Writer. virtual bool DoInit(string path, int num_fields, LogField** fields) = 0; - // Called once per entry to record. Must return false if an error - // occured, in which case the writer will be disabled. The error reason - // should be reported via Error(). + // Called once per entry to record. virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; // Called when the buffering status for this writer is changed. If // buffering is disabled, the writer should attempt to write out - // information as quickly as possible even if that may have an - // performance impact. If enabled (which the writer should assume to be - // the default), then it can buffer things up as necessary and write out - // in a way optimized for performance. The current buffering state can - // alse be queried via IsBuf(). + // information as quickly as possible even if doing so may have an + // performance impact. If enabled (which is the default), it can buffer + // things up as necessary and write out in a way optimized for + // performance. The current buffering state can be queried via IsBuf(). // - // A writer may ignore buffering if it doesn't fit with its semantics. - // Still return true in that case. + // A writer may ignore buffering changes if it doesn't fit with its + // semantics. virtual bool DoSetBuf(bool enabled) = 0; // Called when a log output is to be rotated. Most directly, this only @@ -83,7 +83,7 @@ protected: // still call Error(), but return true. // // A writer may ignore rotation requests if it doesn't fit with its - // semantics. In that case, still return true. + // semantics. virtual bool DoRotate(string rotated_path) = 0; // Called once on termination. Not called when any of the other methods @@ -106,13 +106,20 @@ protected: const string Path() const { return path; } private: - // Delete values as passed into Write(). + friend class LogMgr; + + // When an error occurs, we set this flag. The LogMgr will check it an + // remove any disabled writers. + bool Disabled() { return disabled; } + + // Deletes the values passed into Write(). void DeleteVals(LogVal** vals); string path; int num_fields; LogField** fields; bool buffering; + bool disabled; // For Fmt(). char* buf; From f02e465c92d9b52ebb8bfb75ade68dcc61803329 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 21 Feb 2011 18:29:41 -0800 Subject: [PATCH 27/67] Tests for the logging functionality, including a basic btest configuration. --- testing/btest/.gitignore | 1 + testing/btest/Baseline/logging.events/output | 2 + .../btest/Baseline/logging.path-func/output | 13 ++++ .../Baseline/logging.pred/ssh.failure.log | 2 + .../Baseline/logging.pred/ssh.success.log | 2 + .../Baseline/logging.remove/ssh.failure.log | 3 + .../Baseline/logging.remove/ssh_log_ssh.log | 4 ++ .../logging.test-logging/ssh_log_ssh.log | 6 ++ testing/btest/Scripts/diff-canonifier | 5 ++ testing/btest/btest.cfg | 12 ++++ testing/btest/logging/events.bro | 40 ++++++++++++ testing/btest/logging/path-func.bro | 51 +++++++++++++++ testing/btest/logging/pred.bro | 43 ++++++++++++ testing/btest/logging/remove.bro | 46 +++++++++++++ testing/btest/logging/test-logging.bro | 65 +++++++++++++++++++ 15 files changed, 295 insertions(+) create mode 100644 testing/btest/.gitignore create mode 100644 testing/btest/Baseline/logging.events/output create mode 100644 testing/btest/Baseline/logging.path-func/output create mode 100644 testing/btest/Baseline/logging.pred/ssh.failure.log create mode 100644 testing/btest/Baseline/logging.pred/ssh.success.log create mode 100644 testing/btest/Baseline/logging.remove/ssh.failure.log create mode 100644 testing/btest/Baseline/logging.remove/ssh_log_ssh.log create mode 100644 testing/btest/Baseline/logging.test-logging/ssh_log_ssh.log create mode 100755 testing/btest/Scripts/diff-canonifier create mode 100644 testing/btest/btest.cfg create mode 100644 testing/btest/logging/events.bro create mode 100644 testing/btest/logging/path-func.bro create mode 100644 testing/btest/logging/pred.bro create mode 100644 testing/btest/logging/remove.bro create mode 100644 testing/btest/logging/test-logging.bro diff --git a/testing/btest/.gitignore b/testing/btest/.gitignore new file mode 100644 index 0000000000..d36977dc47 --- /dev/null +++ b/testing/btest/.gitignore @@ -0,0 +1 @@ +.tmp diff --git a/testing/btest/Baseline/logging.events/output b/testing/btest/Baseline/logging.events/output new file mode 100644 index 0000000000..21571f5eee --- /dev/null +++ b/testing/btest/Baseline/logging.events/output @@ -0,0 +1,2 @@ +[t=1298326194.80574, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=] +[t=1298326194.80574, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] diff --git a/testing/btest/Baseline/logging.path-func/output b/testing/btest/Baseline/logging.path-func/output new file mode 100644 index 0000000000..ec12bfbdab --- /dev/null +++ b/testing/btest/Baseline/logging.path-func/output @@ -0,0 +1,13 @@ +static-prefix-0.log +static-prefix-1.log +static-prefix-2.log +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 success unknown +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 success BR +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure MX3 +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure US +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure UK +1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure MX2 diff --git a/testing/btest/Baseline/logging.pred/ssh.failure.log b/testing/btest/Baseline/logging.pred/ssh.failure.log new file mode 100644 index 0000000000..f90e2ad600 --- /dev/null +++ b/testing/btest/Baseline/logging.pred/ssh.failure.log @@ -0,0 +1,2 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298327302.5778 1.2.3.4 66770 2.3.4.5 65616 failure US diff --git a/testing/btest/Baseline/logging.pred/ssh.success.log b/testing/btest/Baseline/logging.pred/ssh.success.log new file mode 100644 index 0000000000..e137961d3d --- /dev/null +++ b/testing/btest/Baseline/logging.pred/ssh.success.log @@ -0,0 +1,2 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298327302.5778 1.2.3.4 66770 2.3.4.5 65616 success - diff --git a/testing/btest/Baseline/logging.remove/ssh.failure.log b/testing/btest/Baseline/logging.remove/ssh.failure.log new file mode 100644 index 0000000000..75385cbbd7 --- /dev/null +++ b/testing/btest/Baseline/logging.remove/ssh.failure.log @@ -0,0 +1,3 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure US +1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure UK diff --git a/testing/btest/Baseline/logging.remove/ssh_log_ssh.log b/testing/btest/Baseline/logging.remove/ssh_log_ssh.log new file mode 100644 index 0000000000..a9fe2b9ffe --- /dev/null +++ b/testing/btest/Baseline/logging.remove/ssh_log_ssh.log @@ -0,0 +1,4 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure US +1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure UK +1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure BR diff --git a/testing/btest/Baseline/logging.test-logging/ssh_log_ssh.log b/testing/btest/Baseline/logging.test-logging/ssh_log_ssh.log new file mode 100644 index 0000000000..b012cffbda --- /dev/null +++ b/testing/btest/Baseline/logging.test-logging/ssh_log_ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 success unknown +1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 failure US +1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 failure UK +1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 success BR +1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 failure MX diff --git a/testing/btest/Scripts/diff-canonifier b/testing/btest/Scripts/diff-canonifier new file mode 100755 index 0000000000..2361355491 --- /dev/null +++ b/testing/btest/Scripts/diff-canonifier @@ -0,0 +1,5 @@ +#! /usr/bin/env bash +# +# Replace anything which looks like timestamps with XXXs. + +sed 's/[0-9]\{10\}\.[0-9]\{4,6\}/XXXXXXXXXX.XXXXXX/g' diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg new file mode 100644 index 0000000000..1e702ce344 --- /dev/null +++ b/testing/btest/btest.cfg @@ -0,0 +1,12 @@ + +[btest] +TestDirs = logging +TmpDir = %(testbase)s/.tmp +BaselineDir = %(testbase)s/Baseline +IgnoreDirs = .svn CVS .tmp +IgnoreFiles = *.tmp *.swp #* + +[environment] +BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` +PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s +TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-canonifier diff --git a/testing/btest/logging/events.bro b/testing/btest/logging/events.bro new file mode 100644 index 0000000000..45135c60f6 --- /dev/null +++ b/testing/btest/logging/events.bro @@ -0,0 +1,40 @@ + +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log_ID += { LOG_SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global ssh_log: event(rec: Log); + +event bro_init() +{ + log_create_stream(LOG_SSH, SSH::Log, ssh_log); + Log_add_default_filter(LOG_SSH); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + log_write(LOG_SSH, r); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + +} + +event ssh_log(rec: Log) + { + print rec; + } diff --git a/testing/btest/logging/path-func.bro b/testing/btest/logging/path-func.bro new file mode 100644 index 0000000000..f76acd6b33 --- /dev/null +++ b/testing/btest/logging/path-func.bro @@ -0,0 +1,51 @@ + +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: ( ls static-*; cat static-* ) >output +# @TEST-EXEC: btest-diff output + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log_ID += { LOG_SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global ssh_log: event(rec: Log); + +global c = -1; + +function path_func(id: Log_ID, path: string) : string + { + c = (c + 1) % 3; + + return fmt("%s-%d", path, c); + } + +event bro_init() +{ + log_create_stream(LOG_SSH, SSH::Log, ssh_log); + + log_add_filter(LOG_SSH, [$name="dyn", $path="static-prefix", $path_func=path_func]); + + log_set_buf(LOG_SSH, F); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX2"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX3"]); +} diff --git a/testing/btest/logging/pred.bro b/testing/btest/logging/pred.bro new file mode 100644 index 0000000000..5e2b259d7a --- /dev/null +++ b/testing/btest/logging/pred.bro @@ -0,0 +1,43 @@ + +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.success.log +# @TEST-EXEC: btest-diff ssh.failure.log + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log_ID += { LOG_SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global ssh_log: event(rec: Log); + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event bro_init() +{ + log_create_stream(LOG_SSH, SSH::Log, ssh_log); + + log_add_filter(LOG_SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); + log_add_filter(LOG_SSH, [$name="f2", $path="ssh.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + log_write(LOG_SSH, r); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + +} diff --git a/testing/btest/logging/remove.bro b/testing/btest/logging/remove.bro new file mode 100644 index 0000000000..69d70337e5 --- /dev/null +++ b/testing/btest/logging/remove.bro @@ -0,0 +1,46 @@ +# +# @TEST-EXEC: bro -B logging %INPUT +# @TEST-EXEC: btest-diff ssh_log_ssh.log +# @TEST-EXEC: btest-diff ssh.failure.log + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log_ID += { LOG_SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global ssh_log: event(rec: Log); + +event bro_init() +{ + log_create_stream(LOG_SSH, SSH::Log,ssh_log); + Log_add_default_filter(LOG_SSH); + log_add_filter(LOG_SSH, [$name="f1", $path="ssh.failure", $pred=function(rec: Log): bool { return rec$status == "failure"; }]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + # Log something. + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + + log_remove_filter(LOG_SSH, "f1"); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="BR"]); + + log_remove_filter(LOG_SSH, "default"); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + + log_remove_filter(LOG_SSH, "doesn-not-exist"); +} + diff --git a/testing/btest/logging/test-logging.bro b/testing/btest/logging/test-logging.bro new file mode 100644 index 0000000000..464461db72 --- /dev/null +++ b/testing/btest/logging/test-logging.bro @@ -0,0 +1,65 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh_log_ssh.log + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log_ID += { LOG_SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global ssh_log: event(rec: Log); + +event bro_init() +{ + # Create the stream. + # First argument is the ID for the stream. + # Second argument is the log record type. + log_create_stream(LOG_SSH, SSH::Log,ssh_log); + + # Add a default filter that simply logs everything to "ssh.log" using the default writer. + Log_add_default_filter(LOG_SSH); + + # Printing headers for the filters doesn't work yet either and needs to + # be considered in the final design. (based on the "select" set). + #Log::add_filter("ssh", [$name="successful logins", + # #$pred(rec: Log) = { print rec$status; return T; }, + # $path="ssh-logins", + # #$select=set("t"), + # $writer=Log::WRITER_CSV]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + # Log something. + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + +#event log(rec: Log) +# { +# print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); +# } +# +# +#module WHATEVER; +# +#event SSH::log(rec: SSH::Log) +# { +# print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); +# } From ab15437339a146bb9b895f4f8de71c1c54a73532 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 27 Feb 2011 15:05:47 -0800 Subject: [PATCH 28/67] Working on the logging API exposed to scripts. - Moving all functions into the Log::* namespace, using the recent bifcl updates. Moved logging-specific stuff to logging.bif. - Log::create_stream() now takes a record Log::Stream as its second argument, which specifies columns and (optionally) the event. - All the internal BiFs are now called "Log::__", with script-level wrappers "Log::". That first allows to add additional code at the script-level, and second makes things better comprehendible as now all relevant functionality is collected (and later documetned) in policy/logging.bro. - New function Log::flush(id), which does the obvious assuming the writer supports it. - add_default_filter() is now called implicitly with every create_stream(). Seems that we usually want that functionality, and when not, remove_default_filter() gets rid of it. - The namespace of a stream's ID is now used as the default "path" (e.g., if the namespace is SSH, the default log file is "ssh.log"). - Updated policy/test-logging.bro as well as the btest tests according to these changes. --- TODO.logging | 11 +- policy/bro.init | 51 +---- policy/logging.bro | 110 +++++++++- policy/test-logging.bro | 47 ++--- src/CMakeLists.txt | 1 + src/Func.cc | 3 + src/LogMgr.cc | 134 +++++++----- src/LogMgr.h | 13 +- src/LogWriter.cc | 11 + src/LogWriter.h | 11 +- src/LogWriterAscii.cc | 6 + src/LogWriterAscii.h | 1 + src/NetVar.cc | 6 +- src/NetVar.h | 3 +- src/Type.cc | 1 - src/Val.h | 2 +- src/bro.bif | 194 ------------------ src/logging.bif | 43 +++- src/types.bif | 11 + .../{ssh_log_ssh.log => ssh.log} | 0 .../{ssh_log_ssh.log => ssh.log} | 0 testing/btest/logging/events.bro | 9 +- testing/btest/logging/path-func.bro | 27 ++- testing/btest/logging/pred.bro | 16 +- testing/btest/logging/remove.bro | 27 +-- testing/btest/logging/test-logging.bro | 50 +---- 26 files changed, 337 insertions(+), 451 deletions(-) rename testing/btest/Baseline/logging.remove/{ssh_log_ssh.log => ssh.log} (100%) rename testing/btest/Baseline/logging.test-logging/{ssh_log_ssh.log => ssh.log} (100%) diff --git a/TODO.logging b/TODO.logging index a73f027750..d5bb3744b7 100644 --- a/TODO.logging +++ b/TODO.logging @@ -4,15 +4,6 @@ List of the things not implemented yet: - Rotation support. - Not sure if the logging does the right thing with &optional and &default values. Needs testing. - - Put script function/constants into namespace. - - Some typchecks are still missing. - - Cleanup code. - Spawning writers in separate threads (not clear if we want that initially). + - Check the new event-value code. -Questions: - - * Is giving the record column twice to create_stream() too - redundant? (once directly, and once via the event):: - - global ssh_log: event(rec: Log); - log_create_stream(LOG_SSH, SSH::Log, ssh_log); diff --git a/policy/bro.init b/policy/bro.init index 02aad1a0cb..3a107d865c 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -273,53 +273,7 @@ type entropy_test_result: record { serial_correlation: double; }; -type Log_Writer: enum { # TODO: Move these into bif and use from C++ as well. - WRITER_DEFAULT, # See default_writer below. - WRITER_ASCII, -}; - -# Each stream gets a unique ID. This type will be extended by -# other scripts. -type Log_ID: enum { - Unknown -}; - -# The default writer to use if a filter does not specify -# anything else. -const Log_default_writer = WRITER_ASCII &redef; - -# A filter defining what to log. -type log_filter: record { - # A name to reference this filter. - name: string; - - # A predicate returning True if the filter wants a log entry - # to be recorded. If not given, an implicit True is assumed - # for all entries. The predicate receives one parameter: - # an instance of the log's record type with the fields to be - # logged. - pred: function(rec: any): bool &optional; - - # A path for outputting everything matching this - # filter. The path is either a string, or a function - # called with a single ``ID`` argument and returning a string. - # - # The specific interpretation of the string is left to the - # Writer, but if it's refering to a file, it's assumed that no - # extension is given; the writer will add whatever is - # appropiate. - path: string &optional; - path_func: function(id: Log_ID, path: string): string &optional; - - # A subset of column names to record. If not given, all - # columns are recorded. - include: set[string] &optional; - exclude: set[string] &optional; - - # The writer to use. - writer: Log_Writer &default=Log_default_writer; -}; - +@load logging # sic! Not logging.bif. # Prototypes of Bro built-in functions. @load strings.bif.bro @@ -1479,7 +1433,6 @@ const dump_selected_source_packets = F &redef; # the connection is not rewritten; if false, the policy script can decide # whether to dump a particular connection by calling dump_packets_of_connection. # -# NOTE: DO NOT SET THIS TO TRUE WHEN ANONYMIZING A TRACE! +# NOTE: DO NOT SET THIS TO TRUE WHEN ANONYMIZING A TRACE! # (TODO: this variable should be disabled when using '-A' option) const dump_original_packets_if_not_rewriting = F &redef; - diff --git a/policy/logging.bro b/policy/logging.bro index f00dcf7fb2..36f866c91f 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -1,10 +1,110 @@ +module Log; -function Log_add_default_filter(id: Log_ID) +# Log::ID and Log::Writer are defined in bro.init due to circular dependencies. + +export { + # A stream defining the logging. + type Stream: record { + # A record type defining the log's columns. + columns: any; + + # A optional event raised for each log entry. It must receive + # a single argument of type $columns. + ev: any &optional; + }; + + # A filter customizes logging. + type Filter: record { + # A name to reference this filter. + name: string; + + # A predicate returning True if the filter wants a log entry + # to be recorded. If not given, an implicit True is assumed + # for all entries. The predicate receives one parameter: + # an instance of the log's record type with the fields to be + # logged. + pred: function(rec: any): bool &optional; + + # A path for outputting everything matching this + # filter. The path is either a string, or a function + # called with a single ``ID`` argument and returning a string. + # + # The specific interpretation of the string is left to the + # Writer, but if it's refering to a file, it's assumed that no + # extension is given; the writer will add whatever is + # appropiate. + path: string &optional; + path_func: function(id: ID, path: string): string &optional; + + # A subset of column names to record. If not given, all + # columns are recorded. + include: set[string] &optional; + exclude: set[string] &optional; + + # The writer to use. + writer: Writer &optional; + }; + + global create_stream: function(id: Log::ID, stream: Log::Stream) : bool; + global add_filter: function(id: Log::ID, filter: Log::Filter) : bool; + global remove_filter: function(id: Log::ID, name: string) : bool; + global write: function(id: Log::ID, columns: any) : bool; + global set_buf: function(id: Log::ID, buffered: bool): bool; + global flush: function(id: Log::ID): bool; + global add_default_filter: function(id: ID) : bool; + global remove_default_filter: function(id: ID) : bool; +} + +@load logging.bif # Needs Log::Filter and Log::Stream defined. + +module Log; + +export { + # The default writer to use if a filter does not specify + # anything else. + const default_writer = Log::WRITER_ASCII &redef; +} + +function create_stream(id: Log::ID, stream: Log::Stream) : bool { - log_add_filter(id, [$name="default"]); + if ( ! Log::__create_stream(id, stream) ) + return F; + + return add_default_filter(id); } - -function Log_remove_default_filter(id: Log_ID): bool + +function add_filter(id: Log::ID, filter: Log::Filter) : bool { - log_remove_filter(id, "default"); + return Log::__add_filter(id, filter); } + +function remove_filter(id: Log::ID, name: string) : bool + { + return Log::__remove_filter(id, name); + } + +function write(id: Log::ID, columns: any) : bool + { + return Log::__write(id, columns); + } + +function set_buf(id: Log::ID, buffered: bool): bool + { + return Log::__set_buf(id, buffered); + } + +function flush(id: Log::ID): bool + { + return Log::__flush(id); + } + +function add_default_filter(id: ID) : bool + { + return add_filter(id, [$name="default"]); + } + +function remove_default_filter(id: ID) : bool + { + return remove_filter(id, "default"); + } + diff --git a/policy/test-logging.bro b/policy/test-logging.bro index dea93be80e..92c168a467 100644 --- a/policy/test-logging.bro +++ b/policy/test-logging.bro @@ -1,10 +1,8 @@ module SSH; -@load logging - export { # Create a new ID for our log stream - redef enum Log_ID += { LOG_SSH }; + redef enum Log::ID += { SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -16,7 +14,7 @@ export { }; } -global ssh_log: event(rec: Log); +global log_ssh: event(rec: Log); function fail(rec: Log): bool { @@ -27,47 +25,26 @@ event bro_init() { # Create the stream. # First argument is the ID for the stream. - # Second argument is the log record type. - # Third argument is the log event, which must receive a single argument of type arg2. - log_create_stream(LOG_SSH, SSH::Log, ssh_log); + # Second argument is a record of type Log::Stream. + Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); - # Add a default filter that simply logs everything to "ssh.log" using the default writer. - Log_add_default_filter(LOG_SSH); - - log_add_filter(LOG_SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); - log_add_filter(LOG_SSH, [$name="f2", $path="ssh.failure", $pred=fail]); - - # Printing headers for the filters doesn't work yet either and needs to - # be considered in the final design. (based on the "select" set). - #Log::add_filter("ssh", [$name="successful logins", - # #$pred(rec: Log) = { print rec$status; return T; }, - # $path="ssh-logins", - # #$select=set("t"), - # $writer=Log::WRITER_CSV]); + Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); + Log::add_filter(SSH, [$name="f2", $path="ssh.failure", $pred=fail]); local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; local r: Log = [$t=network_time(), $id=cid, $status="success"]; # Log something. - log_write(LOG_SSH, r); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); - + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); } -event ssh_log(rec: Log) +event log_ssh(rec: Log) { print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); print rec; } -# -# -#module WHATEVER; -# -#event SSH::log(rec: SSH::Log) -# { -# print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); -# } diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b9ec957ae6..dfc4b449fc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -127,6 +127,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif + logging.bif event.bif const.bif types.bif diff --git a/src/Func.cc b/src/Func.cc index 310b61efd2..00ef713501 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -508,6 +508,7 @@ void builtin_run_time(const char* msg, BroObj* arg) } #include "bro.bif.func_h" +#include "logging.bif.func_h" #include "common-rw.bif.func_h" #include "finger-rw.bif.func_h" @@ -519,6 +520,7 @@ void builtin_run_time(const char* msg, BroObj* arg) #include "dns-rw.bif.func_h" #include "bro.bif.func_def" +#include "logging.bif.func_def" #include "strings.bif.func_def" void init_builtin_funcs() @@ -530,6 +532,7 @@ void init_builtin_funcs() gap_info = internal_type("gap_info")->AsRecordType(); #include "bro.bif.func_init" +#include "logging.bif.func_init" #include "common-rw.bif.func_init" #include "finger-rw.bif.func_init" diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f6de4db2c1..1d272e7049 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -76,9 +76,9 @@ LogMgr::~LogMgr() delete *s; } -LogMgr::Stream* LogMgr::FindStream(EnumVal* stream_id) +LogMgr::Stream* LogMgr::FindStream(EnumVal* id) { - unsigned int idx = stream_id->AsEnum(); + unsigned int idx = id->AsEnum(); if ( idx >= streams.size() || ! streams[idx] ) { @@ -111,14 +111,52 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) } } -bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerPtr handler) +bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) { - // TODO: Should check that the record has only supported types. + RecordType* rtype = sval->Type()->AsRecordType(); - unsigned int idx = stream_id->AsEnum(); + if ( ! same_type(rtype, BifType::Record::Log::Stream, 0) ) + { + run_time("sval argument not of right type"); + return false; + } + + RecordType* columns = sval->Lookup(rtype->FieldOffset("columns"))->AsType()->AsTypeType()->Type()->AsRecordType(); + + Val* event_val = sval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + + if ( event ) + { + // Make sure the event prototyped as expected. + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) + { + run_time("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 1 ) + { + run_time("stream event must take a single argument"); + return false; + } + + if ( ! same_type((*args)[0], columns) ) + { + run_time("stream event's argument type does not match column record type"); + return new Val(0, TYPE_BOOL); + } + } // Make sure the vector has an entries for all streams up to the one // given. + + unsigned int idx = id->AsEnum(); + while ( idx >= streams.size() ) streams.push_back(0); @@ -126,14 +164,14 @@ bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerP // We already know this one, delete the previous definition. delete streams[idx]; - // Create new stream and record the type for the columns. + // Create new stream. streams[idx] = new Stream; - streams[idx]->name = stream_id->Type()->AsEnumType()->Lookup(idx); + streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); + streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; streams[idx]->columns = columns; - streams[idx]->event = handler; columns->Ref(); - DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", streams[idx]->name.c_str(), streams[idx]->event->Name()); + DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : ""); return true; } @@ -207,17 +245,17 @@ bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, T return true; } -bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) +bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) { RecordType* rtype = fval->Type()->AsRecordType(); - if ( ! same_type(rtype, log_filter, 0) ) + if ( ! same_type(rtype, BifType::Record::Log::Filter, 0) ) { run_time("filter argument not of right type"); return false; } - Stream* stream = FindStream(stream_id); + Stream* stream = FindStream(id); if ( ! stream ) return false; @@ -227,17 +265,9 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) Val* writer_val = fval->Lookup(idx); if ( ! writer_val ) - { // Use default. - // FIXME: Shouldn't Lookup() already take care if this? - const Attr* def_attr = log_filter->FieldDecl(idx)->FindAttr(ATTR_DEFAULT); - if ( ! def_attr ) - internal_error("log_filter missing &default for writer attribute"); + writer = BifConst::Log::default_writer->AsEnum(); - writer_val = def_attr->AttrExpr()->Eval(0); - writer = writer_val->AsEnum(); - Unref(writer_val); - } else writer = writer_val->AsEnum(); @@ -269,7 +299,6 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) // Create a new Filter instance. - Val* event = fval->Lookup(rtype->FieldOffset("ev")); Val* pred = fval->Lookup(rtype->FieldOffset("pred")); Val* path_func = fval->Lookup(rtype->FieldOffset("path_func")); @@ -302,27 +331,19 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) else { - // If no path is given, use the Stream ID as the default. - const char* n = stream->name.c_str(); - char* lower = new char[strlen(n) + 1]; - for ( char* s = lower; *n; ++n, ++s ) - { - if ( strncmp(n, "::", 2) == 0 ) - { - // Remove the scope operator. TODO: We need ab better way to - // generate the default here, but let's wait until we have - // everything in the right namespace. - *s = '_'; - ++n; - } + // If no path is given, use the Stream ID's namespace as the default + // if it has one, and it ID itself otherwise. + const char* s = stream->name.c_str(); + const char* e = strstr(s, "::"); - else - *s = tolower(*n); - } + if ( ! e ) + e = s + strlen(s); - filter->path = string(lower); - filter->path_val = new StringVal(lower); - free(lower); + string path(s, e); + std::transform(path.begin(), path.end(), path.begin(), ::tolower); + + filter->path = path; + filter->path_val = new StringVal(path.c_str()); } stream->filters.push_back(filter); @@ -344,9 +365,9 @@ bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval) return true; } -bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) +bool LogMgr::RemoveFilter(EnumVal* id, StringVal* filter) { - Stream* stream = FindStream(stream_id); + Stream* stream = FindStream(id); if ( ! stream ) return false; @@ -369,11 +390,11 @@ bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter) return true; } -bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) +bool LogMgr::Write(EnumVal* id, RecordVal* columns) { bool error = false; - Stream* stream = FindStream(stream_id); + Stream* stream = FindStream(id); if ( ! stream ) return false; @@ -415,7 +436,7 @@ bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns) if ( filter->path_func ) { val_list vl(2); - vl.append(stream_id->Ref()); + vl.append(id->Ref()); vl.append(filter->path_val->Ref()); Val* v = filter->path_func->Call(&vl); @@ -562,9 +583,9 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) return vals; } -bool LogMgr::SetBuf(EnumVal* stream_id, bool enabled) +bool LogMgr::SetBuf(EnumVal* id, bool enabled) { - Stream* stream = FindStream(stream_id); + Stream* stream = FindStream(id); if ( ! stream ) return false; @@ -579,6 +600,23 @@ bool LogMgr::SetBuf(EnumVal* stream_id, bool enabled) return true; } +bool LogMgr::Flush(EnumVal* id) + { + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + { + for ( Filter::WriterMap::iterator j = (*i)->writers.begin(); j != (*i)->writers.end(); j++ ) + j->second->Flush(); + } + + RemoveDisabledWriters(stream); + + return true; + } + void LogMgr::Error(LogWriter* writer, const char* msg) { run_time(fmt("error with writer for %s: %s", writer->Path().c_str(), msg)); diff --git a/src/LogMgr.h b/src/LogMgr.h index bd269ccfde..0f21c57b74 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -55,11 +55,12 @@ public: // These correspond to the BiFs visible on the scripting layer. The // actual BiFs just forward here. - bool CreateStream(EnumVal* stream_id, RecordType* columns, EventHandlerPtr handler); - bool AddFilter(EnumVal* stream_id, RecordVal* filter); - bool RemoveFilter(EnumVal* stream_id, StringVal* filter); - bool Write(EnumVal* stream_id, RecordVal* columns); - bool SetBuf(EnumVal* stream_id, bool enabled); // Changes the state for all writers for that stream. + bool CreateStream(EnumVal* id, RecordVal* stream); + bool AddFilter(EnumVal* id, RecordVal* filter); + bool RemoveFilter(EnumVal* id, StringVal* filter); + bool Write(EnumVal* id, RecordVal* columns); + bool SetBuf(EnumVal* id, bool enabled); // Changes the state for all writers for that stream. + bool Flush(EnumVal* id); // Flushes all writers for the stream. protected: friend class LogWriter; @@ -75,7 +76,7 @@ private: bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); - Stream* FindStream(EnumVal* stream_id); + Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); vector streams; // Indexed by stream enum. diff --git a/src/LogWriter.cc b/src/LogWriter.cc index 4292849a9a..96a4be1ebc 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -60,6 +60,17 @@ bool LogWriter::SetBuf(bool enabled) return true; } +bool LogWriter::Flush() + { + if ( ! DoFlush() ) + { + disabled = true; + return false; + } + + return true; + } + void LogWriter::Finish() { DoFinish(); diff --git a/src/LogWriter.h b/src/LogWriter.h index 780b8b2a4c..606e696e4e 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -31,9 +31,12 @@ public: // occured, in which case the writer must not be used further. bool Write(LogVal** vals); - // Sets the buffering status for the writer, if the writer supports + // Sets the buffering status for the writer, if the writer supports it. bool SetBuf(bool enabled); + // Flushes any currently buffered output, if the writer support it. + bool Flush(); + // Finished writing to this logger. Will not be called if an error has // been indicated earlier. After calling this, no more writing must be // performed. @@ -65,6 +68,12 @@ protected: // semantics. virtual bool DoSetBuf(bool enabled) = 0; + // Called to flush any currently buffered output. + // + // A writer may ignore flush requests if it doesn't fit with its + // semantics. + virtual bool DoFlush() = 0; + // Called when a log output is to be rotated. Most directly, this only // applies to writers outputting files, thoug a writer may also trigger // other regular actions if that fits a similar model diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 7218a96dde..1b9eab68c8 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -52,6 +52,12 @@ write_error: return false; } +bool LogWriterAscii::DoFlush() + { + fflush(file); + return true; + } + void LogWriterAscii::DoFinish() { } diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 7b6b67b758..e4501aa5b7 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -19,6 +19,7 @@ protected: virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals); virtual bool DoSetBuf(bool enabled); virtual bool DoRotate(string rotated_path); + virtual bool DoFlush(); virtual void DoFinish(); private: diff --git a/src/NetVar.cc b/src/NetVar.cc index b164b2cdda..6b88a75b1a 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -260,11 +260,10 @@ int record_all_packets; RecordType* script_id; TableType* id_table; -RecordType* log_filter; - #include "const.bif.netvar_def" #include "types.bif.netvar_def" #include "event.bif.netvar_def" +#include "logging.bif.netvar_def" void init_event_handlers() { @@ -320,6 +319,7 @@ void init_net_var() { #include "const.bif.netvar_init" #include "types.bif.netvar_init" +#include "logging.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); endpoint = internal_type("endpoint")->AsRecordType(); @@ -568,6 +568,4 @@ void init_net_var() script_id = internal_type("script_id")->AsRecordType(); id_table = internal_type("id_table")->AsTableType(); - - log_filter = internal_type("log_filter")->AsRecordType(); } diff --git a/src/NetVar.h b/src/NetVar.h index 2ddb855b52..f1164c0e48 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -264,8 +264,6 @@ extern int record_all_packets; extern RecordType* script_id; extern TableType* id_table; -extern RecordType* log_filter; - // Initializes globals that don't pertain to network/event analysis. extern void init_general_global_var(); @@ -275,5 +273,6 @@ extern void init_net_var(); #include "const.bif.netvar_h" #include "types.bif.netvar_h" #include "event.bif.netvar_h" +#include "logging.bif.netvar_h" #endif diff --git a/src/Type.cc b/src/Type.cc index ea39f106c2..b201373453 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1115,7 +1115,6 @@ void EnumType::AddName(const string& module_name, const char* name, bool is_expo void EnumType::AddName(const string& module_name, const char* name, bro_int_t val, bool is_export) { /* explicit value specified */ - error_t rv; if ( counter > 0 ) { error("cannot mix explicit enumerator assignment and implicit auto-increment"); diff --git a/src/Val.h b/src/Val.h index 75e96cb694..22e134d901 100644 --- a/src/Val.h +++ b/src/Val.h @@ -235,7 +235,7 @@ public: return &val.subnet_val; } - const BroType* AsType() const + BroType* AsType() const { CHECK_TAG(type->Tag(), TYPE_TYPE, "Val::Type", type_name) return type; diff --git a/src/bro.bif b/src/bro.bif index e6888af3db..054fafec0f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -360,200 +360,6 @@ function cat%(...%): string return new StringVal(s); %} -function logging_log%(index: string, rec: any%): any - %{ - // TODO: Verify that rec is a record - // TODO: Coerce to the correct record type to fill in defaults. - RecordVal *recval = rec->AsRecordVal(); - - // Lookup the stream - TableVal *streams = opt_internal_table("Logging::streams"); - VectorVal *columns; - RecordVal *stream_record; - if ( streams ) - { - stream_record = streams->Lookup(index)->AsRecordVal(); - if ( stream_record ) - { - int columns_field = stream_record->Type()->AsRecordType()->FieldOffset("columns"); - columns = stream_record->Lookup(columns_field)->AsVectorVal(); - } - } - else - { - printf("Logging framework is dead (Logging::streams not found).\n"); - return false; - } - - // Lookup all filters for stream - TableVal *filters = opt_internal_table("Logging::filters"); - TableVal *stream_filters; - if ( filters ) - { - Val *lookup_v = filters->Lookup(index); - if ( lookup_v ) - stream_filters = lookup_v->AsTableVal(); - } - else - { - printf("Logging framework is dead (Logging::filters not found).\n"); - return false; - } - - // Generate the event for the log stream - // This happens regardless of all filters. - int name_field = stream_record->Type()->AsRecordType()->FieldOffset("name"); - StringVal *log_type = stream_record->AsRecordVal()->Lookup(name_field)->AsStringVal(); - string ID_module = extract_module_name(log_type->CheckString()); - // The log event that is generated by default is MODULE_NAME::log - string log_event_name = make_full_var_name(ID_module.c_str(), "log"); - EventHandlerPtr ev_ptr = internal_handler(log_event_name.c_str()); - if ( ev_ptr ) - { - val_list* vl = new val_list; - vl->append(recval->Ref()); - mgr.QueueEvent(ev_ptr, vl, SOURCE_LOCAL); - } - - // Iterate over the stream_filters - ListVal* filter_recs = stream_filters->ConvertToList(TYPE_ANY); - for ( int i = 0; i < filter_recs->Length(); ++i ) - { - RecordVal* rv = filter_recs->Index(i)->AsListVal()->Index(0)->AsRecordVal(); - int pred_field_index = rv->Type()->AsRecordType()->FieldOffset("pred"); - Val *pred_func = rv->Lookup(pred_field_index); - if ( pred_func ) - { - val_list args(1); - args.append(recval->Ref()); - int pred_val = pred_func->AsFunc()->Call(&args)->AsBool(); - //printf("predicate val: %d\n", pred_val); - // If the predicate returns false, don't send this record on to a writer. - if ( !pred_val ) - continue; - } - - // Format and print the line - // (send line onward to the filter's WRITER in the future) - - // Get a path name for this filter - int path_field = rv->Type()->AsRecordType()->FieldOffset("path"); - Val *path_val = rv->Lookup(path_field); - StringVal *path; - // If no path is given, use the Stream ID as the path - if ( path_val ) - path = path_val->AsStringVal(); - else - path = index; - - // Get the file with the "path" name found above for this filter. - // Open a new file is one does not exist yet. - TableVal *logging_files = opt_internal_table("Logging::files"); - Val *ff = logging_files->Lookup(path); - if ( !ff ) - { - logging_files->Assign(path, new Val(new BroFile(path->CheckString(), "w"))); - ff = logging_files->Lookup(path); - } - BroFile *f = ff->AsFile(); - if ( ! f->IsOpen() ) - continue; - - // Actually format the log line and print it to the file found above. - ODesc d; - RecordType *rt = rec->Type()->AsRecordType(); - for ( unsigned i = 1; i <= columns->Size(); ++i ) - { - const char *field_name = columns->Lookup(i)->AsStringVal()->CheckString(); - int field = rt->FieldOffset(field_name); - if ( field >= 0 ) // or if there is a default value - { - rec->AsRecordVal()->Lookup(field)->Describe(&d); - d.Add("\t"); - } - } - f->Write(d.Description(), 0); - f->Write("\n",0); - - - } - - Unref(filter_recs); - - return 0; - %} - - -%%{ -#include "LogMgr.h" -%%} - -function log_create_stream%(id: Log_ID, columns: any, ev: any%) : bool - %{ - if ( columns->Type()->Tag() != TYPE_TYPE ) - { - run_time("log columns must be a type"); - return new Val(0, TYPE_BOOL); - } - - if ( columns->Type()->AsTypeType()->Type()->Tag() != TYPE_RECORD ) - { - run_time("log columns must be a record type"); - return new Val(0, TYPE_BOOL); - } - - if ( ev->Type()->Tag() != TYPE_FUNC || ! ev->Type()->AsFuncType()->IsEvent() ) - { - run_time("event argument is not of event type"); - return new Val(0, TYPE_BOOL); - } - - FuncType* et = ev->Type()->AsFuncType(); - type_list* args = et->ArgTypes()->Types(); - - if ( args->length() != 1 ) - { - run_time("event must take a single argument"); - return new Val(0, TYPE_BOOL); - } - - if ( ! same_type((*args)[0], columns->Type()->AsTypeType()->Type()) ) - { - run_time("event argument type does not match log record type"); - return new Val(0, TYPE_BOOL); - } - - EventHandler* handler = event_registry->Lookup(ev->AsFunc()->GetID()->Name()); - assert(handler); - - bool result = log_mgr->CreateStream(id->AsEnumVal(), columns->Type()->AsTypeType()->Type()->AsRecordType(), handler); - return new Val(result, TYPE_BOOL); - %} - -function log_add_filter%(id: Log_ID, filter: log_filter%) : bool - %{ - bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); - return new Val(result, TYPE_BOOL); - %} - -function log_remove_filter%(id: Log_ID, name: string%) : bool - %{ - bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name); - return new Val(result, TYPE_BOOL); - %} - -function log_write%(id: Log_ID, columns: any%) : bool - %{ - bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal()); - return new Val(result, TYPE_BOOL); - %} - -function log_set_buf%(id: Log_ID, buffered: bool%): bool - %{ - bool result = log_mgr->SetBuf(id->AsEnumVal(), buffered); - return new Val(result, TYPE_BOOL); - %} - function record_type_to_vector%(rt: string%): string_vec %{ VectorVal* result = diff --git a/src/logging.bif b/src/logging.bif index 955b3c293f..5c47375284 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -1,23 +1,48 @@ -enum log_writer %{ - WRITER_DEFAULT, - WRITER_TSV, - WRITER_SYSLOG, -}% +module Log; -void log_create%(id: Logging::Stream, columns: any%) +%%{ +#include "LogMgr.h" +#include "NetVar.h" +%%} + +const Log::default_writer: Writer; + +type Filter: record; +type Stream: record; + +function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool %{ + bool result = log_mgr->CreateStream(id->AsEnumVal(), stream->AsRecordVal()); + return new Val(result, TYPE_BOOL); %} -void log_add_filter%(id: Logging::Stream, filter: any%) +function Log::__add_filter%(id: Log::ID, filter: Log::Filter%) : bool %{ + bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val(result, TYPE_BOOL); %} -void log_remove_filter%(id: Logging::Stream, name: string%) +function Log::__remove_filter%(id: Log::ID, name: string%) : bool %{ + bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name); + return new Val(result, TYPE_BOOL); %} -void log_write%(id: Logging::Stream, columns: any%) +function Log::__write%(id: Log::ID, columns: any%) : bool %{ + bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal()); + return new Val(result, TYPE_BOOL); %} +function Log::__set_buf%(id: Log::ID, buffered: bool%): bool + %{ + bool result = log_mgr->SetBuf(id->AsEnumVal(), buffered); + return new Val(result, TYPE_BOOL); + %} + +function Log::__flush%(id: Log::ID%): bool + %{ + bool result = log_mgr->Flush(id->AsEnumVal()); + return new Val(result, TYPE_BOOL); + %} diff --git a/src/types.bif b/src/types.bif index 7b60192155..4496e444a1 100644 --- a/src/types.bif +++ b/src/types.bif @@ -50,3 +50,14 @@ enum rpc_status %{ RPC_AUTH_ERROR, RPC_UNKNOWN_ERROR, %} + +module Log; + +enum Writer %{ + WRITER_DEFAULT, + WRITER_ASCII, +%} + +enum ID %{ + Unknown, +%} diff --git a/testing/btest/Baseline/logging.remove/ssh_log_ssh.log b/testing/btest/Baseline/logging.remove/ssh.log similarity index 100% rename from testing/btest/Baseline/logging.remove/ssh_log_ssh.log rename to testing/btest/Baseline/logging.remove/ssh.log diff --git a/testing/btest/Baseline/logging.test-logging/ssh_log_ssh.log b/testing/btest/Baseline/logging.test-logging/ssh.log similarity index 100% rename from testing/btest/Baseline/logging.test-logging/ssh_log_ssh.log rename to testing/btest/Baseline/logging.test-logging/ssh.log diff --git a/testing/btest/logging/events.bro b/testing/btest/logging/events.bro index 45135c60f6..605ae0d5b6 100644 --- a/testing/btest/logging/events.bro +++ b/testing/btest/logging/events.bro @@ -8,7 +8,7 @@ module SSH; export { # Create a new ID for our log stream - redef enum Log_ID += { LOG_SSH }; + redef enum Log::ID += { SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -24,13 +24,12 @@ global ssh_log: event(rec: Log); event bro_init() { - log_create_stream(LOG_SSH, SSH::Log, ssh_log); - Log_add_default_filter(LOG_SSH); + Log::create_stream(SSH, [$columns=Log, $ev=ssh_log]); local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; local r: Log = [$t=network_time(), $id=cid, $status="success"]; - log_write(LOG_SSH, r); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); } diff --git a/testing/btest/logging/path-func.bro b/testing/btest/logging/path-func.bro index f76acd6b33..3efa9d55a4 100644 --- a/testing/btest/logging/path-func.bro +++ b/testing/btest/logging/path-func.bro @@ -9,7 +9,7 @@ module SSH; export { # Create a new ID for our log stream - redef enum Log_ID += { LOG_SSH }; + redef enum Log::ID += { SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -21,11 +21,9 @@ export { }; } -global ssh_log: event(rec: Log); - global c = -1; -function path_func(id: Log_ID, path: string) : string +function path_func(id: Log::ID, path: string) : string { c = (c + 1) % 3; @@ -34,18 +32,19 @@ function path_func(id: Log_ID, path: string) : string event bro_init() { - log_create_stream(LOG_SSH, SSH::Log, ssh_log); + Log::create_stream(SSH, [$columns=Log]); + Log::remove_default_filter(SSH); - log_add_filter(LOG_SSH, [$name="dyn", $path="static-prefix", $path_func=path_func]); + Log::add_filter(SSH, [$name="dyn", $path="static-prefix", $path_func=path_func]); - log_set_buf(LOG_SSH, F); + Log::set_buf(SSH, F); local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX2"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX3"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX2"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX3"]); } diff --git a/testing/btest/logging/pred.bro b/testing/btest/logging/pred.bro index 5e2b259d7a..38a7a53548 100644 --- a/testing/btest/logging/pred.bro +++ b/testing/btest/logging/pred.bro @@ -9,7 +9,7 @@ module SSH; export { # Create a new ID for our log stream - redef enum Log_ID += { LOG_SSH }; + redef enum Log::ID += { SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -21,8 +21,6 @@ export { }; } -global ssh_log: event(rec: Log); - function fail(rec: Log): bool { return rec$status != "success"; @@ -30,14 +28,14 @@ function fail(rec: Log): bool event bro_init() { - log_create_stream(LOG_SSH, SSH::Log, ssh_log); - - log_add_filter(LOG_SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); - log_add_filter(LOG_SSH, [$name="f2", $path="ssh.failure", $pred=fail]); + Log::create_stream(SSH, [$columns=Log]); + Log::remove_default_filter(SSH); + Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); + Log::add_filter(SSH, [$name="f2", $path="ssh.failure", $pred=fail]); local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; local r: Log = [$t=network_time(), $id=cid, $status="success"]; - log_write(LOG_SSH, r); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); } diff --git a/testing/btest/logging/remove.bro b/testing/btest/logging/remove.bro index 69d70337e5..a3fd16d760 100644 --- a/testing/btest/logging/remove.bro +++ b/testing/btest/logging/remove.bro @@ -1,15 +1,13 @@ # # @TEST-EXEC: bro -B logging %INPUT -# @TEST-EXEC: btest-diff ssh_log_ssh.log +# @TEST-EXEC: btest-diff ssh.log # @TEST-EXEC: btest-diff ssh.failure.log module SSH; -@load logging - export { # Create a new ID for our log stream - redef enum Log_ID += { LOG_SSH }; + redef enum Log::ID += { SSH }; # Define a record with all the columns the log file can have. # (I'm using a subset of fields from ssh-ext for demonstration.) @@ -21,26 +19,23 @@ export { }; } -global ssh_log: event(rec: Log); - event bro_init() { - log_create_stream(LOG_SSH, SSH::Log,ssh_log); - Log_add_default_filter(LOG_SSH); - log_add_filter(LOG_SSH, [$name="f1", $path="ssh.failure", $pred=function(rec: Log): bool { return rec$status == "failure"; }]); + Log::create_stream(SSH, [$columns=Log]); + Log::add_filter(SSH, [$name="f1", $path="ssh.failure", $pred=function(rec: Log): bool { return rec$status == "failure"; }]); local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; # Log something. - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); - log_remove_filter(LOG_SSH, "f1"); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="BR"]); + Log::remove_filter(SSH, "f1"); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="BR"]); - log_remove_filter(LOG_SSH, "default"); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + Log::remove_filter(SSH, "default"); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); - log_remove_filter(LOG_SSH, "doesn-not-exist"); + Log::remove_filter(SSH, "doesn-not-exist"); } diff --git a/testing/btest/logging/test-logging.bro b/testing/btest/logging/test-logging.bro index 464461db72..5011ecd26c 100644 --- a/testing/btest/logging/test-logging.bro +++ b/testing/btest/logging/test-logging.bro @@ -1,17 +1,12 @@ # # @TEST-EXEC: bro %INPUT -# @TEST-EXEC: btest-diff ssh_log_ssh.log +# @TEST-EXEC: btest-diff ssh.log module SSH; -@load logging - export { - # Create a new ID for our log stream - redef enum Log_ID += { LOG_SSH }; + redef enum Log::ID += { SSH }; - # Define a record with all the columns the log file can have. - # (I'm using a subset of fields from ssh-ext for demonstration.) type Log: record { t: time; id: conn_id; # Will be rolled out into individual columns. @@ -20,46 +15,17 @@ export { }; } -global ssh_log: event(rec: Log); - event bro_init() { - # Create the stream. - # First argument is the ID for the stream. - # Second argument is the log record type. - log_create_stream(LOG_SSH, SSH::Log,ssh_log); - - # Add a default filter that simply logs everything to "ssh.log" using the default writer. - Log_add_default_filter(LOG_SSH); - - # Printing headers for the filters doesn't work yet either and needs to - # be considered in the final design. (based on the "select" set). - #Log::add_filter("ssh", [$name="successful logins", - # #$pred(rec: Log) = { print rec$status; return T; }, - # $path="ssh-logins", - # #$select=set("t"), - # $writer=Log::WRITER_CSV]); + Log::create_stream(SSH, [$columns=Log]); local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; - # Log something. - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); - log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); } -#event log(rec: Log) -# { -# print fmt("Ran the log handler from the same module. Extracting time: %0.6f", rec$t); -# } -# -# -#module WHATEVER; -# -#event SSH::log(rec: SSH::Log) -# { -# print fmt("Ran the SSH::log handler from a different module. Extracting time: %0.6f", rec$t); -# } From e6edc52d54c1393614b6d00e21a925c5f3b4c13f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 28 Feb 2011 18:24:19 -0800 Subject: [PATCH 29/67] Moving the map of writers from the filters to the streams. This is in preparation for remote logging. --- src/LogMgr.cc | 59 +++++++++++++++++++++++---------------------------- 1 file changed, 26 insertions(+), 33 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 1d272e7049..f2732120f1 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -32,18 +32,20 @@ struct LogMgr::Filter { LogField** fields; vector > indices; // List of record indices per field. - typedef map WriterMap; - WriterMap writers; // Writers indexed by path. - ~Filter(); }; struct LogMgr::Stream { + int id; string name; RecordType* columns; EventHandlerPtr event; list filters; + typedef pair IdPathPair; + typedef map WriterMap; + WriterMap writers; // Writers indexed by id/path pair. + ~Stream(); }; @@ -53,15 +55,16 @@ LogMgr::Filter::~Filter() for ( int i = 0; i < num_fields; ++i ) delete fields[i]; - for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) - delete i->second; - Unref(path_val); } LogMgr::Stream::~Stream() { Unref(columns); + + for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) + delete i->second; + for ( list::iterator f = filters.begin(); f != filters.end(); ++f ) delete *f; } @@ -91,24 +94,19 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) void LogMgr::RemoveDisabledWriters(Stream* stream) { - for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) + list disabled; + + for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - Filter* filter = (*i); - - list disabled; - - for ( Filter::WriterMap::iterator j = filter->writers.begin(); j != filter->writers.end(); j++ ) + if ( j->second->Disabled() ) { - if ( j->second->Disabled() ) - { - delete j->second; - disabled.push_back(j->first); - } + delete j->second; + disabled.push_back(j->first); } - - for ( list::iterator j = disabled.begin(); j != disabled.end(); j++ ) - filter->writers.erase(*j); } + + for ( list::iterator j = disabled.begin(); j != disabled.end(); j++ ) + stream->writers.erase(*j); } bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) @@ -166,6 +164,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) // Create new stream. streams[idx] = new Stream; + streams[idx]->id = id->AsEnum(); streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; streams[idx]->columns = columns; @@ -455,10 +454,10 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) } // See if we already have a writer for this path. - Filter::WriterMap::iterator w = filter->writers.find(path); + Stream::WriterMap::iterator w = stream->writers.find(Stream::IdPathPair(stream->id, path)); LogWriter* writer = 0; - if ( w == filter->writers.end() ) + if ( w == stream->writers.end() ) { // No, need to create one. assert(filter->writer->factory); @@ -476,7 +475,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return false; } - filter->writers.insert(Filter::WriterMap::value_type(path, writer)); + stream->writers.insert(Stream::WriterMap::value_type(Stream::IdPathPair(stream->id, path), writer)); } else @@ -589,11 +588,8 @@ bool LogMgr::SetBuf(EnumVal* id, bool enabled) if ( ! stream ) return false; - for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) - { - for ( Filter::WriterMap::iterator j = (*i)->writers.begin(); j != (*i)->writers.end(); j++ ) - j->second->SetBuf(enabled); - } + for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) + i->second->SetBuf(enabled); RemoveDisabledWriters(stream); @@ -606,11 +602,8 @@ bool LogMgr::Flush(EnumVal* id) if ( ! stream ) return false; - for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) - { - for ( Filter::WriterMap::iterator j = (*i)->writers.begin(); j != (*i)->writers.end(); j++ ) - j->second->Flush(); - } + for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) + i->second->Flush(); RemoveDisabledWriters(stream); From ba2c6f6139c5710ae2a17a74e558d97ec4a05f65 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 28 Feb 2011 21:43:48 -0800 Subject: [PATCH 30/67] Internal refactoring to provide injection points for remotely received log records. Also added some additional type-safety check to later make sure that what a peer sends is actually of the expected format. --- src/LogMgr.cc | 177 +++++++++++++++++++++++++++++++---------------- src/LogMgr.h | 18 ++--- src/LogWriter.cc | 11 ++- src/LogWriter.h | 11 +-- 4 files changed, 142 insertions(+), 75 deletions(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index f2732120f1..828fd95a4a 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -7,17 +7,17 @@ #include "LogWriterAscii.h" struct LogWriterDefinition { - LogWriterType::Type type; // The type. + bro_int_t type; // The type. const char *name; // Descriptive name for error messages. bool (*init)(); // An optional one-time initialization function. LogWriter* (*factory)(); // A factory function creating instances. }; LogWriterDefinition log_writers[] = { - { LogWriterType::Ascii, "Ascii", 0, LogWriterAscii::Instantiate }, + { BifEnum::Log::WRITER_ASCII, "Ascii", 0, LogWriterAscii::Instantiate }, // End marker. - { LogWriterType::None, "None", 0, (LogWriter* (*)())0 } + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (LogWriter* (*)())0 } }; struct LogMgr::Filter { @@ -26,7 +26,7 @@ struct LogMgr::Filter { Func* path_func; string path; Val* path_val; - LogWriterDefinition* writer; + EnumVal* writer; int num_fields; LogField** fields; @@ -36,14 +36,14 @@ struct LogMgr::Filter { }; struct LogMgr::Stream { - int id; + EnumVal* id; string name; RecordType* columns; EventHandlerPtr event; list filters; - typedef pair IdPathPair; - typedef map WriterMap; + typedef pair WriterPathPair; + typedef map WriterMap; WriterMap writers; // Writers indexed by id/path pair. ~Stream(); @@ -94,7 +94,7 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) void LogMgr::RemoveDisabledWriters(Stream* stream) { - list disabled; + list disabled; for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { @@ -104,8 +104,8 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) disabled.push_back(j->first); } } - - for ( list::iterator j = disabled.begin(); j != disabled.end(); j++ ) + + for ( list::iterator j = disabled.begin(); j != disabled.end(); j++ ) stream->writers.erase(*j); } @@ -164,11 +164,10 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) // Create new stream. streams[idx] = new Stream; - streams[idx]->id = id->AsEnum(); + streams[idx]->id = id->Ref()->AsEnumVal(); streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; - streams[idx]->columns = columns; - columns->Ref(); + streams[idx]->columns = columns->Ref()->AsRecordType(); DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : ""); @@ -270,32 +269,6 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) else writer = writer_val->AsEnum(); - LogWriterDefinition* ld; - for ( ld = log_writers; ld->type != LogWriterType::None; ++ld ) - { - if ( ld->type == writer ) - break; - } - - if ( ld->type == LogWriterType::None ) - internal_error("unknow writer in add_filter"); - - if ( ! ld->factory ) - // Oops, we can't instantuate this guy. - return true; // Count as success, as we will have reported it earlier already. - - // If the writer has an init function, call it. - if ( ld->init ) - { - if ( (*ld->init)() ) - // Clear the init function so that we won't call it again later. - ld->init = 0; - else - // Init failed, disable by deleting factory function. - ld->factory = 0; - return false; - } - // Create a new Filter instance. Val* pred = fval->Lookup(rtype->FieldOffset("pred")); @@ -305,7 +278,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); filter->pred = pred ? pred->AsFunc() : 0; filter->path_func = path_func ? path_func->AsFunc() : 0; - filter->writer = ld; + filter->writer = id->Ref()->AsEnumVal(); // TODO: Check that the predciate is of the right type. @@ -349,7 +322,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) #ifdef DEBUG DBG_LOG(DBG_LOGGING, "Created new filter '%s' for stream '%s'", filter->name.c_str(), stream->name.c_str()); - DBG_LOG(DBG_LOGGING, " writer : %s", ld->name); + DBG_LOG(DBG_LOGGING, " writer : %d", filter->writer); DBG_LOG(DBG_LOGGING, " path : %s", filter->path.c_str()); DBG_LOG(DBG_LOGGING, " path_func : %s", (filter->path_func ? "set" : "not set")); DBG_LOG(DBG_LOGGING, " pred : %s", (filter->pred ? "set" : "not set")); @@ -454,14 +427,16 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) } // See if we already have a writer for this path. - Stream::WriterMap::iterator w = stream->writers.find(Stream::IdPathPair(stream->id, path)); + Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); LogWriter* writer = 0; - if ( w == stream->writers.end() ) + + if ( w != stream->writers.end() ) + // We have a writer already. + writer = w->second; + else { // No, need to create one. - assert(filter->writer->factory); - writer = (*filter->writer->factory)(); // Copy the fields for LogWriter::Init() as it will take // ownership. @@ -469,22 +444,18 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) for ( int j = 0; j < filter->num_fields; ++j ) arg_fields[j] = new LogField(*filter->fields[j]); - if ( ! writer->Init(path, filter->num_fields, arg_fields) ) + writer = CreateWriter(stream->id, filter->writer, path, filter->num_fields, arg_fields); + + if ( ! writer ) { Unref(columns); return false; } - - stream->writers.insert(Stream::WriterMap::value_type(Stream::IdPathPair(stream->id, path), writer)); } - else - // We have a writer already. - writer = w->second; - // Alright, can do the write now. LogVal** vals = RecordToFilterVals(filter, columns); - if ( ! writer->Write(vals) ) + if ( ! writer->Write(filter->num_fields, vals) ) error = true; #ifdef DEBUG @@ -506,6 +477,7 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) for ( int i = 0; i < filter->num_fields; ++i ) { + TypeTag type = TYPE_ERROR; Val* val = columns; // For each field, first find the right value, which can potentially @@ -514,12 +486,13 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) for ( list::iterator j = indices.begin(); j != indices.end(); ++j ) { + type = val->Type()->AsRecordType()->FieldType(*j)->Tag(); val = val->AsRecordVal()->Lookup(*j); if ( ! val ) { // Value, or any of its parents, is not set. - vals[i] = new LogVal(false); + vals[i] = new LogVal(type, false); break; } } @@ -531,26 +504,26 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) case TYPE_BOOL: case TYPE_INT: case TYPE_ENUM: - vals[i] = new LogVal(); + vals[i] = new LogVal(type); vals[i]->val.int_val = val->InternalInt(); break; case TYPE_COUNT: case TYPE_COUNTER: case TYPE_PORT: - vals[i] = new LogVal(); + vals[i] = new LogVal(type); vals[i]->val.uint_val = val->InternalUnsigned(); break; case TYPE_SUBNET: - vals[i] = new LogVal(); + vals[i] = new LogVal(type); vals[i]->val.subnet_val = *val->AsSubNet(); break; case TYPE_NET: case TYPE_ADDR: { - vals[i] = new LogVal(); + vals[i] = new LogVal(type); addr_type t = val->AsAddr(); copy_addr(&t, &vals[i]->val.addr_val); break; @@ -559,7 +532,7 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - vals[i] = new LogVal(); + vals[i] = new LogVal(type); vals[i]->val.double_val = val->InternalDouble(); break; @@ -567,7 +540,7 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) { const BroString* s = val->AsString(); LogVal* lval = (LogVal*) new char[sizeof(LogVal) + sizeof(log_string_type) + s->Len()]; - new (lval) LogVal(); // Run ctor. + new (lval) LogVal(type); // Run ctor. lval->val.string_val.len = s->Len(); memcpy(&lval->val.string_val.string, s->Bytes(), s->Len()); vals[i] = lval; @@ -582,6 +555,90 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) return vals; } +LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, LogField** fields) + { + Stream* stream = FindStream(id); + + if ( ! stream ) + // Don't know this stream. + return false; + + Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); + + if ( w != stream->writers.end() ) + // If we already have a writer for this. That's fine, we just return + // it. + return w->second; + + // Need to instantiate a new writer. + + LogWriterDefinition* ld = log_writers; + + while ( true ) + { + if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) + { + run_time("unknow writer when creating writer"); + return 0; + } + + if ( ld->type == writer->AsEnum() ) + break; + + if ( ! ld->factory ) + // Oops, we can't instantuate this guy. + return 0; + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't call it again later. + ld->init = 0; + else + // Init failed, disable by deleting factory function. + ld->factory = 0; + + return false; + } + + ++ld; + } + + assert(ld->factory); + LogWriter* writer_obj = (*ld->factory)(); + + if ( ! writer_obj->Init(path, num_fields, fields) ) + return 0; + + stream->writers.insert(Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), writer_obj)); + + return writer_obj; + } + +bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, LogVal** vals) + { + Stream* stream = FindStream(id); + + if ( ! stream ) + // Don't know this stream. + return false; + + Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); + + if ( w == stream->writers.end() ) + // Don't know this writer. + return false; + + bool success = w->second->Write(num_fields, vals); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to '%s' on stream '%s'", path.c_str(), stream->name.c_str()); +#endif + + return success; + } + bool LogMgr::SetBuf(EnumVal* id, bool enabled) { Stream* stream = FindStream(id); diff --git a/src/LogMgr.h b/src/LogMgr.h index 0f21c57b74..ca1cda8e96 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -7,14 +7,6 @@ #include "Val.h" #include "EventHandler.h" -// One value per writer type we have. -namespace LogWriterType { - enum Type { - None, - Ascii - }; -}; - struct LogField { LogField() { } LogField(const LogField& other) : name(other.name), type(other.type) { } @@ -30,8 +22,9 @@ struct log_string_type { // All values that can be directly logged by a Writer. struct LogVal { - LogVal(bool arg_present = true) : present(arg_present) {} + LogVal(TypeTag arg_type, bool arg_present = true) : type(arg_type), present(arg_present) {} + TypeTag type; bool present; // If false, the field is unset (i.e., &optional and not initialzed). // The following union is a subset of BroValUnion, including only the @@ -47,6 +40,7 @@ struct LogVal { }; class LogWriter; +class RemoteSerializer; class LogMgr { public: @@ -64,6 +58,12 @@ public: protected: friend class LogWriter; + friend class RemoteSerializer; + + // These function are also used by the RemoteSerializer to inject + // received logs. + LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, LogField** fields); + bool Write(EnumVal* id, EnumVal* writer, string path, int num_fields, LogVal** vals); /// Functions also used by the writers. diff --git a/src/LogWriter.cc b/src/LogWriter.cc index 96a4be1ebc..53a36476a3 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -33,8 +33,17 @@ bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields) return true; } -bool LogWriter::Write(LogVal** vals) +bool LogWriter::Write(int arg_num_fields, LogVal** vals) { + // Double-check that the arguments match. If we get this from remote, + // something might be mixed up. + if ( num_fields != arg_num_fields ) + return false; + + for ( int i = 0; i < num_fields; ++i ) + if ( vals[i]->type != fields[i]->type ) + return false; + bool result = DoWrite(num_fields, fields, vals); DeleteVals(vals); diff --git a/src/LogWriter.h b/src/LogWriter.h index 606e696e4e..86ca4d10a8 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -25,11 +25,12 @@ public: // when done. bool Init(string path, int num_fields, LogField** fields); - // Writes one log entry. The method takes ownership of "vals" and will - // return immediately after queueing the write request, potentially - // before the output has actually taken place. Returns false if an error - // occured, in which case the writer must not be used further. - bool Write(LogVal** vals); + // Writes one log entry. The method takes ownership of "vals" and will + // return immediately after queueing the write request, potentially + // before the output has actually taken place. Returns false if an error + // occured, in which case the writer must not be used further. num_fields + // and types must match what was passed to Init(). + bool Write(int num_fields, LogVal** vals); // Sets the buffering status for the writer, if the writer supports it. bool SetBuf(bool enabled); From d673c8c64c402a4100a0bc0abd8759e1ad3160a4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 28 Feb 2011 22:20:29 -0800 Subject: [PATCH 31/67] Adding a new RecordVal method LookupWithDefault(). The method honors a potential &default attribute, and refs the Val it returns. --- src/Val.cc | 17 +++++++++++++++-- src/Val.h | 3 ++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/Val.cc b/src/Val.cc index f7767714ac..ba1cda41e7 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1734,7 +1734,7 @@ void TableVal::CheckExpireAttr(attr_tag at) a->AttrExpr()->Error("value of timeout not fixed"); return; } - + expire_time = timeout->AsInterval(); if ( timer ) @@ -2893,6 +2893,19 @@ Val* RecordVal::Lookup(int field) const return (*AsRecord())[field]; } +Val* RecordVal::LookupWithDefault(int field) const + { + Val* val = (*AsRecord())[field]; + + if ( val ) + return val->Ref(); + + // Check for &default. + const Attr* def_attr = record_type->FieldDecl(field)->attrs->FindAttr(ATTR_DEFAULT); + + return def_attr ? def_attr->AttrExpr()->Eval(0) : 0; + } + RecordVal* RecordVal::CoerceTo(const RecordType* t, Val* aggr) const { if ( ! record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) ) @@ -3348,7 +3361,7 @@ Val* check_and_promote(Val* v, const BroType* t, int is_init) TypeTag t_tag = t->Tag(); TypeTag v_tag = vt->Tag(); - + // More thought definitely needs to go into this. if ( t_tag == TYPE_ANY || v_tag == TYPE_ANY ) return v; diff --git a/src/Val.h b/src/Val.h index 22e134d901..763a32db54 100644 --- a/src/Val.h +++ b/src/Val.h @@ -909,7 +909,8 @@ public: { return new Val(record_type->NumFields(), TYPE_COUNT); } void Assign(int field, Val* new_val, Opcode op = OP_ASSIGN); - Val* Lookup(int field) const; + Val* Lookup(int field) const; // does not Ref() value. + Val* LookupWithDefault(int field) const; // does Ref() value. void Describe(ODesc* d) const; From c355f5d1fa0209658d24abaecaf6489fdedc2b55 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 28 Feb 2011 22:21:53 -0800 Subject: [PATCH 32/67] Adding options to enable/disable local and remote logging. Only the local option has an effect right now. Also moving Log::default_writer out of the bif into logging.bro. --- policy/logging.bro | 23 ++++++++++++++------- src/LogMgr.cc | 17 +++++++-------- src/logging.bif | 2 -- testing/btest/logging/no-local.bro | 33 ++++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 17 deletions(-) create mode 100644 testing/btest/logging/no-local.bro diff --git a/policy/logging.bro b/policy/logging.bro index 36f866c91f..c5613e4ada 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -3,6 +3,15 @@ module Log; # Log::ID and Log::Writer are defined in bro.init due to circular dependencies. export { + # The default writer to use. + const default_writer = Log::WRITER_ASCII &redef; + + # If true, local logging is by default enabled for all filters. + const enable_local_logging = T &redef; + + # If true, remote logging is by default enabled for all filters. + const enable_remote_logging = T &redef; + # A stream defining the logging. type Stream: record { # A record type defining the log's columns. @@ -41,8 +50,14 @@ export { include: set[string] &optional; exclude: set[string] &optional; + # If true, record all log records locally. + log_local: bool &default=Log::enable_local_logging; + + # If true, pass all log records on to remote peers if they request it. + log_remote: bool &default=Log::enable_remote_logging; + # The writer to use. - writer: Writer &optional; + writer: Writer &default=Log::default_writer; }; global create_stream: function(id: Log::ID, stream: Log::Stream) : bool; @@ -59,12 +74,6 @@ export { module Log; -export { - # The default writer to use if a filter does not specify - # anything else. - const default_writer = Log::WRITER_ASCII &redef; -} - function create_stream(id: Log::ID, stream: Log::Stream) : bool { if ( ! Log::__create_stream(id, stream) ) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 828fd95a4a..ca566a84b3 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -27,6 +27,8 @@ struct LogMgr::Filter { string path; Val* path_val; EnumVal* writer; + bool local; + bool remote; int num_fields; LogField** fields; @@ -260,14 +262,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) // Find the right writer type. int writer = 0; int idx = rtype->FieldOffset("writer"); - Val* writer_val = fval->Lookup(idx); - - if ( ! writer_val ) - // Use default. - writer = BifConst::Log::default_writer->AsEnum(); - - else - writer = writer_val->AsEnum(); + writer = fval->LookupWithDefault(idx)->AsEnum(); // Create a new Filter instance. @@ -279,6 +274,8 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) filter->pred = pred ? pred->AsFunc() : 0; filter->path_func = path_func ? path_func->AsFunc() : 0; filter->writer = id->Ref()->AsEnumVal(); + filter->local = fval->LookupWithDefault(rtype->FieldOffset("log_local"))->AsBool(); + filter->remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote"))->AsBool(); // TODO: Check that the predciate is of the right type. @@ -426,6 +423,10 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) #endif } + if ( ! filter->local ) + // Skip the subsequent local logging code. + continue; + // See if we already have a writer for this path. Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); diff --git a/src/logging.bif b/src/logging.bif index 5c47375284..e31c56276e 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -6,8 +6,6 @@ module Log; #include "NetVar.h" %%} -const Log::default_writer: Writer; - type Filter: record; type Stream: record; diff --git a/testing/btest/logging/no-local.bro b/testing/btest/logging/no-local.bro new file mode 100644 index 0000000000..43eccce2c5 --- /dev/null +++ b/testing/btest/logging/no-local.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +redef Log::enable_local_logging = F; + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From 3f413a2539e180b2a287ae108acfc8a6d64bbec2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 3 Mar 2011 16:35:51 -0800 Subject: [PATCH 33/67] Remote logging for the new logging framework. It works with a simple example, but that's as much testing as it has seen so far. Remote::Destination has a new attribute "request_logs: bool" indicating whether we are interested in the peer's log. Default is false. If true, Bro will send an explicit "I want your logs" message over to the other side, which will then start sending log records back. When such log records are received, they will be recorded exactly in the same way as on the remote side, i.e., same fields/writer/path. All filtering is already performed on the remote side. Log::Filter has two new attributes, "log_local: bool" and "log_remote: bool" (both true by default). If log_local is false, this filter will not record anything locally but still process everything normally otherwise and potentially forward to remote. If log_remote is false, this filter will never send anything to remote even if a peer has requested logs. (Note that with the defaults, requesting logs will mean getting everything.) Note that with log forwarding, *both* sides must create the Filter::Stream. If the remote sends log records for a specific stream, but the local side hasn't created it, the data will be discarded. Filtes on the other hand shouldn't created locally; and if they are, they are ignored for records received from remote). --- policy/remote.bro | 9 + src/LogMgr.cc | 243 +++++++++++++++++++++--- src/LogMgr.h | 44 +++-- src/LogWriter.cc | 6 + src/LogWriter.h | 4 + src/LogWriterAscii.cc | 2 +- src/RemoteSerializer.cc | 366 +++++++++++++++++++++++++++++++++++-- src/RemoteSerializer.h | 22 +++ src/SerializationFormat.cc | 33 ++++ src/SerializationFormat.h | 14 ++ src/bro.bif | 6 + 11 files changed, 690 insertions(+), 59 deletions(-) diff --git a/policy/remote.bro b/policy/remote.bro index fad2beb900..ae9fc14224 100644 --- a/policy/remote.bro +++ b/policy/remote.bro @@ -43,6 +43,9 @@ export { # Whether to perform state synchronization with peer. sync: bool &default = T; + # Whether to requests logs from the peer. + request_logs: bool &default = F; + # When performing state synchronization, whether we consider # our state to be authoritative. If so, we will send the peer # our current set when the connection is set up. @@ -176,6 +179,12 @@ function setup_peer(p: event_peer, dst: Destination) request_remote_sync(p, dst$auth); } + if ( dst$request_logs ) + { + do_script_log(p, "requesting logs"); + request_remote_logs(p); + } + dst$peer = p; dst$connected = T; connected_peers[p$id] = dst; diff --git a/src/LogMgr.cc b/src/LogMgr.cc index ca566a84b3..15db1cb927 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -51,6 +51,154 @@ struct LogMgr::Stream { ~Stream(); }; +bool LogVal::Read(SerializationFormat* fmt) + { + int ty; + + if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) + return false; + + type = (TypeTag)(ty); + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + return fmt->Read(&val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Read(&val.uint_val, "uint"); + + case TYPE_SUBNET: + { + uint32 net[4]; + if ( ! (fmt->Read(&net[0], "net0") + && fmt->Read(&net[1], "net1") + && fmt->Read(&net[2], "net2") + && fmt->Read(&net[3], "net3") + && fmt->Read(&val.subnet_val.width, "width")) ) + return false; + +#ifdef BROv6 + val.subnet_val.net[0] = net[0]; + val.subnet_val.net[1] = net[1]; + val.subnet_val.net[2] = net[2]; + val.subnet_val.net[3] = net[3]; +#else + val.subnet_val.net = net[0]; +#endif + return true; + } + + case TYPE_NET: + case TYPE_ADDR: + { + uint32 addr[4]; + if ( ! (fmt->Read(&addr[0], "addr0") + && fmt->Read(&addr[1], "addr1") + && fmt->Read(&addr[2], "addr2") + && fmt->Read(&addr[3], "addr3")) ) + return false; + +#ifdef BROv6 + val.addr_val[0] = addr[0]; + val.addr_val[1] = addr[1]; + val.addr_val[2] = addr[2]; + val.addr_val[3] = addr[3]; +#else + val.addr_val = addr[0]; +#endif + return true; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Read(&val.double_val, "double"); + + case TYPE_STRING: + { + val.string_val = new string; + return fmt->Read(val.string_val, "string"); + } + + default: + internal_error(::fmt("unsupported type %s in LogVal::Write", type_name(type))); + } + + return false; + } + +bool LogVal::Write(SerializationFormat* fmt) const + { + if ( ! (fmt->Write((int)type, "type") && fmt->Write(present, "present")) ) + return false; + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + return fmt->Write(val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Write(val.uint_val, "uint"); + + case TYPE_SUBNET: + { +#ifdef BROv6 + uint32* net = val.subnet_val; +#else + uint32 net[4]; + net[0] = val.subnet_val.net; + net[1] = net[2] = net[3] = 0; +#endif + return fmt->Write(net[0], "net0") + && fmt->Write(net[1], "net1") + && fmt->Write(net[2], "net2") + && fmt->Write(net[3], "net3") + && fmt->Write(val.subnet_val.width, "width"); + } + + case TYPE_NET: + case TYPE_ADDR: + { +#ifdef BROv6 + uint32* addr = val.addr_val; +#else + uint32 addr[4]; + addr[0] = val.addr_val; + addr[1] = addr[2] = addr[3] = 0; +#endif + return fmt->Write(addr[0], "addr0") + && fmt->Write(addr[1], "addr1") + && fmt->Write(addr[2], "addr2") + && fmt->Write(addr[3], "addr3"); + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Write(val.double_val, "double"); + + case TYPE_STRING: + return fmt->Write(*val.string_val, "string"); + + default: + internal_error(::fmt("unsupported type %s in LogVal::REad", type_name(type))); + } + + return false; + } LogMgr::Filter::~Filter() { @@ -87,7 +235,7 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) if ( idx >= streams.size() || ! streams[idx] ) { - run_time("unknown log stream"); + run_time(fmt("unknown log stream (%d)", id->AsEnum())); return 0; } @@ -260,9 +408,8 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) return false; // Find the right writer type. - int writer = 0; int idx = rtype->FieldOffset("writer"); - writer = fval->LookupWithDefault(idx)->AsEnum(); + EnumVal* writer = fval->LookupWithDefault(idx)->AsEnumVal(); // Create a new Filter instance. @@ -273,7 +420,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); filter->pred = pred ? pred->AsFunc() : 0; filter->path_func = path_func ? path_func->AsFunc() : 0; - filter->writer = id->Ref()->AsEnumVal(); + filter->writer = writer->Ref()->AsEnumVal(); filter->local = fval->LookupWithDefault(rtype->FieldOffset("log_local"))->AsBool(); filter->remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote"))->AsBool(); @@ -318,8 +465,11 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) stream->filters.push_back(filter); #ifdef DEBUG + ODesc desc; + writer->Describe(&desc); + DBG_LOG(DBG_LOGGING, "Created new filter '%s' for stream '%s'", filter->name.c_str(), stream->name.c_str()); - DBG_LOG(DBG_LOGGING, " writer : %d", filter->writer); + DBG_LOG(DBG_LOGGING, " writer : %s", desc.Description()); DBG_LOG(DBG_LOGGING, " path : %s", filter->path.c_str()); DBG_LOG(DBG_LOGGING, " path_func : %s", (filter->path_func ? "set" : "not set")); DBG_LOG(DBG_LOGGING, " pred : %s", (filter->pred ? "set" : "not set")); @@ -371,7 +521,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( ! columns ) { - run_time("imcompatible log record type"); + run_time("incompatible log record type"); return false; } @@ -423,10 +573,6 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) #endif } - if ( ! filter->local ) - // Skip the subsequent local logging code. - continue; - // See if we already have a writer for this path. Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); @@ -445,18 +591,29 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) for ( int j = 0; j < filter->num_fields; ++j ) arg_fields[j] = new LogField(*filter->fields[j]); - writer = CreateWriter(stream->id, filter->writer, path, filter->num_fields, arg_fields); - - if ( ! writer ) + if ( filter->local ) { - Unref(columns); - return false; + writer = CreateWriter(stream->id, filter->writer, path, filter->num_fields, arg_fields); + + if ( ! writer ) + { + Unref(columns); + return false; + } } + + if ( filter->remote ) + remote_serializer->SendLogCreateWriter(stream->id, filter->writer, path, filter->num_fields, arg_fields); } // Alright, can do the write now. + LogVal** vals = RecordToFilterVals(filter, columns); - if ( ! writer->Write(filter->num_fields, vals) ) + + if ( filter->remote ) + remote_serializer->SendLogWrite(stream->id, filter->writer, path, filter->num_fields, vals); + + if ( filter->local && ! writer->Write(filter->num_fields, vals) ) error = true; #ifdef DEBUG @@ -501,30 +658,28 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) if ( ! val ) continue; + vals[i] = new LogVal(type); + switch ( val->Type()->Tag() ) { case TYPE_BOOL: case TYPE_INT: case TYPE_ENUM: - vals[i] = new LogVal(type); vals[i]->val.int_val = val->InternalInt(); break; case TYPE_COUNT: case TYPE_COUNTER: case TYPE_PORT: - vals[i] = new LogVal(type); vals[i]->val.uint_val = val->InternalUnsigned(); break; case TYPE_SUBNET: - vals[i] = new LogVal(type); vals[i]->val.subnet_val = *val->AsSubNet(); break; case TYPE_NET: case TYPE_ADDR: { - vals[i] = new LogVal(type); addr_type t = val->AsAddr(); copy_addr(&t, &vals[i]->val.addr_val); break; @@ -533,18 +688,13 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) case TYPE_DOUBLE: case TYPE_TIME: case TYPE_INTERVAL: - vals[i] = new LogVal(type); vals[i]->val.double_val = val->InternalDouble(); break; case TYPE_STRING: { const BroString* s = val->AsString(); - LogVal* lval = (LogVal*) new char[sizeof(LogVal) + sizeof(log_string_type) + s->Len()]; - new (lval) LogVal(type); // Run ctor. - lval->val.string_val.len = s->Len(); - memcpy(&lval->val.string_val.string, s->Bytes(), s->Len()); - vals[i] = lval; + vals[i]->val.string_val = new string((const char*) s->Bytes(), s->Len()); break; } @@ -600,6 +750,7 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n // Init failed, disable by deleting factory function. ld->factory = 0; + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", ld->name); return false; } @@ -610,7 +761,10 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n LogWriter* writer_obj = (*ld->factory)(); if ( ! writer_obj->Init(path, num_fields, fields) ) + { + DBG_LOG(DBG_LOGGING, "failed to init instance of writer %s", ld->name); return 0; + } stream->writers.insert(Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), writer_obj)); @@ -622,24 +776,54 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, Lo Stream* stream = FindStream(id); if ( ! stream ) + { // Don't know this stream. +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_LOGGING, "unknown stream %s in LogMgr::Write()", desc.Description()); +#endif return false; + } Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); if ( w == stream->writers.end() ) + { // Don't know this writer. +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_LOGGING, "unknown writer %s in LogMgr::Write()", desc.Description()); +#endif return false; + } bool success = w->second->Write(num_fields, vals); -#ifdef DEBUG - DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to '%s' on stream '%s'", path.c_str(), stream->name.c_str()); -#endif + DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); return success; } +void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + { + Stream* stream = (*s); + + if ( ! stream ) + continue; + + for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) + { + LogWriter* writer = i->second; + EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); + remote_serializer->SendLogCreateWriter(peer, (*s)->id, &writer_val, i->first.second, writer->NumFields(), writer->Fields()); + } + } + } + bool LogMgr::SetBuf(EnumVal* id, bool enabled) { Stream* stream = FindStream(id); @@ -672,3 +856,4 @@ void LogMgr::Error(LogWriter* writer, const char* msg) { run_time(fmt("error with writer for %s: %s", writer->Path().c_str(), msg)); } + diff --git a/src/LogMgr.h b/src/LogMgr.h index ca1cda8e96..069ddfdff2 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -6,24 +6,31 @@ #include "Val.h" #include "EventHandler.h" +#include "RemoteSerializer.h" + +class SerializationFormat; struct LogField { - LogField() { } - LogField(const LogField& other) : name(other.name), type(other.type) { } string name; TypeTag type; -}; -// A string that we can directly include as part of the value union below. -struct log_string_type { - int len; - char string[]; // The string starts right here. + LogField() { } + LogField(const LogField& other) : name(other.name), type(other.type) { } + + bool Read(SerializationFormat* fmt) + { + int t; + bool success = fmt->Read(&name, "name") && fmt->Read(&t, "type"); + type = (TypeTag) t; + return success; + } + + bool Write(SerializationFormat* fmt) const + { return fmt->Write(name, "name") && fmt->Write((int)type, "type"); } }; // All values that can be directly logged by a Writer. struct LogVal { - LogVal(TypeTag arg_type, bool arg_present = true) : type(arg_type), present(arg_present) {} - TypeTag type; bool present; // If false, the field is unset (i.e., &optional and not initialzed). @@ -35,8 +42,17 @@ struct LogVal { addr_type addr_val; subnet_type subnet_val; double double_val; - log_string_type string_val; + string* string_val; } val; + + LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} + ~LogVal() { if ( type == TYPE_STRING && present ) delete val.string_val; } + + bool Read(SerializationFormat* fmt); + bool Write(SerializationFormat* fmt) const; + +private: + LogVal(const LogVal& other) { } }; class LogWriter; @@ -60,10 +76,10 @@ protected: friend class LogWriter; friend class RemoteSerializer; - // These function are also used by the RemoteSerializer to inject - // received logs. - LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, LogField** fields); - bool Write(EnumVal* id, EnumVal* writer, string path, int num_fields, LogVal** vals); + // These function are also used by the RemoteSerializer. + LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, LogField** fields); // takes ownership of fields. + bool Write(EnumVal* id, EnumVal* writer, string path, int num_fields, LogVal** vals); // takes ownership of vals. + void SendAllWritersTo(RemoteSerializer::PeerID peer); /// Functions also used by the writers. diff --git a/src/LogWriter.cc b/src/LogWriter.cc index 53a36476a3..6c27bb55b5 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -38,11 +38,17 @@ bool LogWriter::Write(int arg_num_fields, LogVal** vals) // Double-check that the arguments match. If we get this from remote, // something might be mixed up. if ( num_fields != arg_num_fields ) + { + DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)", arg_num_fields, num_fields); return false; + } for ( int i = 0; i < num_fields; ++i ) if ( vals[i]->type != fields[i]->type ) + { + DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)", vals[i]->type, fields[i]->type); return false; + } bool result = DoWrite(num_fields, fields, vals); DeleteVals(vals); diff --git a/src/LogWriter.h b/src/LogWriter.h index 86ca4d10a8..0dd37d2881 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -43,6 +43,10 @@ public: // performed. void Finish(); + + int NumFields() const { return num_fields; } + const LogField* const * Fields() const { return fields; } + protected: // Methods for Writers to override. If any of these returs false, it will diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 1b9eab68c8..902cbf7ea2 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -114,7 +114,7 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) break; case TYPE_STRING: - desc.AddN((const char*)&val->val.string_val.string, val->val.string_val.len); + desc.AddN(val->val.string_val->data(), val->val.string_val->size()); break; default: diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 15b1872680..f1224797bc 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -183,6 +183,7 @@ #include "Sessions.h" #include "File.h" #include "Conn.h" +#include "LogMgr.h" extern "C" { #include "setsignal.h" @@ -190,7 +191,7 @@ extern "C" { // Gets incremented each time there's an incompatible change // to the communication internals. -static const unsigned short PROTOCOL_VERSION = 0x06; +static const unsigned short PROTOCOL_VERSION = 0x07; static const char MSG_NONE = 0x00; static const char MSG_VERSION = 0x01; @@ -216,9 +217,12 @@ static const char MSG_SYNC_POINT = 0x14; static const char MSG_TERMINATE = 0x15; static const char MSG_DEBUG_DUMP = 0x16; static const char MSG_REMOTE_PRINT = 0x17; +static const char MSG_LOG_CREATE_WRITER = 0x18; +static const char MSG_LOG_WRITE = 0x19; +static const char MSG_REQUEST_LOGS = 0x20; // Update this one whenever adding a new ID: -static const char MSG_ID_MAX = MSG_REMOTE_PRINT; +static const char MSG_ID_MAX = MSG_REQUEST_LOGS; static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U; @@ -226,6 +230,9 @@ static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U; static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; +// Buffer size for remote-log data. +static const int LOG_BUFFER_SIZE = 50 * 1024; + struct ping_args { uint32 seq; double time1; // Round-trip time parent1<->parent2 @@ -303,6 +310,9 @@ static const char* msgToStr(int msg) MSG_STR(MSG_TERMINATE) MSG_STR(MSG_DEBUG_DUMP) MSG_STR(MSG_REMOTE_PRINT) + MSG_STR(MSG_LOG_CREATE_WRITER) + MSG_STR(MSG_LOG_WRITE) + MSG_STR(MSG_REQUEST_LOGS) default: return "UNKNOWN_MSG"; } @@ -469,7 +479,10 @@ static inline bool is_peer_msg(int msg) msg == MSG_CAPS || msg == MSG_COMPRESS || msg == MSG_SYNC_POINT || - msg == MSG_REMOTE_PRINT; + msg == MSG_REMOTE_PRINT || + msg == MSG_LOG_CREATE_WRITER || + msg == MSG_LOG_WRITE || + msg == MSG_REQUEST_LOGS; } bool RemoteSerializer::IsConnectedPeer(PeerID id) @@ -704,6 +717,7 @@ bool RemoteSerializer::CloseConnection(Peer* peer) return true; FlushPrintBuffer(peer); + FlushLogBuffer(peer); Log(LogInfo, "closing connection", peer); @@ -738,6 +752,31 @@ bool RemoteSerializer::RequestSync(PeerID id, bool auth) return true; } +bool RemoteSerializer::RequestLogs(PeerID id) + { + if ( ! using_communication ) + return true; + + Peer* peer = LookupPeer(id, true); + if ( ! peer ) + { + run_time(fmt("unknown peer id %d for request logs", int(id))); + return false; + } + + if ( peer->phase != Peer::HANDSHAKE ) + { + run_time(fmt("can't request logs from peer; wrong phase %d", + peer->phase)); + return false; + } + + if ( ! SendToChild(MSG_REQUEST_LOGS, peer, 0) ) + return false; + + return true; + } + bool RemoteSerializer::RequestEvents(PeerID id, RE_Matcher* pattern) { if ( ! using_communication ) @@ -1443,6 +1482,7 @@ bool RemoteSerializer::Poll(bool may_block) case MSG_PHASE_DONE: case MSG_TERMINATE: case MSG_DEBUG_DUMP: + case MSG_REQUEST_LOGS: { // No further argument chunk. msgstate = TYPE; @@ -1465,6 +1505,8 @@ bool RemoteSerializer::Poll(bool may_block) case MSG_LOG: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: { // One further argument chunk. msgstate = ARGS; @@ -1601,6 +1643,15 @@ bool RemoteSerializer::DoMessage() case MSG_REMOTE_PRINT: return ProcessRemotePrint(); + case MSG_LOG_CREATE_WRITER: + return ProcessLogCreateWriter(); + + case MSG_LOG_WRITE: + return ProcessLogWrite(); + + case MSG_REQUEST_LOGS: + return ProcessRequestLogs(); + default: DEBUG_COMM(fmt("unexpected msg type: %d", int(current_msgtype))); @@ -1663,6 +1714,7 @@ void RemoteSerializer::PeerConnected(Peer* peer) peer->cache_out->Clear(); peer->our_runtime = int(current_time(true) - bro_start_time); peer->sync_point = 0; + peer->logs_requested = false; if ( ! SendCMsgToChild(MSG_VERSION, peer) ) return; @@ -1725,6 +1777,7 @@ RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port, peer->orig = false; peer->accept_state = false; peer->send_state = false; + peer->logs_requested = false; peer->caps = 0; peer->comp_level = 0; peer->suspended_processing = false; @@ -1735,6 +1788,8 @@ RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port, peer->sync_point = 0; peer->print_buffer = 0; peer->print_buffer_used = 0; + peer->log_buffer = 0; + peer->log_buffer_used = 0; peers.append(peer); Log(LogInfo, "added peer", peer); @@ -1767,6 +1822,7 @@ void RemoteSerializer::RemovePeer(Peer* peer) int id = peer->id; Unref(peer->val); delete [] peer->print_buffer; + delete [] peer->log_buffer; delete peer->cache_in; delete peer->cache_out; delete peer; @@ -1960,6 +2016,19 @@ bool RemoteSerializer::ProcessRequestSyncMsg() return true; } +bool RemoteSerializer::ProcessRequestLogs() + { + if ( ! current_peer ) + return false; + + Log(LogInfo, "peer requested logs", current_peer); + + current_peer->logs_requested = true; + current_peer->log_buffer = new char[LOG_BUFFER_SIZE]; + current_peer->log_buffer_used = 0; + return true; + } + bool RemoteSerializer::ProcessPhaseDone() { switch ( current_peer->phase ) { @@ -2024,6 +2093,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer) if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) ) Log(LogInfo, "peer supports keep-in-cache; using that", peer); + if ( peer->logs_requested ) + log_mgr->SendAllWritersTo(peer->id); + if ( peer->sync_requested != Peer::NONE ) { if ( in_sync ) @@ -2377,6 +2449,260 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) + { + loop_over_list(peers, i) + { + SendLogCreateWriter(peers[i]->id, id, writer, path, num_fields, fields); + } + + return true; + } + +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) + { + SetErrorDescr("logging"); + + ChunkedIO::Chunk* c = 0; + + Peer* peer = LookupPeer(peer_id, true); + if ( ! peer ) + return false; + + if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) + return false; + + BinarySerializationFormat fmt; + + fmt.StartWrite(); + + bool success = fmt.Write(id->AsEnum(), "id") && + fmt.Write(writer->AsEnum(), "writer") && + fmt.Write(path, "path") && + fmt.Write(num_fields, "num_fields"); + + if ( ! success ) + goto error; + + for ( int i = 0; i < num_fields; i++ ) + { + if ( ! fields[i]->Write(&fmt) ) + goto error; + } + + c = new ChunkedIO::Chunk; + c->len = fmt.EndWrite(&c->data); + + if ( ! SendToChild(MSG_LOG_CREATE_WRITER, peer, 0) ) + goto error; + + if ( ! SendToChild(c) ) + goto error; + + return true; + +error: + FatalError(io->Error()); + return false; + } + +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) + { + loop_over_list(peers, i) + { + SendLogWrite(peers[i], id, writer, path, num_fields, vals); + } + + return true; + } + +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) + { + if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) + return false; + + if ( ! peer->logs_requested ) + return false; + + assert(peer->log_buffer); + + // Serialize the log record entry. + + BinarySerializationFormat fmt; + + fmt.StartWrite(); + + bool success = fmt.Write(id->AsEnum(), "id") && + fmt.Write(writer->AsEnum(), "writer") && + fmt.Write(path, "path") && + fmt.Write(num_fields, "num_fields"); + + if ( ! success ) + goto error; + + for ( int i = 0; i < num_fields; i++ ) + { + if ( ! vals[i]->Write(&fmt) ) + goto error; + } + + // Ok, we have the binary data now. + char* data; + int len; + + len = fmt.EndWrite(&data); + + // Do we have enough space in the buffer? If not, flush first. + if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) ) + FlushLogBuffer(peer); + + // If the data is actually larger than our complete buffer, just send it out. + if ( len > LOG_BUFFER_SIZE ) + return SendToChild(MSG_LOG_WRITE, peer, data, len); + + // Now we have space in the buffer, copy it into there. + memcpy(peer->log_buffer + peer->log_buffer_used, data, len); + peer->log_buffer_used += len; + assert(peer->log_buffer_used <= LOG_BUFFER_SIZE); + + FlushLogBuffer(peer); + return false; + +error: + FatalError(io->Error()); + return false; + } + +bool RemoteSerializer::FlushLogBuffer(Peer* p) + { + if ( p->state == Peer::CLOSING ) + return false; + + if ( ! p->log_buffer ) + return true; + + SendToChild(MSG_LOG_WRITE, p, p->log_buffer, p->log_buffer_used); + + p->log_buffer = new char[LOG_BUFFER_SIZE]; + p->log_buffer_used = 0; + return true; + } + +bool RemoteSerializer::ProcessLogCreateWriter() + { + if ( current_peer->state == Peer::CLOSING ) + return false; + + assert(current_args); + + EnumVal* id_val = 0; + EnumVal* writer_val = 0; + LogField** fields = 0; + + BinarySerializationFormat fmt; + fmt.StartRead(current_args->data, current_args->len); + + int id, writer; + string path; + int num_fields; + + bool success = fmt.Read(&id, "id") && + fmt.Read(&writer, "writer") && + fmt.Read(&path, "path") && + fmt.Read(&num_fields, "num_fields"); + + if ( ! success ) + goto error; + + fields = new LogField* [num_fields]; + + for ( int i = 0; i < num_fields; i++ ) + { + fields[i] = new LogField; + if ( ! fields[i]->Read(&fmt) ) + goto error; + } + + fmt.EndRead(); + + id_val = new EnumVal(id, BifType::Enum::Log::ID); + writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); + + if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) ) + goto error; + + Unref(id_val); + Unref(writer_val); + + return true; + +error: + Unref(id_val); + Unref(writer_val); + + Error("write error for creating writer"); + return false; + } + +bool RemoteSerializer::ProcessLogWrite() + { + if ( current_peer->state == Peer::CLOSING ) + return false; + + assert(current_args); + + BinarySerializationFormat fmt; + fmt.StartRead(current_args->data, current_args->len); + + while ( fmt.BytesRead() != (int)current_args->len ) + { + // Unserialize one entry. + EnumVal* id_val = 0; + EnumVal* writer_val = 0; + LogVal** vals = 0; + + int id, writer; + string path; + int num_fields; + + bool success = fmt.Read(&id, "id") && + fmt.Read(&writer, "writer") && + fmt.Read(&path, "path") && + fmt.Read(&num_fields, "num_fields"); + + if ( ! success ) + goto error; + + vals = new LogVal* [num_fields]; + + for ( int i = 0; i < num_fields; i++ ) + { + vals[i] = new LogVal; + if ( ! vals[i]->Read(&fmt) ) + goto error; + } + + id_val = new EnumVal(id, BifType::Enum::Log::ID); + writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); + + success = log_mgr->Write(id_val, writer_val, path, num_fields, vals); + + Unref(id_val); + Unref(writer_val); + + if ( ! success ) + goto error; + + } + + fmt.EndRead(); + + return true; + +error: + Error("write error for log entry"); + return false; + } void RemoteSerializer::GotEvent(const char* name, double time, EventHandlerPtr event, val_list* args) @@ -3048,6 +3374,7 @@ bool SocketComm::ProcessParentMessage() case MSG_TERMINATE: case MSG_PHASE_DONE: case MSG_DEBUG_DUMP: + case MSG_REQUEST_LOGS: { // No further argument chunk. parent_msgstate = TYPE; @@ -3067,6 +3394,8 @@ bool SocketComm::ProcessParentMessage() case MSG_CAPS: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: { // One further argument chunk. parent_msgstate = ARGS; @@ -3167,18 +3496,6 @@ bool SocketComm::DoParentMessage() return true; } - case MSG_PHASE_DONE: - { - // No argument block follows. - if ( parent_peer && parent_peer->connected ) - { - DEBUG_COMM("child: forwarding with MSG_PHASE_DONE to peer"); - if ( ! SendToPeer(parent_peer, MSG_PHASE_DONE, 0) ) - return false; - } - return true; - } - case MSG_LISTEN: return ProcessListen(); @@ -3206,6 +3523,20 @@ bool SocketComm::DoParentMessage() return ForwardChunkToPeer(); } + case MSG_PHASE_DONE: + case MSG_REQUEST_LOGS: + { + // No argument block follows. + if ( parent_peer && parent_peer->connected ) + { + DEBUG_COMM(fmt("child: forwarding %s to peer", msgToStr(parent_msgtype))); + if ( ! SendToPeer(parent_peer, parent_msgtype, 0) ) + return false; + } + + return true; + } + case MSG_REQUEST_EVENTS: case MSG_REQUEST_SYNC: case MSG_SERIAL: @@ -3214,6 +3545,8 @@ bool SocketComm::DoParentMessage() case MSG_CAPS: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: assert(parent_args); return ForwardChunkToPeer(); @@ -3332,6 +3665,7 @@ bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) switch ( msg->Type() ) { case MSG_PHASE_DONE: + case MSG_REQUEST_LOGS: // No further argument block. DEBUG_COMM("child: forwarding with 0 args to parent"); if ( ! SendToParent(msg->Type(), peer, 0) ) @@ -3388,6 +3722,8 @@ bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) case MSG_CAPS: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: { // Messages with one further argument block which we simply // forward to our parent. diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 6afec4ec6f..20886544c6 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -16,6 +16,8 @@ // FIXME: Change this to network byte order class IncrementalSendTimer; +class LogField; +class LogVal; // This class handles the communication done in Bro's main loop. class RemoteSerializer : public Serializer, public IOSource { @@ -42,6 +44,9 @@ public: // the peer right after the handshake. bool RequestSync(PeerID peer, bool auth); + // Requests logs from the remote side. + bool RequestLogs(PeerID id); + // Sets flag whether we're accepting state from this peer // (default: yes). bool SetAcceptState(PeerID peer, bool accept); @@ -92,6 +97,15 @@ public: // Broadcast remote print. bool SendPrintHookEvent(BroFile* f, const char* txt); + // Send a request to create a writer on a remote side. + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + + // Broadcasts a request to create a writer. + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + + // Broadcast a log entry to everybody interested. + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. uint32 SendSyncPoint(); @@ -205,6 +219,7 @@ protected: bool accept_state; // True if we accept state from peer. bool send_state; // True if we're supposed to initially sent our state. int comp_level; // Compression level. + bool logs_requested; // True if the peer has requested logs. // True if this peer triggered a net_suspend_processing(). bool suspended_processing; @@ -217,6 +232,8 @@ protected: uint32 sync_point; // Highest sync-point received so far char* print_buffer; // Buffer for remote print or null. int print_buffer_used; // Number of bytes used in buffer. + char* log_buffer; // Buffer for remote log or null. + int log_buffer_used; // Number of bytes used in buffer. }; // Shuts down remote serializer. @@ -255,6 +272,9 @@ protected: bool ProcessCapsMsg(); bool ProcessSyncPointMsg(); bool ProcessRemotePrint(); + bool ProcessLogCreateWriter(); + bool ProcessLogWrite(); + bool ProcessRequestLogs(); Peer* AddPeer(uint32 ip, uint16 port, PeerID id = PEER_NONE); Peer* LookupPeer(PeerID id, bool only_if_connected); @@ -282,11 +302,13 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); bool EnterPhaseRunning(Peer* peer); bool FlushPrintBuffer(Peer* p); + bool FlushLogBuffer(Peer* p); void ChildDied(); void InternalCommError(const char* msg); diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 55e35eb30e..c181107870 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -44,6 +44,7 @@ void SerializationFormat::StartWrite() output_pos = 0; bytes_written = 0; + bytes_read = 0; } uint32 SerializationFormat::EndWrite(char** data) @@ -64,6 +65,8 @@ bool SerializationFormat::ReadData(void* b, size_t count) memcpy(b, input + input_pos, count); input_pos += count; + bytes_read += count; + return true; } @@ -214,6 +217,20 @@ bool BinarySerializationFormat::Read(char** str, int* len, const char* tag) return true; } +bool BinarySerializationFormat::Read(string* v, const char* tag) + { + char* buffer; + int len; + + if ( ! Read(&buffer, &len, tag) ) + return false; + + *v = string(buffer, len); + + delete [] buffer; + return true; + } + bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -278,6 +295,11 @@ bool BinarySerializationFormat::Write(const char* s, const char* tag) return Write(s, strlen(s), tag); } +bool BinarySerializationFormat::Write(const string& s, const char* tag) + { + return Write(s.data(), s.size(), tag); + } + bool BinarySerializationFormat::WriteOpenTag(const char* tag) { return true; @@ -362,6 +384,12 @@ bool XMLSerializationFormat::Read(char** str, int* len, const char* tag) return false; } +bool XMLSerializationFormat::Read(string* s, const char* tag) + { + internal_error("no reading of xml"); + return false; + } + bool XMLSerializationFormat::Write(char v, const char* tag) { return WriteElem(tag, "char", &v, 1); @@ -416,6 +444,11 @@ bool XMLSerializationFormat::Write(const char* s, const char* tag) return WriteElem(tag, "string", s, strlen(s)); } +bool XMLSerializationFormat::Write(const string& s, const char* tag) + { + return WriteElem(tag, "string", s.data(), s.size()); + } + bool XMLSerializationFormat::WriteOpenTag(const char* tag) { return WriteData("<", 1) && WriteData(tag, strlen(tag) && WriteData(">", 1)); diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index 8127343274..b9c7ec1549 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -5,6 +5,10 @@ #ifndef SERIALIZATION_FORMAT #define SERIALIZATION_FORMAT +#include + +using namespace std; + #include "util.h" // Abstract base class. @@ -25,6 +29,10 @@ public: virtual bool Read(char* v, const char* tag) = 0; virtual bool Read(bool* v, const char* tag) = 0; virtual bool Read(double* d, const char* tag) = 0; + virtual bool Read(string* s, const char* tag) = 0; + + // Returns number of raw bytes read since last call to StartRead(). + int BytesRead() const { return bytes_read; } // Passes ownership of string. virtual bool Read(char** str, int* len, const char* tag) = 0; @@ -43,6 +51,7 @@ public: virtual bool Write(double d, const char* tag) = 0; virtual bool Write(const char* s, const char* tag) = 0; virtual bool Write(const char* buf, int len, const char* tag) = 0; + virtual bool Write(const string& s, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -65,6 +74,7 @@ protected: uint32 input_pos; int bytes_written; + int bytes_read; }; class BinarySerializationFormat : public SerializationFormat { @@ -81,6 +91,7 @@ public: virtual bool Read(bool* v, const char* tag); virtual bool Read(double* d, const char* tag); virtual bool Read(char** str, int* len, const char* tag); + virtual bool Read(string* s, const char* tag); virtual bool Write(int v, const char* tag); virtual bool Write(uint16 v, const char* tag); virtual bool Write(uint32 v, const char* tag); @@ -91,6 +102,7 @@ public: virtual bool Write(double d, const char* tag); virtual bool Write(const char* s, const char* tag); virtual bool Write(const char* buf, int len, const char* tag); + virtual bool Write(const string& s, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -112,6 +124,7 @@ public: virtual bool Write(double d, const char* tag); virtual bool Write(const char* s, const char* tag); virtual bool Write(const char* buf, int len, const char* tag); + virtual bool Write(const string& s, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -126,6 +139,7 @@ public: virtual bool Read(bool* v, const char* tag); virtual bool Read(double* d, const char* tag); virtual bool Read(char** str, int* len, const char* tag); + virtual bool Read(string* s, const char* tag); private: // Encodes non-printable characters. diff --git a/src/bro.bif b/src/bro.bif index 054fafec0f..b9140befb3 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -2108,6 +2108,12 @@ function request_remote_sync%(p: event_peer, auth: bool%) : bool return new Val(remote_serializer->RequestSync(id, auth), TYPE_BOOL); %} +function request_remote_logs%(p: event_peer%) : bool + %{ + RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); + return new Val(remote_serializer->RequestLogs(id), TYPE_BOOL); + %} + function set_accept_state%(p: event_peer, accept: bool%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); From 90af0d06c3de782826067c963e9270e62bf9d067 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 3 Mar 2011 21:47:08 -0800 Subject: [PATCH 34/67] A first test for remote logging. Needs the updated btest, which has two new scripts for running processes in the background. --- .../logging.remote/sender.ssh.failure.log | 4 + .../Baseline/logging.remote/sender.ssh.log | 6 ++ .../logging.remote/sender.ssh.success.log | 3 + testing/btest/logging/remote.bro | 79 +++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 testing/btest/Baseline/logging.remote/sender.ssh.failure.log create mode 100644 testing/btest/Baseline/logging.remote/sender.ssh.log create mode 100644 testing/btest/Baseline/logging.remote/sender.ssh.success.log create mode 100644 testing/btest/logging/remote.bro diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log new file mode 100644 index 0000000000..fe087414a1 --- /dev/null +++ b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log @@ -0,0 +1,4 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure US +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure UK +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.log b/testing/btest/Baseline/logging.remote/sender.ssh.log new file mode 100644 index 0000000000..2971ac2fc7 --- /dev/null +++ b/testing/btest/Baseline/logging.remote/sender.ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success - +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure US +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure UK +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success BR +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.success.log b/testing/btest/Baseline/logging.remote/sender.ssh.success.log new file mode 100644 index 0000000000..10156943a3 --- /dev/null +++ b/testing/btest/Baseline/logging.remote/sender.ssh.success.log @@ -0,0 +1,3 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success - +1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success BR diff --git a/testing/btest/logging/remote.bro b/testing/btest/logging/remote.bro new file mode 100644 index 0000000000..18fb7e4f2e --- /dev/null +++ b/testing/btest/logging/remote.bro @@ -0,0 +1,79 @@ +# +# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-diff sender/ssh.log +# @TEST-EXEC: btest-diff sender/ssh.failure.log +# @TEST-EXEC: btest-diff sender/ssh.success.log +# @TEST-EXEC: cmp receiver/ssh.log sender/ssh.log +# @TEST-EXEC: cmp receiver/ssh.failure.log sender/ssh.failure.log +# @TEST-EXEC: cmp receiver/ssh.success.log sender/ssh.success.log + +# This is the common part loaded by both sender and receiver. +module SSH; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global log_ssh: event(rec: Log); + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); + Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); +} + +##### + +@TEST-START-FILE sender.bro + +module SSH; + +@load listen-clear + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event remote_connection_handshake_done(p: event_peer) + { + Log::add_filter(SSH, [$name="f2", $path="ssh.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + + # Log something. + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +@load remote + +redef Remote::destinations += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE From d6cef16f77778dce83f27e9a0dea279ac9e47fbf Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 6 Mar 2011 19:28:48 -0800 Subject: [PATCH 35/67] Rotation support. This follows rather closely how rotation currently works in rotate-logs.bro. logging.bro now defines: # Default rotation interval; zero disables rotation. const default_rotation_interval = 0secs &redef; # Default naming suffix format. const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; # Default postprocessor for writers outputting into files. const default_rotation_postprocessor = "" &redef; # Default function to construct the name of the rotated file. # The default implementation includes # default_rotation_date_format into the file name. global default_rotation_path_func: function(info: RotationInfo) : string &redef; Writer support for rotation is optional, usually it will only make sense for file-based writers. TODO: Currently, there's no way to customize rotation on a per file basis, there are only the global defaults as described above. Individual customization is coming next. --- policy/bro.init | 4 +- policy/logging.bro | 34 ++++- src/LogMgr.cc | 166 ++++++++++++++++++++-- src/LogMgr.h | 5 + src/LogWriter.cc | 54 ++++++- src/LogWriter.h | 38 ++--- src/LogWriterAscii.cc | 34 +++-- src/LogWriterAscii.h | 8 +- src/Scope.cc | 5 +- src/Scope.h | 2 +- src/logging.bif | 4 + src/parse.y | 18 ++- testing/btest/Baseline/logging.rotate/out | 50 +++++++ testing/btest/btest.cfg | 2 +- testing/btest/logging/rotate.bro | 31 ++++ testing/btest/logging/rotation.trace | Bin 0 -> 1144 bytes 16 files changed, 387 insertions(+), 68 deletions(-) create mode 100644 testing/btest/Baseline/logging.rotate/out create mode 100644 testing/btest/logging/rotate.bro create mode 100644 testing/btest/logging/rotation.trace diff --git a/policy/bro.init b/policy/bro.init index 3a107d865c..84f0094a71 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -273,12 +273,12 @@ type entropy_test_result: record { serial_correlation: double; }; -@load logging # sic! Not logging.bif. - # Prototypes of Bro built-in functions. @load strings.bif.bro @load bro.bif.bro +@load logging # sic! Not logging.bif. + global bro_alarm_file: file &redef; global alarm_hook: function(msg: string): bool &redef; global log_file_name: function(tag: string): string &redef; diff --git a/policy/logging.bro b/policy/logging.bro index c5613e4ada..f3d4e35c37 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -3,8 +3,12 @@ module Log; # Log::ID and Log::Writer are defined in bro.init due to circular dependencies. export { - # The default writer to use. - const default_writer = Log::WRITER_ASCII &redef; + # Information passed to a rotation callback function. + type RotationInfo: record { + path: string; # Original path value. + open: time; # Time when opened. + close: time; # Time when closed. + }; # If true, local logging is by default enabled for all filters. const enable_local_logging = T &redef; @@ -12,6 +16,23 @@ export { # If true, remote logging is by default enabled for all filters. const enable_remote_logging = T &redef; + # The default writer to use. + const default_writer = Log::WRITER_ASCII &redef; + + # Default rotation interval; zero disables rotation. + const default_rotation_interval = 0secs &redef; + + # Default naming suffix format. + const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; + + # Default postprocessor for writers outputting into files. + const default_rotation_postprocessor = "" &redef; + + # Default function to construct the name of the rotated file. + # The default implementation includes + # default_rotation_date_format into the file name. + global default_rotation_path_func: function(info: RotationInfo) : string &redef; + # A stream defining the logging. type Stream: record { # A record type defining the log's columns. @@ -51,10 +72,10 @@ export { exclude: set[string] &optional; # If true, record all log records locally. - log_local: bool &default=Log::enable_local_logging; + log_local: bool &default=enable_local_logging; # If true, pass all log records on to remote peers if they request it. - log_remote: bool &default=Log::enable_remote_logging; + log_remote: bool &default=enable_remote_logging; # The writer to use. writer: Writer &default=Log::default_writer; @@ -74,6 +95,11 @@ export { module Log; +function default_rotation_path_func(info: RotationInfo) : string + { + return fmt("%s-%s", info$path, strftime(default_rotation_date_format, info$open)); + } + function create_stream(id: Log::ID, stream: Log::Stream) : bool { if ( ! Log::__create_stream(id, stream) ) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 15db1cb927..46293e71fe 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -3,6 +3,7 @@ #include "Event.h" #include "EventHandler.h" #include "NetVar.h" +#include "Net.h" #include "LogWriterAscii.h" @@ -22,6 +23,7 @@ LogWriterDefinition log_writers[] = { struct LogMgr::Filter { string name; + EnumVal* id; Func* pred; Func* path_func; string path; @@ -37,6 +39,12 @@ struct LogMgr::Filter { ~Filter(); }; +struct LogMgr::WriterInfo { + double open_time; + Timer* rotation_timer; + LogWriter *writer; + }; + struct LogMgr::Stream { EnumVal* id; string name; @@ -45,7 +53,9 @@ struct LogMgr::Stream { list filters; typedef pair WriterPathPair; - typedef map WriterMap; + + typedef map WriterMap; + WriterMap writers; // Writers indexed by id/path pair. ~Stream(); @@ -213,7 +223,15 @@ LogMgr::Stream::~Stream() Unref(columns); for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) + { + WriterInfo* winfo = i->second; + + if ( winfo->rotation_timer ) + timer_mgr->Cancel(winfo->rotation_timer); + + delete winfo->writer; delete i->second; + } for ( list::iterator f = filters.begin(); f != filters.end(); ++f ) delete *f; @@ -248,7 +266,7 @@ void LogMgr::RemoveDisabledWriters(Stream* stream) for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) { - if ( j->second->Disabled() ) + if ( j->second->writer->Disabled() ) { delete j->second; disabled.push_back(j->first); @@ -413,16 +431,26 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) // Create a new Filter instance. - Val* pred = fval->Lookup(rtype->FieldOffset("pred")); - Val* path_func = fval->Lookup(rtype->FieldOffset("path_func")); + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); + Val* path_func = fval->LookupWithDefault(rtype->FieldOffset("path_func")); + Val* log_local = fval->LookupWithDefault(rtype->FieldOffset("log_local")); + Val* log_remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote")); Filter* filter = new Filter; - filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString(); + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); filter->pred = pred ? pred->AsFunc() : 0; filter->path_func = path_func ? path_func->AsFunc() : 0; filter->writer = writer->Ref()->AsEnumVal(); - filter->local = fval->LookupWithDefault(rtype->FieldOffset("log_local"))->AsBool(); - filter->remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote"))->AsBool(); + filter->local = log_local->AsBool(); + filter->remote = log_remote->AsBool(); + + Unref(name); + Unref(pred); + Unref(path_func); + Unref(log_local); + Unref(log_remote); // TODO: Check that the predciate is of the right type. @@ -580,7 +608,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( w != stream->writers.end() ) // We have a writer already. - writer = w->second; + writer = w->second->writer; else { // No, need to create one. @@ -669,10 +697,13 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) case TYPE_COUNT: case TYPE_COUNTER: - case TYPE_PORT: vals[i]->val.uint_val = val->InternalUnsigned(); break; + case TYPE_PORT: + vals[i]->val.uint_val = val->AsPortVal()->Port(); + break; + case TYPE_SUBNET: vals[i]->val.subnet_val = *val->AsSubNet(); break; @@ -719,7 +750,7 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n if ( w != stream->writers.end() ) // If we already have a writer for this. That's fine, we just return // it. - return w->second; + return w->second->writer; // Need to instantiate a new writer. @@ -766,7 +797,13 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n return 0; } - stream->writers.insert(Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), writer_obj)); + WriterInfo* winfo = new WriterInfo; + winfo->writer = writer_obj; + winfo->open_time = network_time; + winfo->rotation_timer = 0; + InstallRotationTimer(winfo); + + stream->writers.insert(Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), winfo)); return writer_obj; } @@ -799,7 +836,7 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, Lo return false; } - bool success = w->second->Write(num_fields, vals); + bool success = w->second->writer->Write(num_fields, vals); DBG_LOG(DBG_LOGGING, "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); @@ -817,7 +854,7 @@ void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) { - LogWriter* writer = i->second; + LogWriter* writer = i->second->writer; EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); remote_serializer->SendLogCreateWriter(peer, (*s)->id, &writer_val, i->first.second, writer->NumFields(), writer->Fields()); } @@ -831,7 +868,7 @@ bool LogMgr::SetBuf(EnumVal* id, bool enabled) return false; for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - i->second->SetBuf(enabled); + i->second->writer->SetBuf(enabled); RemoveDisabledWriters(stream); @@ -845,7 +882,7 @@ bool LogMgr::Flush(EnumVal* id) return false; for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) - i->second->Flush(); + i->second->writer->Flush(); RemoveDisabledWriters(stream); @@ -857,3 +894,102 @@ void LogMgr::Error(LogWriter* writer, const char* msg) run_time(fmt("error with writer for %s: %s", writer->Path().c_str(), msg)); } +// Timer which on dispatching rotates the filter. +class RotationTimer : public Timer { +public: + RotationTimer(double t, LogMgr::WriterInfo* arg_winfo, bool arg_rotate) + : Timer(t, TIMER_ROTATE) { winfo = arg_winfo; rotate = arg_rotate; } + ~RotationTimer(); + + void Dispatch(double t, int is_expire); + +protected: + LogMgr::WriterInfo* winfo; + bool rotate; +}; + +RotationTimer::~RotationTimer() + { + if ( winfo->rotation_timer == this ) + winfo->rotation_timer = 0; + } + +void RotationTimer::Dispatch(double t, int is_expire) + { + winfo->rotation_timer = 0; + + if ( rotate ) + log_mgr->Rotate(winfo); + + if ( ! is_expire ) + { + winfo->open_time = network_time; + log_mgr->InstallRotationTimer(winfo); + } + } + +void LogMgr::InstallRotationTimer(WriterInfo* winfo) + { + if ( terminating ) + return; + + if ( winfo->rotation_timer ) + { + timer_mgr->Cancel(winfo->rotation_timer); + winfo->rotation_timer = 0; + } + + double rotation_interval = BifConst::Log::default_rotation_interval; + + if ( rotation_interval ) + { + // When this is called for the first time, network_time can still be + // zero. If so, we set a timer which fires immediately but doesn't + // rotate when it expires. + if ( ! network_time ) + winfo->rotation_timer = new RotationTimer(1, winfo, false); + else + { + if ( ! winfo->open_time ) + winfo->open_time = network_time; + + const char* base_time = log_rotate_base_time ? + log_rotate_base_time->AsString()->CheckString() : 0; + + double delta_t = + calc_next_rotate(rotation_interval, base_time); + + winfo->rotation_timer = new RotationTimer(network_time + delta_t, winfo, true); + } + + timer_mgr->Add(winfo->rotation_timer); + + DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f", winfo->writer->Path().c_str(), winfo->rotation_timer->Time()); + } + } + +void LogMgr::Rotate(WriterInfo* winfo) + { + DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f", winfo->writer->Path().c_str(), network_time); + + // Create the RotationInfo record. + RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo); + info->Assign(0, new StringVal(winfo->writer->Path().c_str())); + info->Assign(1, new Val(winfo->open_time, TYPE_TIME)); + info->Assign(2, new Val(network_time, TYPE_TIME)); + + // Call the function building us the new path. + + Func* rotation_path_func = internal_func("Log::default_rotation_path_func"); + string rotation_postprocessor = BifConst::Log::default_rotation_postprocessor->AsString()->CheckString(); + + val_list vl(1); + vl.append(info); + Val* result = rotation_path_func->Call(&vl); + string new_path = result->AsString()->CheckString(); + Unref(result); + + winfo->writer->Rotate(new_path, rotation_postprocessor, winfo->open_time, network_time, terminating); + } + + diff --git a/src/LogMgr.h b/src/LogMgr.h index 069ddfdff2..1188b80336 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -57,6 +57,7 @@ private: class LogWriter; class RemoteSerializer; +class RotationTimer; class LogMgr { public: @@ -75,6 +76,7 @@ public: protected: friend class LogWriter; friend class RemoteSerializer; + friend class RotationTimer; // These function are also used by the RemoteSerializer. LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, LogField** fields); // takes ownership of fields. @@ -89,11 +91,14 @@ protected: private: struct Filter; struct Stream; + struct WriterInfo; bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); + void InstallRotationTimer(WriterInfo* winfo); + void Rotate(WriterInfo* info); vector streams; // Indexed by stream enum. }; diff --git a/src/LogWriter.cc b/src/LogWriter.cc index 6c27bb55b5..2298004185 100644 --- a/src/LogWriter.cc +++ b/src/LogWriter.cc @@ -18,7 +18,7 @@ LogWriter::~LogWriter() delete [] fields; } -bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields) +bool LogWriter::Init(string arg_path, int arg_num_fields, const LogField* const * arg_fields) { path = arg_path; num_fields = arg_num_fields; @@ -75,6 +75,17 @@ bool LogWriter::SetBuf(bool enabled) return true; } +bool LogWriter::Rotate(string rotated_path, string postprocessor, double open, double close, bool terminating) + { + if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) ) + { + disabled = true; + return false; + } + + return true; + } + bool LogWriter::Flush() { if ( ! DoFlush() ) @@ -126,4 +137,45 @@ void LogWriter::DeleteVals(LogVal** vals) delete vals[i]; } +bool LogWriter::RunPostProcessor(string fname, string postprocessor, string old_name, double open, double close, bool terminating) + { + // This function operates in way backwards-compatible with the old Bro + // log rotation scheme. + + if ( ! postprocessor.size() ) + return true; + + const char* const fmt = "%y-%m-%d_%H.%M.%S"; + + struct tm tm1; + struct tm tm2; + + time_t tt1 = (time_t)open; + time_t tt2 = (time_t)close; + + localtime_r(&tt1, &tm1); + localtime_r(&tt2, &tm2); + + char buf1[128]; + char buf2[128]; + + strftime(buf1, sizeof(buf1), fmt, &tm1); + strftime(buf2, sizeof(buf2), fmt, &tm2); + + string cmd = postprocessor; + cmd += " " + fname; + cmd += " " + old_name; + cmd += " " + string(buf1); + cmd += " " + string(buf2); + cmd += " " + string(terminating ? "1" : "0"); + cmd += " &"; + + system(cmd.c_str()); + + return true; + } + + + + diff --git a/src/LogWriter.h b/src/LogWriter.h index 0dd37d2881..5a89e6be9e 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -23,7 +23,7 @@ public: // // The new instance takes ownership of "fields", and will delete them // when done. - bool Init(string path, int num_fields, LogField** fields); + bool Init(string path, int num_fields, const LogField* const * fields); // Writes one log entry. The method takes ownership of "vals" and will // return immediately after queueing the write request, potentially @@ -35,14 +35,19 @@ public: // Sets the buffering status for the writer, if the writer supports it. bool SetBuf(bool enabled); - // Flushes any currently buffered output, if the writer support it. + // Flushes any currently buffered output, if the writer supports it. bool Flush(); + // Triggers rotation, if the writer supports it. + bool Rotate(string rotated_path, string postprocessor, double open, double close, bool terminating); + // Finished writing to this logger. Will not be called if an error has // been indicated earlier. After calling this, no more writing must be // performed. void Finish(); + // Returns the path as passed to Init(). + const string Path() const { return path; } int NumFields() const { return num_fields; } const LogField* const * Fields() const { return fields; } @@ -57,10 +62,10 @@ protected: // always return true. // Called once for initialization of the Writer. - virtual bool DoInit(string path, int num_fields, LogField** fields) = 0; + virtual bool DoInit(string path, int num_fields, const LogField* const * fields) = 0; // Called once per entry to record. - virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0; + virtual bool DoWrite(int num_fields, const LogField* const * fields, LogVal** vals) = 0; // Called when the buffering status for this writer is changed. If // buffering is disabled, the writer should attempt to write out @@ -80,25 +85,24 @@ protected: virtual bool DoFlush() = 0; // Called when a log output is to be rotated. Most directly, this only - // applies to writers outputting files, thoug a writer may also trigger - // other regular actions if that fits a similar model + // applies to writers outputting files, though a writer may also trigger + // other regular actions if semantics are similar. // // The string "rotate_path" is interpreted in writer-specific way, yet - // should generally should have similar semantics that the "path" passed + // should generally should have similar semantics as the "path" passed // into DoInit(), except that now it reflects the name to where the // rotated output is to be moved. After rotation, output should continue // normally with the standard "path". As an example, for file-based // output, the rotate_path may be the original filename with an embedded - // timestamp. - // - // The writer must return false if an error occured that prevent the - // writer for continuing operation; it will then be disabled. The error - // reason should be reported via Error(). If a recoverable error occurs, - // still call Error(), but return true. + // timestamp. "postprocessor" is the name of a command to execute on the + // rotated file. If empty, no such processing should take place; if given + // but the writer doesn't support postprocessing, it can be ignored. + // "open" and "close" are the network time's at opening and closeing the + // current file, respetively. // // A writer may ignore rotation requests if it doesn't fit with its // semantics. - virtual bool DoRotate(string rotated_path) = 0; + virtual bool DoRotate(string rotated_path, string postprocessor, double open, double close, bool terminating) = 0; // Called once on termination. Not called when any of the other methods // has previously signaled an error, i.e., executing this method signals @@ -116,8 +120,8 @@ protected: // Reports an error. void Error(const char *msg); - // Returns the path as passed to Init(). - const string Path() const { return path; } + // Runs a post-processor on the given file. + bool RunPostProcessor(string fname, string postprocessor, string old_name, double open, double close, bool terminating); private: friend class LogMgr; @@ -131,7 +135,7 @@ private: string path; int num_fields; - LogField** fields; + const LogField* const * fields; bool buffering; bool disabled; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 902cbf7ea2..6ba3f812f7 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -6,26 +6,22 @@ LogWriterAscii::LogWriterAscii() { - fname = 0; file = 0; } LogWriterAscii::~LogWriterAscii() { - if ( fname ) - free(fname); - if ( file ) fclose(file); } -bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) +bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * fields) { - fname = strdup(Fmt("%s.log", path.c_str())); + fname = path + ".log"; - if ( ! (file = fopen(fname, "w")) ) + if ( ! (file = fopen(fname.c_str(), "w")) ) { - Error(Fmt("cannot open %s: %s", fname, strerror(errno))); + Error(Fmt("cannot open %s: %s", fname.c_str(), strerror(errno))); return false; } @@ -34,7 +30,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) for ( int i = 0; i < num_fields; i++ ) { - LogField* field = fields[i]; + const LogField* field = fields[i]; if ( fputs(field->name.c_str(), file) == EOF ) goto write_error; @@ -48,7 +44,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields) return true; write_error: - Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno))); return false; } @@ -62,7 +58,7 @@ void LogWriterAscii::DoFinish() { } -bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) +bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, LogVal** vals) { ODesc desc(DESC_READABLE); @@ -72,7 +68,7 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) desc.Add("\t"); LogVal* val = vals[i]; - LogField* field = fields[i]; + const LogField* field = fields[i]; if ( ! val->present ) { @@ -127,7 +123,7 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 ) { - Error(Fmt("error writing to %s: %s", fname, strerror(errno))); + Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno))); return false; } @@ -137,9 +133,17 @@ bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals) return true; } -bool LogWriterAscii::DoRotate(string rotated_path) +bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open, double close, bool terminating) { - return true; + fclose(file); + + string nname = rotated_path + ".log"; + rename(fname.c_str(), nname.c_str()); + + if ( postprocessor.size() && ! RunPostProcessor(nname, postprocessor, fname.c_str(), open, close, terminating) ) + return false; + + return DoInit(Path(), NumFields(), Fields()); } bool LogWriterAscii::DoSetBuf(bool enabled) diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index e4501aa5b7..9732242fba 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -15,16 +15,16 @@ public: static LogWriter* Instantiate() { return new LogWriterAscii; } protected: - virtual bool DoInit(string path, int num_fields, LogField** fields); - virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals); + virtual bool DoInit(string path, int num_fields, const LogField* const * fields); + virtual bool DoWrite(int num_fields, const LogField* const * fields, LogVal** vals); virtual bool DoSetBuf(bool enabled); - virtual bool DoRotate(string rotated_path); + virtual bool DoRotate(string rotated_path, string postprocessr, double open, double close, bool terminating); virtual bool DoFlush(); virtual void DoFinish(); private: FILE* file; - char* fname; + string fname; }; #endif diff --git a/src/Scope.cc b/src/Scope.cc index 64cf61080f..eda38d9c80 100644 --- a/src/Scope.cc +++ b/src/Scope.cc @@ -113,9 +113,10 @@ TraversalCode Scope::Traverse(TraversalCallback* cb) const } -ID* lookup_ID(const char* name, const char* curr_module, bool no_global) +ID* lookup_ID(const char* name, const char* curr_module, bool no_global, bool same_module_only) { string fullname = make_full_var_name(curr_module, name); + string ID_module = extract_module_name(fullname.c_str()); bool need_export = ID_module != GLOBAL_MODULE_NAME && ID_module != curr_module; @@ -134,7 +135,7 @@ ID* lookup_ID(const char* name, const char* curr_module, bool no_global) } } - if ( ! no_global ) + if ( ! no_global && (strcmp(GLOBAL_MODULE_NAME, curr_module) == 0 || ! same_module_only) ) { string globalname = make_full_var_name(GLOBAL_MODULE_NAME, name); ID* id = global_scope()->Lookup(globalname.c_str()); diff --git a/src/Scope.h b/src/Scope.h index ca86a5ec1f..d4b7ba9ac9 100644 --- a/src/Scope.h +++ b/src/Scope.h @@ -65,7 +65,7 @@ extern bool in_debug; // If no_global is true, don't search in the default "global" namespace. extern ID* lookup_ID(const char* name, const char* module, - bool no_global = false); + bool no_global = false, bool same_module_only=false); extern ID* install_ID(const char* name, const char* module_name, bool is_global, bool is_export); diff --git a/src/logging.bif b/src/logging.bif index e31c56276e..a561c6c5a3 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -8,6 +8,10 @@ module Log; type Filter: record; type Stream: record; +type RotationInfo: record; + +const Log::default_rotation_interval: interval; +const Log::default_rotation_postprocessor: string; function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool %{ diff --git a/src/parse.y b/src/parse.y index a982935b2b..4f81347cde 100644 --- a/src/parse.y +++ b/src/parse.y @@ -42,7 +42,7 @@ %left '$' '[' ']' '(' ')' TOK_HAS_FIELD TOK_HAS_ATTR %type TOK_ID TOK_PATTERN_TEXT single_pattern -%type local_id global_id event_id global_or_event_id resolve_id begin_func +%type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class %type opt_init @@ -102,6 +102,7 @@ Expr* bro_this = 0; int in_init = 0; bool in_debug = false; bool resolving_global_ID = false; +bool defining_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; @@ -842,10 +843,10 @@ decl: | TOK_EXPORT '{' { is_export = true; } decl_list '}' { is_export = false; } - | TOK_GLOBAL global_id opt_type init_class opt_init opt_attr ';' + | TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REGULAR); } - | TOK_CONST global_id opt_type init_class opt_init opt_attr ';' + | TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_CONST); } | TOK_REDEF global_id opt_type init_class opt_init opt_attr ';' @@ -855,7 +856,7 @@ decl: '{' { parser_redef_enum($3); } enum_body '}' ';' { /* no action */ } - | TOK_TYPE global_id ':' refined_type opt_attr ';' + | TOK_TYPE def_global_id ':' refined_type opt_attr ';' { add_type($2, $4, $5, 0); } @@ -883,7 +884,7 @@ conditional: ; func_hdr: - TOK_FUNCTION global_id func_params + TOK_FUNCTION def_global_id func_params { begin_func($2, current_module.c_str(), FUNC_FLAVOR_FUNCTION, 0, $3); @@ -1265,6 +1266,11 @@ global_id: { $$ = $2; } ; +def_global_id: + { defining_global_ID = 1; } global_id { defining_global_ID = 0; } + { $$ = $2; } + ; + event_id: { resolving_global_ID = 0; } global_or_event_id { $$ = $2; } @@ -1275,7 +1281,7 @@ global_or_event_id: { set_location(@1); - $$ = lookup_ID($1, current_module.c_str(), false); + $$ = lookup_ID($1, current_module.c_str(), false, defining_global_ID); if ( $$ ) { if ( ! $$->IsGlobal() ) diff --git a/testing/btest/Baseline/logging.rotate/out b/testing/btest/Baseline/logging.rotate/out new file mode 100644 index 0000000000..27bd5179cf --- /dev/null +++ b/testing/btest/Baseline/logging.rotate/out @@ -0,0 +1,50 @@ +test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0 +test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0 +test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0 +test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0 +test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0 +test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0 +test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0 +test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0 +test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0 +test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 +> test-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +1299502795.0 10.0.0.2 20 10.0.0.3 9 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 1e702ce344..a49fc6af02 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -4,7 +4,7 @@ TestDirs = logging TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* +IgnoreFiles = *.tmp *.swp #* *.trace [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/logging/rotate.bro b/testing/btest/logging/rotate.bro new file mode 100644 index 0000000000..96d5068db7 --- /dev/null +++ b/testing/btest/logging/rotate.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out +# @TEST-EXEC: for i in test-*.log; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: btest-diff out + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { Test }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + }; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor = "echo"; + +event bro_init() +{ + Log::create_stream(Test, [$columns=Log]); +} + +event new_connection(c: connection) + { + Log::write(Test, [$t=network_time(), $id=c$id]); + } diff --git a/testing/btest/logging/rotation.trace b/testing/btest/logging/rotation.trace new file mode 100644 index 0000000000000000000000000000000000000000..9954b22e1597624b5f9d944b219e77092d141cb7 GIT binary patch literal 1144 zcma))yGnyW5Jpe_XuQ;U083xM$|g-3MX(VBYtz_9QcNdU+1OYKf`ye!)7n#bDX@vPRvsxR>&3S#F@;Oe1a2Ipdx9EJD^I@FwIZmEn6m! Date: Tue, 8 Mar 2011 16:26:31 -0800 Subject: [PATCH 36/67] Updating test baseline after fixing an older bug in previous commit. --- testing/btest/Baseline/logging.path-func/output | 14 +++++++------- .../btest/Baseline/logging.pred/ssh.failure.log | 2 +- .../btest/Baseline/logging.pred/ssh.success.log | 2 +- .../Baseline/logging.remote/sender.ssh.failure.log | 6 +++--- .../btest/Baseline/logging.remote/sender.ssh.log | 10 +++++----- .../Baseline/logging.remote/sender.ssh.success.log | 4 ++-- .../btest/Baseline/logging.remove/ssh.failure.log | 4 ++-- testing/btest/Baseline/logging.remove/ssh.log | 6 +++--- .../btest/Baseline/logging.test-logging/ssh.log | 10 +++++----- 9 files changed, 29 insertions(+), 29 deletions(-) diff --git a/testing/btest/Baseline/logging.path-func/output b/testing/btest/Baseline/logging.path-func/output index ec12bfbdab..2b44fb2948 100644 --- a/testing/btest/Baseline/logging.path-func/output +++ b/testing/btest/Baseline/logging.path-func/output @@ -2,12 +2,12 @@ static-prefix-0.log static-prefix-1.log static-prefix-2.log # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 success unknown -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 success BR -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure MX3 +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 success unknown +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 success BR +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure MX3 # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure US -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure MX +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure US +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure MX # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure UK -1298337158.29096 1.2.3.4 66770 2.3.4.5 65616 failure MX2 +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure UK +1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure MX2 diff --git a/testing/btest/Baseline/logging.pred/ssh.failure.log b/testing/btest/Baseline/logging.pred/ssh.failure.log index f90e2ad600..a5dcba878c 100644 --- a/testing/btest/Baseline/logging.pred/ssh.failure.log +++ b/testing/btest/Baseline/logging.pred/ssh.failure.log @@ -1,2 +1,2 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298327302.5778 1.2.3.4 66770 2.3.4.5 65616 failure US +1299630368.46042 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/logging.pred/ssh.success.log b/testing/btest/Baseline/logging.pred/ssh.success.log index e137961d3d..1402f6f06d 100644 --- a/testing/btest/Baseline/logging.pred/ssh.success.log +++ b/testing/btest/Baseline/logging.pred/ssh.success.log @@ -1,2 +1,2 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298327302.5778 1.2.3.4 66770 2.3.4.5 65616 success - +1299630368.46042 1.2.3.4 1234 2.3.4.5 80 success - diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log index fe087414a1..1a9b7f64b1 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log @@ -1,4 +1,4 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure US -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure UK -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure MX +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure US +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure UK +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.log b/testing/btest/Baseline/logging.remote/sender.ssh.log index 2971ac2fc7..f780e80992 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.log @@ -1,6 +1,6 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success - -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure US -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure UK -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success BR -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 failure MX +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success - +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure US +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure UK +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success BR +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.success.log b/testing/btest/Baseline/logging.remote/sender.ssh.success.log index 10156943a3..d662d5af9f 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.success.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.success.log @@ -1,3 +1,3 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success - -1299215124.66932 1.2.3.4 66770 2.3.4.5 65616 success BR +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success - +1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/logging.remove/ssh.failure.log b/testing/btest/Baseline/logging.remove/ssh.failure.log index 75385cbbd7..5628867595 100644 --- a/testing/btest/Baseline/logging.remove/ssh.failure.log +++ b/testing/btest/Baseline/logging.remove/ssh.failure.log @@ -1,3 +1,3 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure US -1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure UK +1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure US +1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/logging.remove/ssh.log b/testing/btest/Baseline/logging.remove/ssh.log index a9fe2b9ffe..476442eec8 100644 --- a/testing/btest/Baseline/logging.remove/ssh.log +++ b/testing/btest/Baseline/logging.remove/ssh.log @@ -1,4 +1,4 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure US -1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure UK -1298341623.76997 1.2.3.4 66770 2.3.4.5 65616 failure BR +1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure US +1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure UK +1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure BR diff --git a/testing/btest/Baseline/logging.test-logging/ssh.log b/testing/btest/Baseline/logging.test-logging/ssh.log index b012cffbda..d73dc32ec0 100644 --- a/testing/btest/Baseline/logging.test-logging/ssh.log +++ b/testing/btest/Baseline/logging.test-logging/ssh.log @@ -1,6 +1,6 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p status country -1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 success unknown -1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 failure US -1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 failure UK -1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 success BR -1298326194.91688 1.2.3.4 66770 2.3.4.5 65616 failure MX +1299630368.69723 1.2.3.4 1234 2.3.4.5 80 success unknown +1299630368.69723 1.2.3.4 1234 2.3.4.5 80 failure US +1299630368.69723 1.2.3.4 1234 2.3.4.5 80 failure UK +1299630368.69723 1.2.3.4 1234 2.3.4.5 80 success BR +1299630368.69723 1.2.3.4 1234 2.3.4.5 80 failure MX From 83bd6584eee4877c8d8c014f9c6342e01f6eb118 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 16:27:07 -0800 Subject: [PATCH 37/67] Enabling record coercion for a table's &default attribute. --- src/Attr.cc | 10 ++++++++++ src/Val.cc | 17 ++++++++++++++++- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/src/Attr.cc b/src/Attr.cc index e03367c41e..98be9a9465 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -219,7 +219,17 @@ void Attributes::CheckAttr(Attr* a) Error("&default function type clash"); } else + { + BroType* ytype = tt->YieldType(); + + // Table defaults may be promotable. + if ( atype->Tag() == TYPE_RECORD && ytype->Tag() == TYPE_RECORD && + record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType()) ) + // Ok. + break; + Error("&default value has inconsistent type"); + } } } break; diff --git a/src/Val.cc b/src/Val.cc index ba1cda41e7..a665249539 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2023,7 +2023,22 @@ Val* TableVal::Default(Val* index) return 0; if ( ! def_val ) - def_val = def_attr->AttrExpr()->Eval(0); + { + BroType* ytype = Type()->YieldType(); + BroType* dtype = def_attr->AttrExpr()->Type(); + + if ( dtype->Tag() == TYPE_RECORD && ytype->Tag() == TYPE_RECORD && + ! same_type(dtype, ytype) && + record_promotion_compatible(dtype->AsRecordType(), ytype->AsRecordType()) ) + { + Expr* coerce = new RecordCoerceExpr(def_attr->AttrExpr(), ytype->AsRecordType()); + def_val = coerce->Eval(0); + Unref(coerce); + } + + else + def_val = def_attr->AttrExpr()->Eval(0); + } if ( ! def_val ) { From eb736a34b34b1eb55ecd4df5ecaa97064049a2a3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 16:28:59 -0800 Subject: [PATCH 38/67] '[]' is now a valid record ctor. --- src/parse.y | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/parse.y b/src/parse.y index 4f81347cde..300c338b5c 100644 --- a/src/parse.y +++ b/src/parse.y @@ -403,6 +403,12 @@ expr: $$ = $2; } + | '[' ']' + { + // We take this as an empty record constructor. + $$ = new RecordConstructorExpr(new ListExpr); + } + | TOK_RECORD '(' expr_list ')' { From df54cc6e78feb8ecebd5d21996fda7bf2e9e1ef8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 16:30:53 -0800 Subject: [PATCH 39/67] New table Log::rotation_control that enables to control rotation for individual files, overriding defaults. The interface isn't the greatest but the best I can come up with right now. --- TODO.logging | 22 +++++++++++++++--- policy/logging.bro | 57 ++++++++++++++++++++++++++++------------------ src/LogMgr.cc | 43 +++++++++++++++++++++++++++------- src/LogMgr.h | 1 + src/logging.bif | 4 ++-- 5 files changed, 92 insertions(+), 35 deletions(-) diff --git a/TODO.logging b/TODO.logging index d5bb3744b7..3e612eab5a 100644 --- a/TODO.logging +++ b/TODO.logging @@ -1,9 +1,25 @@ List of the things not implemented yet: - - Cluster-style remote_print. - - Rotation support. - Not sure if the logging does the right thing with &optional and &default values. Needs testing. - Spawning writers in separate threads (not clear if we want that initially). - - Check the new event-value code. + - Check the new event-value code. + - Configure Ascii Writer: + - "redef LogAscii::output_to_stdout = T" + - "redef LogAscii::separator = '\t'" + - "redef LogAscii::headers = T" + + - Extended filter manipualtion interface on the script level: + - Disalbe stream altogether. + - Change individual options of an existing filter. + +Notes about remote logging: + + - The receiver must create the stream locally via + Log::create_stream() in order to receive data for it. If not + created, anything sent will be ignored. + + - However, the receiver does not need to create filter locally. + Filter processing is done and the sender side, and as long as + stream exists at the receiver, it will record whatever it gets. diff --git a/policy/logging.bro b/policy/logging.bro index f3d4e35c37..90f42a8874 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -3,13 +3,6 @@ module Log; # Log::ID and Log::Writer are defined in bro.init due to circular dependencies. export { - # Information passed to a rotation callback function. - type RotationInfo: record { - path: string; # Original path value. - open: time; # Time when opened. - close: time; # Time when closed. - }; - # If true, local logging is by default enabled for all filters. const enable_local_logging = T &redef; @@ -19,20 +12,6 @@ export { # The default writer to use. const default_writer = Log::WRITER_ASCII &redef; - # Default rotation interval; zero disables rotation. - const default_rotation_interval = 0secs &redef; - - # Default naming suffix format. - const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; - - # Default postprocessor for writers outputting into files. - const default_rotation_postprocessor = "" &redef; - - # Default function to construct the name of the rotated file. - # The default implementation includes - # default_rotation_date_format into the file name. - global default_rotation_path_func: function(info: RotationInfo) : string &redef; - # A stream defining the logging. type Stream: record { # A record type defining the log's columns. @@ -81,6 +60,39 @@ export { writer: Writer &default=Log::default_writer; }; + ### Log rotation support. + + # Information passed to a rotation callback function. + type RotationInfo: record { + writer: Writer; # The writer. + path: string; # Original path value. + open: time; # Time when opened. + close: time; # Time when closed. + }; + + # Default rotation interval; zero disables rotation. + const default_rotation_interval = 0secs &redef; + + # Default naming suffix format. + const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; + + # Default postprocessor for writers outputting into files. + const default_rotation_postprocessor = "" &redef; + + # Default function to construct the name of the rotated file. + # The default implementation includes + # default_rotation_date_format into the file name. + global default_rotation_path_func: function(info: RotationInfo) : string &redef; + + type RotationControl: record { + interv: interval &default=default_rotation_interval; + date_fmt: string &default=default_rotation_date_format; + postprocessor: string &default=default_rotation_postprocessor; + }; + + # Defines rotation parameters per (id, path) tuple. + const rotation_control: table[Writer, string] of Log::RotationControl &default=[] &redef; + global create_stream: function(id: Log::ID, stream: Log::Stream) : bool; global add_filter: function(id: Log::ID, filter: Log::Filter) : bool; global remove_filter: function(id: Log::ID, name: string) : bool; @@ -97,7 +109,8 @@ module Log; function default_rotation_path_func(info: RotationInfo) : string { - return fmt("%s-%s", info$path, strftime(default_rotation_date_format, info$open)); + local date_fmt = rotation_control[info$writer, info$path]$date_fmt; + return fmt("%s-%s", info$path, strftime(date_fmt, info$open)); } function create_stream(id: Log::ID, stream: Log::Stream) : bool diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 46293e71fe..8f44c4a520 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -40,9 +40,10 @@ struct LogMgr::Filter { }; struct LogMgr::WriterInfo { - double open_time; - Timer* rotation_timer; - LogWriter *writer; + EnumVal* type; + double open_time; + Timer* rotation_timer; + LogWriter *writer; }; struct LogMgr::Stream { @@ -229,6 +230,7 @@ LogMgr::Stream::~Stream() if ( winfo->rotation_timer ) timer_mgr->Cancel(winfo->rotation_timer); + Unref(winfo->type); delete winfo->writer; delete i->second; } @@ -798,6 +800,7 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n } WriterInfo* winfo = new WriterInfo; + winfo->type = writer->Ref()->AsEnumVal(); winfo->writer = writer_obj; winfo->open_time = network_time; winfo->rotation_timer = 0; @@ -928,6 +931,22 @@ void RotationTimer::Dispatch(double t, int is_expire) } } +RecordVal* LogMgr::LookupRotationControl(EnumVal* writer, string path) + { + TableVal* rc = BifConst::Log::rotation_control->AsTableVal(); + + ListVal* index = new ListVal(TYPE_ANY); + index->Append(writer->Ref()); + index->Append(new StringVal(path.c_str())); + + Val* r = rc->Lookup(index); + assert(r); + + Unref(index); + + return r->AsRecordVal(); + } + void LogMgr::InstallRotationTimer(WriterInfo* winfo) { if ( terminating ) @@ -939,7 +958,10 @@ void LogMgr::InstallRotationTimer(WriterInfo* winfo) winfo->rotation_timer = 0; } - double rotation_interval = BifConst::Log::default_rotation_interval; + RecordVal* rc = LookupRotationControl(winfo->type, winfo->writer->Path()); + + int idx = rc->Type()->AsRecordType()->FieldOffset("interv"); + double rotation_interval = rc->LookupWithDefault(idx)->AsInterval(); if ( rotation_interval ) { @@ -974,14 +996,19 @@ void LogMgr::Rotate(WriterInfo* winfo) // Create the RotationInfo record. RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo); - info->Assign(0, new StringVal(winfo->writer->Path().c_str())); - info->Assign(1, new Val(winfo->open_time, TYPE_TIME)); - info->Assign(2, new Val(network_time, TYPE_TIME)); + info->Assign(0, winfo->type->Ref()); + info->Assign(1, new StringVal(winfo->writer->Path().c_str())); + info->Assign(2, new Val(winfo->open_time, TYPE_TIME)); + info->Assign(3, new Val(network_time, TYPE_TIME)); // Call the function building us the new path. Func* rotation_path_func = internal_func("Log::default_rotation_path_func"); - string rotation_postprocessor = BifConst::Log::default_rotation_postprocessor->AsString()->CheckString(); + + RecordVal* rc = LookupRotationControl(winfo->type, winfo->writer->Path()); + + int idx = rc->Type()->AsRecordType()->FieldOffset("postprocessor"); + string rotation_postprocessor = rc->LookupWithDefault(idx)->AsString()->CheckString(); val_list vl(1); vl.append(info); diff --git a/src/LogMgr.h b/src/LogMgr.h index 1188b80336..6a0c16159f 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -99,6 +99,7 @@ private: void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); void Rotate(WriterInfo* info); + RecordVal* LookupRotationControl(EnumVal* writer, string path); vector streams; // Indexed by stream enum. }; diff --git a/src/logging.bif b/src/logging.bif index a561c6c5a3..0babf75a06 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -9,9 +9,9 @@ module Log; type Filter: record; type Stream: record; type RotationInfo: record; +type RotationControl: record; -const Log::default_rotation_interval: interval; -const Log::default_rotation_postprocessor: string; +const Log::rotation_control : RotationControl; function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool %{ From b8ee425e0d15124458b12a6f09a0cb9179dd9eaa Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 16:33:06 -0800 Subject: [PATCH 40/67] Adding test for per-file rotation. --- .../btest/Baseline/logging.rotate-custom/out | 134 ++++++++++++++++++ testing/btest/logging/rotate-custom.bro | 37 +++++ 2 files changed, 171 insertions(+) create mode 100644 testing/btest/Baseline/logging.rotate-custom/out create mode 100644 testing/btest/logging/rotate-custom.bro diff --git a/testing/btest/Baseline/logging.rotate-custom/out b/testing/btest/Baseline/logging.rotate-custom/out new file mode 100644 index 0000000000..423b15651d --- /dev/null +++ b/testing/btest/Baseline/logging.rotate-custom/out @@ -0,0 +1,134 @@ +2nd test2-11-03-06_19.00.05.log test2.log 11-03-06_19.00.05 11-03-06_19.59.55 0 +1st test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0 +2nd test2-11-03-06_19.59.55.log test2.log 11-03-06_19.59.55 11-03-06_20.00.05 0 +2nd test2-11-03-06_20.00.05.log test2.log 11-03-06_20.00.05 11-03-06_20.59.55 0 +1st test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0 +2nd test2-11-03-06_20.59.55.log test2.log 11-03-06_20.59.55 11-03-06_21.00.05 0 +2nd test2-11-03-06_21.00.05.log test2.log 11-03-06_21.00.05 11-03-06_21.59.55 0 +1st test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0 +2nd test2-11-03-06_21.59.55.log test2.log 11-03-06_21.59.55 11-03-06_22.00.05 0 +2nd test2-11-03-06_22.00.05.log test2.log 11-03-06_22.00.05 11-03-06_22.59.55 0 +1st test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0 +2nd test2-11-03-06_22.59.55.log test2.log 11-03-06_22.59.55 11-03-06_23.00.05 0 +2nd test2-11-03-06_23.00.05.log test2.log 11-03-06_23.00.05 11-03-06_23.59.55 0 +1st test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0 +2nd test2-11-03-06_23.59.55.log test2.log 11-03-06_23.59.55 11-03-07_00.00.05 0 +2nd test2-11-03-07_00.00.05.log test2.log 11-03-07_00.00.05 11-03-07_00.59.55 0 +1st test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0 +2nd test2-11-03-07_00.59.55.log test2.log 11-03-07_00.59.55 11-03-07_01.00.05 0 +2nd test2-11-03-07_01.00.05.log test2.log 11-03-07_01.00.05 11-03-07_01.59.55 0 +1st test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0 +2nd test2-11-03-07_01.59.55.log test2.log 11-03-07_01.59.55 11-03-07_02.00.05 0 +2nd test2-11-03-07_02.00.05.log test2.log 11-03-07_02.00.05 11-03-07_02.59.55 0 +1st test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0 +2nd test2-11-03-07_02.59.55.log test2.log 11-03-07_02.59.55 11-03-07_03.00.05 0 +2nd test2-11-03-07_03.00.05.log test2.log 11-03-07_03.00.05 11-03-07_03.59.55 0 +1st test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0 +2nd test2-11-03-07_03.59.55.log test2.log 11-03-07_03.59.55 11-03-07_04.00.05 0 +2nd test2-11-03-07_04.00.05.log test2.log 11-03-07_04.00.05 11-03-07_04.59.55 0 +1st test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 +2nd test2-11-03-07_04.59.55.log test2.log 11-03-07_04.59.55 11-03-07_04.59.55 1 +> test-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +1299502795.0 10.0.0.2 20 10.0.0.3 9 +> test.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +> test2-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +> test2-11-03-06_19.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test2-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +> test2-11-03-06_20.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test2-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +> test2-11-03-06_21.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test2-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +> test2-11-03-06_22.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test2-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +> test2-11-03-06_23.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test2-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +> test2-11-03-07_00.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test2-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +> test2-11-03-07_01.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test2-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +> test2-11-03-07_02.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test2-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +> test2-11-03-07_03.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test2-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +> test2-11-03-07_04.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299502795.0 10.0.0.2 20 10.0.0.3 9 +> test2.log +# t id.orig_h id.orig_p id.resp_h id.resp_p diff --git a/testing/btest/logging/rotate-custom.bro b/testing/btest/logging/rotate-custom.bro new file mode 100644 index 0000000000..e77a0eced9 --- /dev/null +++ b/testing/btest/logging/rotate-custom.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out +# @TEST-EXEC: for i in test*.log; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: btest-diff out + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { Test }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + }; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor = "echo 1st"; + +redef Log::rotation_control += { + [Log::WRITER_ASCII, "test2"] = [$interv=30mins, $postprocessor="echo 2nd"] +}; + +event bro_init() +{ + Log::create_stream(Test, [$columns=Log]); + Log::add_filter(Test, [$name="2nd", $path="test2"]); + +} + +event new_connection(c: connection) + { + Log::write(Test, [$t=network_time(), $id=c$id]); + } From 5d8b9f2e7479a1861dca559542d6a77c3082a966 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 17:26:37 -0800 Subject: [PATCH 41/67] New function Log::get_filter() that allows to retrieve previously installed filter by name. This filter can then be modified and reinstalled via add_filter(), which will replace the old one with the same name. --- policy/logging.bro | 18 ++++++++++ src/LogMgr.cc | 13 +++++-- src/LogMgr.h | 4 ++- .../logging.adapt-filter/ssh-new-default.log | 3 ++ testing/btest/logging/adapt-filter.bro | 35 +++++++++++++++++++ 5 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log create mode 100644 testing/btest/logging/adapt-filter.bro diff --git a/policy/logging.bro b/policy/logging.bro index 90f42a8874..0bb9f6df83 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -93,9 +93,14 @@ export { # Defines rotation parameters per (id, path) tuple. const rotation_control: table[Writer, string] of Log::RotationControl &default=[] &redef; + ### Function. + + const no_filter: Filter = [$name=""]; # Sentinel. + global create_stream: function(id: Log::ID, stream: Log::Stream) : bool; global add_filter: function(id: Log::ID, filter: Log::Filter) : bool; global remove_filter: function(id: Log::ID, name: string) : bool; + global get_filter: function(id: Log::ID, name: string) : Filter; # Returns no_filter if not found. global write: function(id: Log::ID, columns: any) : bool; global set_buf: function(id: Log::ID, buffered: bool): bool; global flush: function(id: Log::ID): bool; @@ -103,6 +108,9 @@ export { global remove_default_filter: function(id: ID) : bool; } +# We keep a script-level copy of all filters so that we can directly manipulate them. +global filters: table[ID, string] of Filter; + @load logging.bif # Needs Log::Filter and Log::Stream defined. module Log; @@ -123,14 +131,24 @@ function create_stream(id: Log::ID, stream: Log::Stream) : bool function add_filter(id: Log::ID, filter: Log::Filter) : bool { + filters[id, filter$name] = filter; return Log::__add_filter(id, filter); } function remove_filter(id: Log::ID, name: string) : bool { + delete filters[id, name]; return Log::__remove_filter(id, name); } +function get_filter(id: Log::ID, name: string) : Filter + { + if ( [id, name] in filters ) + return filters[id, name]; + + return no_filter; + } + function write(id: Log::ID, columns: any) : bool { return Log::__write(id, columns); diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 8f44c4a520..76feb0e0ea 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -492,6 +492,10 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) filter->path_val = new StringVal(path.c_str()); } + // Remove any filter with the same name we might already have. + RemoveFilter(id, filter->name); + + // Add the new one. stream->filters.push_back(filter); #ifdef DEBUG @@ -514,14 +518,17 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) return true; } -bool LogMgr::RemoveFilter(EnumVal* id, StringVal* filter) +bool LogMgr::RemoveFilter(EnumVal* id, StringVal* name) + { + return RemoveFilter(id, name->AsString()->CheckString()); + } + +bool LogMgr::RemoveFilter(EnumVal* id, string name) { Stream* stream = FindStream(id); if ( ! stream ) return false; - string name = filter->AsString()->CheckString(); - for ( list::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i ) { if ( (*i)->name == name ) diff --git a/src/LogMgr.h b/src/LogMgr.h index 6a0c16159f..3a356b4e88 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -68,7 +68,8 @@ public: // actual BiFs just forward here. bool CreateStream(EnumVal* id, RecordVal* stream); bool AddFilter(EnumVal* id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, StringVal* filter); + bool RemoveFilter(EnumVal* id, StringVal* name); + bool RemoveFilter(EnumVal* id, string name); bool Write(EnumVal* id, RecordVal* columns); bool SetBuf(EnumVal* id, bool enabled); // Changes the state for all writers for that stream. bool Flush(EnumVal* id); // Flushes all writers for the stream. @@ -100,6 +101,7 @@ private: void InstallRotationTimer(WriterInfo* winfo); void Rotate(WriterInfo* info); RecordVal* LookupRotationControl(EnumVal* writer, string path); + Filter* FindFilter(EnumVal* id, StringVal* filter); vector streams; // Indexed by stream enum. }; diff --git a/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log new file mode 100644 index 0000000000..e0f76c1dfc --- /dev/null +++ b/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log @@ -0,0 +1,3 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299633982.99326 1.2.3.4 1234 2.3.4.5 80 success unknown +1299633982.99326 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/logging/adapt-filter.bro b/testing/btest/logging/adapt-filter.bro new file mode 100644 index 0000000000..393f46eba9 --- /dev/null +++ b/testing/btest/logging/adapt-filter.bro @@ -0,0 +1,35 @@ + +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh-new-default.log +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local filter = Log::get_filter(SSH, "default"); + filter$path= "ssh-new-default"; + Log::add_filter(SSH, filter); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); +} From 88d114053c771bafda742630295ba22a42b1d79b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 17:32:21 -0800 Subject: [PATCH 42/67] Removing unnessary namespace qualifiers in logging.bro. --- policy/logging.bro | 48 +++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/policy/logging.bro b/policy/logging.bro index 0bb9f6df83..e362435cf4 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -10,7 +10,7 @@ export { const enable_remote_logging = T &redef; # The default writer to use. - const default_writer = Log::WRITER_ASCII &redef; + const default_writer = WRITER_ASCII &redef; # A stream defining the logging. type Stream: record { @@ -57,7 +57,7 @@ export { log_remote: bool &default=enable_remote_logging; # The writer to use. - writer: Writer &default=Log::default_writer; + writer: Writer &default=default_writer; }; ### Log rotation support. @@ -91,19 +91,19 @@ export { }; # Defines rotation parameters per (id, path) tuple. - const rotation_control: table[Writer, string] of Log::RotationControl &default=[] &redef; + const rotation_control: table[Writer, string] of RotationControl &default=[] &redef; ### Function. const no_filter: Filter = [$name=""]; # Sentinel. - global create_stream: function(id: Log::ID, stream: Log::Stream) : bool; - global add_filter: function(id: Log::ID, filter: Log::Filter) : bool; - global remove_filter: function(id: Log::ID, name: string) : bool; - global get_filter: function(id: Log::ID, name: string) : Filter; # Returns no_filter if not found. - global write: function(id: Log::ID, columns: any) : bool; - global set_buf: function(id: Log::ID, buffered: bool): bool; - global flush: function(id: Log::ID): bool; + global create_stream: function(id: ID, stream: Stream) : bool; + global add_filter: function(id: ID, filter: Filter) : bool; + global remove_filter: function(id: ID, name: string) : bool; + global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found. + global write: function(id: ID, columns: any) : bool; + global set_buf: function(id: ID, buffered: bool): bool; + global flush: function(id: ID): bool; global add_default_filter: function(id: ID) : bool; global remove_default_filter: function(id: ID) : bool; } @@ -111,7 +111,7 @@ export { # We keep a script-level copy of all filters so that we can directly manipulate them. global filters: table[ID, string] of Filter; -@load logging.bif # Needs Log::Filter and Log::Stream defined. +@load logging.bif # Needs Filter and Stream defined. module Log; @@ -121,27 +121,27 @@ function default_rotation_path_func(info: RotationInfo) : string return fmt("%s-%s", info$path, strftime(date_fmt, info$open)); } -function create_stream(id: Log::ID, stream: Log::Stream) : bool +function create_stream(id: ID, stream: Stream) : bool { - if ( ! Log::__create_stream(id, stream) ) + if ( ! __create_stream(id, stream) ) return F; return add_default_filter(id); } -function add_filter(id: Log::ID, filter: Log::Filter) : bool +function add_filter(id: ID, filter: Filter) : bool { filters[id, filter$name] = filter; - return Log::__add_filter(id, filter); + return __add_filter(id, filter); } -function remove_filter(id: Log::ID, name: string) : bool +function remove_filter(id: ID, name: string) : bool { delete filters[id, name]; - return Log::__remove_filter(id, name); + return __remove_filter(id, name); } -function get_filter(id: Log::ID, name: string) : Filter +function get_filter(id: ID, name: string) : Filter { if ( [id, name] in filters ) return filters[id, name]; @@ -149,19 +149,19 @@ function get_filter(id: Log::ID, name: string) : Filter return no_filter; } -function write(id: Log::ID, columns: any) : bool +function write(id: ID, columns: any) : bool { - return Log::__write(id, columns); + return __write(id, columns); } -function set_buf(id: Log::ID, buffered: bool): bool +function set_buf(id: ID, buffered: bool): bool { - return Log::__set_buf(id, buffered); + return __set_buf(id, buffered); } -function flush(id: Log::ID): bool +function flush(id: ID): bool { - return Log::__flush(id); + return __flush(id); } function add_default_filter(id: ID) : bool From 4b7c5905f118a5e7fa3535402c190df3156475f3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 17:54:11 -0800 Subject: [PATCH 43/67] New functions Log::disable_stream() and Log::enable_stream(). When disabled, all outout to a stream will be ignored (but no error raised). --- TODO.logging | 4 --- policy/logging.bro | 8 +++++ src/LogMgr.cc | 41 ++++++++++++++++++++++++ src/LogMgr.h | 2 ++ src/logging.bif | 12 +++++++ testing/btest/logging/disable-stream.bro | 33 +++++++++++++++++++ 6 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 testing/btest/logging/disable-stream.bro diff --git a/TODO.logging b/TODO.logging index 3e612eab5a..5ddca90ec7 100644 --- a/TODO.logging +++ b/TODO.logging @@ -10,10 +10,6 @@ List of the things not implemented yet: - "redef LogAscii::separator = '\t'" - "redef LogAscii::headers = T" - - Extended filter manipualtion interface on the script level: - - Disalbe stream altogether. - - Change individual options of an existing filter. - Notes about remote logging: - The receiver must create the stream locally via diff --git a/policy/logging.bro b/policy/logging.bro index e362435cf4..07baa98902 100644 --- a/policy/logging.bro +++ b/policy/logging.bro @@ -98,6 +98,8 @@ export { const no_filter: Filter = [$name=""]; # Sentinel. global create_stream: function(id: ID, stream: Stream) : bool; + global enable_stream: function(id: ID) : bool; + global disable_stream: function(id: ID) : bool; global add_filter: function(id: ID, filter: Filter) : bool; global remove_filter: function(id: ID, name: string) : bool; global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found. @@ -128,6 +130,12 @@ function create_stream(id: ID, stream: Stream) : bool return add_default_filter(id); } + +function disable_stream(id: ID) : bool + { + if ( ! __disable_stream(id) ) + return F; + } function add_filter(id: ID, filter: Filter) : bool { diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 76feb0e0ea..0873ca991e 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -48,6 +48,7 @@ struct LogMgr::WriterInfo { struct LogMgr::Stream { EnumVal* id; + bool enabled; string name; RecordType* columns; EventHandlerPtr event; @@ -335,6 +336,7 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) // Create new stream. streams[idx] = new Stream; streams[idx]->id = id->Ref()->AsEnumVal(); + streams[idx]->enabled = true; streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; streams[idx]->columns = columns->Ref()->AsRecordType(); @@ -344,6 +346,36 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) return true; } +bool LogMgr::EnableStream(EnumVal* id) + { + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + if ( stream->enabled ) + return true; + + stream->enabled = true; + + DBG_LOG(DBG_LOGGING, "Reenabled logging stream '%s'", stream->name.c_str()); + return true; + } + +bool LogMgr::DisableStream(EnumVal* id) + { + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + if ( ! stream->enabled ) + return true; + + stream->enabled = false; + + DBG_LOG(DBG_LOGGING, "Disabled logging stream '%s'", stream->name.c_str()); + return true; + } + // Helper for recursive record field unrolling. bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) { @@ -554,6 +586,9 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) if ( ! stream ) return false; + if ( ! stream->enabled ) + return true; + columns = columns->CoerceTo(stream->columns); if ( ! columns ) @@ -833,6 +868,9 @@ bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, Lo return false; } + if ( ! stream->enabled ) + return true; + Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); if ( w == stream->writers.end() ) @@ -891,6 +929,9 @@ bool LogMgr::Flush(EnumVal* id) if ( ! stream ) return false; + if ( ! stream->enabled ) + return true; + for ( Stream::WriterMap::iterator i = stream->writers.begin(); i != stream->writers.end(); i++ ) i->second->writer->Flush(); diff --git a/src/LogMgr.h b/src/LogMgr.h index 3a356b4e88..8f8d991c00 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -67,6 +67,8 @@ public: // These correspond to the BiFs visible on the scripting layer. The // actual BiFs just forward here. bool CreateStream(EnumVal* id, RecordVal* stream); + bool EnableStream(EnumVal* id); + bool DisableStream(EnumVal* id); bool AddFilter(EnumVal* id, RecordVal* filter); bool RemoveFilter(EnumVal* id, StringVal* name); bool RemoveFilter(EnumVal* id, string name); diff --git a/src/logging.bif b/src/logging.bif index 0babf75a06..a4b56bb8d3 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -19,6 +19,18 @@ function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool return new Val(result, TYPE_BOOL); %} +function Log::__enable_stream%(id: Log::ID%) : bool + %{ + bool result = log_mgr->EnableStream(id->AsEnumVal()); + return new Val(result, TYPE_BOOL); + %} + +function Log::__disable_stream%(id: Log::ID%) : bool + %{ + bool result = log_mgr->DisableStream(id->AsEnumVal()); + return new Val(result, TYPE_BOOL); + %} + function Log::__add_filter%(id: Log::ID, filter: Log::Filter%) : bool %{ bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); diff --git a/testing/btest/logging/disable-stream.bro b/testing/btest/logging/disable-stream.bro new file mode 100644 index 0000000000..97b47298e9 --- /dev/null +++ b/testing/btest/logging/disable-stream.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + Log::disable_stream(SSH); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From 26eab74ecc75dca0fb7f93f877167f3fe9e79ec4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 17:58:03 -0800 Subject: [PATCH 44/67] The ASCII writer can now deal with /dev/* paths. It will not longer try to add a ".log" extension. --- TODO.logging | 1 - src/LogWriterAscii.cc | 6 +++- src/LogWriterAscii.h | 2 ++ testing/btest/Baseline/logging.stdout/output | 6 ++++ testing/btest/logging/stdout.bro | 36 ++++++++++++++++++++ 5 files changed, 49 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/logging.stdout/output create mode 100644 testing/btest/logging/stdout.bro diff --git a/TODO.logging b/TODO.logging index 5ddca90ec7..1aa9132a55 100644 --- a/TODO.logging +++ b/TODO.logging @@ -2,7 +2,6 @@ List of the things not implemented yet: - Not sure if the logging does the right thing with &optional and &default values. Needs testing. - - Spawning writers in separate threads (not clear if we want that initially). - Check the new event-value code. - Configure Ascii Writer: diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 6ba3f812f7..c38e49eb21 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -17,7 +17,7 @@ LogWriterAscii::~LogWriterAscii() bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * fields) { - fname = path + ".log"; + fname = IsSpecial(path) ? path : path + ".log"; if ( ! (file = fopen(fname.c_str(), "w")) ) { @@ -135,6 +135,10 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, Log bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open, double close, bool terminating) { + if ( ! IsSpecial(Path()) ) + // Don't rotate special files. + return true; + fclose(file); string nname = rotated_path + ".log"; diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 9732242fba..0e060ea97e 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -23,6 +23,8 @@ protected: virtual void DoFinish(); private: + bool IsSpecial(string path) { return path.find("/dev/") == 0; } + FILE* file; string fname; }; diff --git a/testing/btest/Baseline/logging.stdout/output b/testing/btest/Baseline/logging.stdout/output new file mode 100644 index 0000000000..d7dcbd4e48 --- /dev/null +++ b/testing/btest/Baseline/logging.stdout/output @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299635864.89679 1.2.3.4 1234 2.3.4.5 80 success unknown +1299635864.89679 1.2.3.4 1234 2.3.4.5 80 failure US +1299635864.89679 1.2.3.4 1234 2.3.4.5 80 failure UK +1299635864.89679 1.2.3.4 1234 2.3.4.5 80 success BR +1299635864.89679 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/logging/stdout.bro b/testing/btest/logging/stdout.bro new file mode 100644 index 0000000000..15fd071b58 --- /dev/null +++ b/testing/btest/logging/stdout.bro @@ -0,0 +1,36 @@ +# +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local filter = Log::get_filter(SSH, "default"); + filter$path= "/dev/stdout"; + Log::add_filter(SSH, filter); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From c6d20dbfdfdca18fffea6b6c1e6ea9d8e324ae19 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 8 Mar 2011 21:43:52 -0800 Subject: [PATCH 45/67] Adding a few options to the ASCII writer. module LogAscii; export { # Output everything to stdout rather than into files. This is primarily # for testing purposes. const output_to_stdout = F &redef; # The separator between fields. const separator = "\t" &redef; # True to include a header line with column names. const include_header = T &redef; } --- TODO.logging | 7 +--- policy/bro.init | 1 + policy/logging-ascii.bro | 15 ++++++++ src/LogWriter.h | 4 +- src/LogWriterAscii.cc | 37 +++++++++++++------ src/LogWriterAscii.h | 5 +++ src/logging.bif | 11 +++++- .../Baseline/logging.ascii-options/output | 5 +++ testing/btest/logging/ascii-options.bro | 35 ++++++++++++++++++ 9 files changed, 101 insertions(+), 19 deletions(-) create mode 100644 policy/logging-ascii.bro create mode 100644 testing/btest/Baseline/logging.ascii-options/output create mode 100644 testing/btest/logging/ascii-options.bro diff --git a/TODO.logging b/TODO.logging index 1aa9132a55..2e6c93b392 100644 --- a/TODO.logging +++ b/TODO.logging @@ -3,11 +3,8 @@ List of the things not implemented yet: - Not sure if the logging does the right thing with &optional and &default values. Needs testing. - Check the new event-value code. - - - Configure Ascii Writer: - - "redef LogAscii::output_to_stdout = T" - - "redef LogAscii::separator = '\t'" - - "redef LogAscii::headers = T" + - The Ascii writer doesn't escape the delimiter if it appears + within a field's value. Seems we need to do that. Notes about remote logging: diff --git a/policy/bro.init b/policy/bro.init index 84f0094a71..b3f78f0689 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -278,6 +278,7 @@ type entropy_test_result: record { @load bro.bif.bro @load logging # sic! Not logging.bif. +@load logging-ascii global bro_alarm_file: file &redef; global alarm_hook: function(msg: string): bool &redef; diff --git a/policy/logging-ascii.bro b/policy/logging-ascii.bro new file mode 100644 index 0000000000..0cabeae940 --- /dev/null +++ b/policy/logging-ascii.bro @@ -0,0 +1,15 @@ + +module LogAscii; + +export { + # Output everything to stdout rather than into files. This is primarily + # for testing purposes. + const output_to_stdout = F &redef; + + # The separator between fields. + const separator = "\t" &redef; + + # True to include a header line with column names. + const include_header = T &redef; +} + diff --git a/src/LogWriter.h b/src/LogWriter.h index 5a89e6be9e..3d3da905df 100644 --- a/src/LogWriter.h +++ b/src/LogWriter.h @@ -3,7 +3,9 @@ // // Note than classes derived from LogWriter must be fully thread-safe and not // use any non-safe Bro functionality (which is almost all ...). In -// particular, do not use fmt() but LogWriter::Fmt()!. +// particular, do not use fmt() but LogWriter::Fmt()!. The one exception is +// the constructor: that is guaranteed to be executed inside the main thread +// and it can thus access in particular global script variables. #ifndef LOGWRITER_H #define LOGWRITER_H diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index c38e49eb21..d5e550aa91 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -3,20 +3,30 @@ #include #include "LogWriterAscii.h" +#include "NetVar.h" LogWriterAscii::LogWriterAscii() { file = 0; + + output_to_stdout = BifConst::LogAscii::output_to_stdout; + include_header = BifConst::LogAscii::include_header; + separator = strdup(BifConst::LogAscii::separator->CheckString()); } LogWriterAscii::~LogWriterAscii() { if ( file ) fclose(file); + + free(separator); } bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * fields) { + if ( output_to_stdout ) + path = "/dev/stdout"; + fname = IsSpecial(path) ? path : path + ".log"; if ( ! (file = fopen(fname.c_str(), "w")) ) @@ -25,22 +35,25 @@ bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * return false; } - if ( fputs("# ", file) == EOF ) - goto write_error; - - for ( int i = 0; i < num_fields; i++ ) + if ( include_header ) { - const LogField* field = fields[i]; - if ( fputs(field->name.c_str(), file) == EOF ) + if ( fputs("# ", file) == EOF ) goto write_error; - if ( fputc('\t', file) == EOF ) + for ( int i = 0; i < num_fields; i++ ) + { + const LogField* field = fields[i]; + if ( fputs(field->name.c_str(), file) == EOF ) + goto write_error; + + if ( fputs(separator, file) == EOF ) + goto write_error; + } + + if ( fputc('\n', file) == EOF ) goto write_error; } - if ( fputc('\n', file) == EOF ) - goto write_error; - return true; write_error: @@ -65,7 +78,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, Log for ( int i = 0; i < num_fields; i++ ) { if ( i > 0 ) - desc.Add("\t"); + desc.Add(separator); LogVal* val = vals[i]; const LogField* field = fields[i]; @@ -135,7 +148,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, Log bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open, double close, bool terminating) { - if ( ! IsSpecial(Path()) ) + if ( IsSpecial(Path()) ) // Don't rotate special files. return true; diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 0e060ea97e..0f42874da6 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -27,6 +27,11 @@ private: FILE* file; string fname; + + // Options from the script-level + bool output_to_stdout; + bool include_header; + char* separator; }; #endif diff --git a/src/logging.bif b/src/logging.bif index a4b56bb8d3..2095264910 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -11,7 +11,7 @@ type Stream: record; type RotationInfo: record; type RotationControl: record; -const Log::rotation_control : RotationControl; +const Log::rotation_control: RotationControl; function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool %{ @@ -60,3 +60,12 @@ function Log::__flush%(id: Log::ID%): bool bool result = log_mgr->Flush(id->AsEnumVal()); return new Val(result, TYPE_BOOL); %} + +# Options for the ASCII writer. + +module LogAscii; + +const output_to_stdout: bool; +const separator: string; +const include_header: bool; + diff --git a/testing/btest/Baseline/logging.ascii-options/output b/testing/btest/Baseline/logging.ascii-options/output new file mode 100644 index 0000000000..9a3e1f8d20 --- /dev/null +++ b/testing/btest/Baseline/logging.ascii-options/output @@ -0,0 +1,5 @@ +1299649281.43936|1.2.3.4|1234|2.3.4.5|80|success|unknown +1299649281.43936|1.2.3.4|1234|2.3.4.5|80|failure|US +1299649281.43936|1.2.3.4|1234|2.3.4.5|80|failure|UK +1299649281.43936|1.2.3.4|1234|2.3.4.5|80|success|BR +1299649281.43936|1.2.3.4|1234|2.3.4.5|80|failure|MX diff --git a/testing/btest/logging/ascii-options.bro b/testing/btest/logging/ascii-options.bro new file mode 100644 index 0000000000..747249342b --- /dev/null +++ b/testing/btest/logging/ascii-options.bro @@ -0,0 +1,35 @@ +# +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +redef LogAscii::output_to_stdout = T; +redef LogAscii::separator = "|"; +redef LogAscii::include_header = F; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From cb9e0a5d5a1965fc7126f965a8c17767f4d16d32 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 16:26:11 -0800 Subject: [PATCH 46/67] If a field value contains the separator, that is now escape with hex characters. --- src/Desc.cc | 44 ++++++++++++++++++ src/Desc.h | 10 ++++ src/LogWriterAscii.cc | 12 +++-- src/LogWriterAscii.h | 1 + .../Baseline/logging.ascii-escape/ssh.log | Bin 0 -> 373 bytes testing/btest/logging/ascii-escape.bro | 33 +++++++++++++ 6 files changed, 96 insertions(+), 4 deletions(-) create mode 100644 testing/btest/Baseline/logging.ascii-escape/ssh.log create mode 100644 testing/btest/logging/ascii-escape.bro diff --git a/src/Desc.cc b/src/Desc.cc index baf3ad1160..b73f8ca8cd 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -41,6 +41,8 @@ ODesc::ODesc(desc_type t, BroFile* arg_f) want_quotes = 0; do_flush = 1; include_stats = 0; + escape = 0; + escape_len = 0; } ODesc::~ODesc() @@ -54,6 +56,12 @@ ODesc::~ODesc() free(base); } +void ODesc::SetEscape(const char* arg_escape, int len) + { + escape = arg_escape; + escape_len = len; + } + void ODesc::PushIndent() { ++indent_level; @@ -183,8 +191,44 @@ void ODesc::Indent() Add("\t", 0); } +static const char hex_chars[] = "0123456789ABCDEF"; void ODesc::AddBytes(const void* bytes, unsigned int n) + { + if ( ! escape ) + return AddBytesRaw(bytes, n); + + const char* s = (const char*) bytes; + const char* e = (const char*) bytes + n; + + while ( s < e ) + { + const char* t = (const char*) memchr(s, escape[0], e - s); + + if ( ! t ) + break; + + if ( memcmp(t, escape, escape_len) != 0 ) + break; + + AddBytesRaw(s, t - s); + + for ( int i = 0; i < escape_len; ++i ) + { + char hex[5] = "\\x00"; + hex[2] = hex_chars[(*t) >> 4]; + hex[3] = hex_chars[(*t) & 0x0f]; + AddBytesRaw(hex, sizeof(hex)); + ++t; + } + + s = t; + } + + AddBytesRaw(s, e - s); + } + +void ODesc::AddBytesRaw(const void* bytes, unsigned int n) { if ( n == 0 ) return; diff --git a/src/Desc.h b/src/Desc.h index 49b331d51b..55891223cc 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -49,6 +49,9 @@ public: void SetFlush(int arg_do_flush) { do_flush = arg_do_flush; } + // The string passed in must remain valid as long as this object lives. + void SetEscape(const char* escape, int len); + void PushIndent(); void PopIndent(); int GetIndentLevel() const { return indent_level; } @@ -93,6 +96,9 @@ public: Add("\n", 0); } + // Bypasses the escaping enabled via SetEscape(). + void AddRaw(const char* s, int len) { AddBytesRaw(s, len); } + // Returns the description as a string. const char* Description() const { return (const char*) base; } @@ -115,6 +121,7 @@ protected: void Indent(); void AddBytes(const void* bytes, unsigned int n); + void AddBytesRaw(const void* bytes, unsigned int n); // Make buffer big enough for n bytes beyond bufp. void Grow(unsigned int n); @@ -128,6 +135,9 @@ protected: unsigned int offset; // where we are in the buffer unsigned int size; // size of buffer in bytes + int escape_len; // number of bytes in to escape sequence + const char* escape; // bytes to escape on output + BroFile* f; // or the file we're using. int indent_level; diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index d5e550aa91..050616efd4 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -11,7 +11,10 @@ LogWriterAscii::LogWriterAscii() output_to_stdout = BifConst::LogAscii::output_to_stdout; include_header = BifConst::LogAscii::include_header; - separator = strdup(BifConst::LogAscii::separator->CheckString()); + + separator_len = BifConst::LogAscii::separator->Len(); + separator = new char[separator_len]; + memcpy(separator, BifConst::LogAscii::separator->Bytes(), separator_len); } LogWriterAscii::~LogWriterAscii() @@ -19,7 +22,7 @@ LogWriterAscii::~LogWriterAscii() if ( file ) fclose(file); - free(separator); + delete [] separator; } bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * fields) @@ -46,7 +49,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * if ( fputs(field->name.c_str(), file) == EOF ) goto write_error; - if ( fputs(separator, file) == EOF ) + if ( fwrite(separator, separator_len, 1, file) != 1 ) goto write_error; } @@ -74,11 +77,12 @@ void LogWriterAscii::DoFinish() bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, LogVal** vals) { ODesc desc(DESC_READABLE); + desc.SetEscape(separator, separator_len); for ( int i = 0; i < num_fields; i++ ) { if ( i > 0 ) - desc.Add(separator); + desc.AddRaw(separator, separator_len); LogVal* val = vals[i]; const LogField* field = fields[i]; diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 0f42874da6..04968e1a8b 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -32,6 +32,7 @@ private: bool output_to_stdout; bool include_header; char* separator; + int separator_len; }; #endif diff --git a/testing/btest/Baseline/logging.ascii-escape/ssh.log b/testing/btest/Baseline/logging.ascii-escape/ssh.log new file mode 100644 index 0000000000000000000000000000000000000000..9892e1aeee15040732851bcfff5b9d27885b195b GIT binary patch literal 373 zcmY#ZD5XKrjoy%%ard0;njM2^1|ZNh~QXuBl1RFU>0{s;sHuGBmQZ zG&eLeH#O9=G&L}(sWH?u(lgcrF^r6XB$%UTT2o_T0MuTZoSa$=)L)vHotIypN1E=m t#LS%1qSTt2&|p$E##ER)Gk^(*0p6q;P+W>I0PKFJAkqwgJKZ;e3ji!Wa_|5E literal 0 HcmV?d00001 diff --git a/testing/btest/logging/ascii-escape.bro b/testing/btest/logging/ascii-escape.bro new file mode 100644 index 0000000000..1789c1bb63 --- /dev/null +++ b/testing/btest/logging/ascii-escape.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +redef LogAscii::separator = "||"; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="fa||ure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="su||ess", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From b69ecff3ee58d38f0fb2fd5457b4531498d27b36 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 16:52:46 -0800 Subject: [PATCH 47/67] More options for the ASCII writer. # The prefix for the header line if included. const header_prefix = "# " &redef; # The string to use for empty string fields. const empty_field = "" &redef; # The string to use for an unset optional field. const unset_field = "-" &redef; --- policy/logging-ascii.bro | 14 ++++++- src/LogWriterAscii.cc | 27 +++++++++++-- src/LogWriterAscii.h | 10 +++++ src/logging.bif | 5 ++- .../btest/Baseline/logging.ascii-empty/output | 6 +++ testing/btest/logging/ascii-empty.bro | 38 +++++++++++++++++++ testing/btest/logging/ascii-escape.bro | 3 +- 7 files changed, 95 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/logging.ascii-empty/output create mode 100644 testing/btest/logging/ascii-empty.bro diff --git a/policy/logging-ascii.bro b/policy/logging-ascii.bro index 0cabeae940..151d29de99 100644 --- a/policy/logging-ascii.bro +++ b/policy/logging-ascii.bro @@ -6,10 +6,20 @@ export { # for testing purposes. const output_to_stdout = F &redef; + # True to include a header line with column names. + const include_header = T &redef; + + # The prefix for the header line if included. + const header_prefix = "# " &redef; + # The separator between fields. const separator = "\t" &redef; - # True to include a header line with column names. - const include_header = T &redef; + # The string to use for empty string fields. + const empty_field = "" &redef; + + # The string to use for an unset optional field. + const unset_field = "-" &redef; } + diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 050616efd4..71bd0cf2e0 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -15,6 +15,18 @@ LogWriterAscii::LogWriterAscii() separator_len = BifConst::LogAscii::separator->Len(); separator = new char[separator_len]; memcpy(separator, BifConst::LogAscii::separator->Bytes(), separator_len); + + empty_field_len = BifConst::LogAscii::empty_field->Len(); + empty_field = new char[empty_field_len]; + memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), empty_field_len); + + unset_field_len = BifConst::LogAscii::unset_field->Len(); + unset_field = new char[unset_field_len]; + memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), unset_field_len); + + header_prefix_len = BifConst::LogAscii::header_prefix->Len(); + header_prefix = new char[header_prefix_len]; + memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(), header_prefix_len); } LogWriterAscii::~LogWriterAscii() @@ -23,6 +35,9 @@ LogWriterAscii::~LogWriterAscii() fclose(file); delete [] separator; + delete [] empty_field; + delete [] unset_field; + delete [] header_prefix; } bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * fields) @@ -40,7 +55,7 @@ bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * if ( include_header ) { - if ( fputs("# ", file) == EOF ) + if ( fwrite(header_prefix, header_prefix_len, 1, file) != 1 ) goto write_error; for ( int i = 0; i < num_fields; i++ ) @@ -89,7 +104,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, Log if ( ! val->present ) { - desc.Add("-"); // TODO: Probably want to get rid of the "-". + desc.AddN(unset_field, unset_field_len); continue; } @@ -127,8 +142,14 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, Log break; case TYPE_STRING: - desc.AddN(val->val.string_val->data(), val->val.string_val->size()); + { + int size = val->val.string_val->size(); + if ( size ) + desc.AddN(val->val.string_val->data(), val->val.string_val->size()); + else + desc.AddN(empty_field, empty_field_len); break; + } default: Error(Fmt("unsupported field format %d for %s", field->type, field->name.c_str())); diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 04968e1a8b..384be65f19 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -31,8 +31,18 @@ private: // Options from the script-level bool output_to_stdout; bool include_header; + char* separator; int separator_len; + + char* empty_field; + int empty_field_len; + + char* unset_field; + int unset_field_len; + + char* header_prefix; + int header_prefix_len; }; #endif diff --git a/src/logging.bif b/src/logging.bif index 2095264910..2eb8eea35b 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -66,6 +66,9 @@ function Log::__flush%(id: Log::ID%): bool module LogAscii; const output_to_stdout: bool; -const separator: string; const include_header: bool; +const header_prefix: string; +const separator: string; +const empty_field: string; +const unset_field: string; diff --git a/testing/btest/Baseline/logging.ascii-empty/output b/testing/btest/Baseline/logging.ascii-empty/output new file mode 100644 index 0000000000..1a7ada05ae --- /dev/null +++ b/testing/btest/Baseline/logging.ascii-empty/output @@ -0,0 +1,6 @@ +PREFIX<>t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b| +1299718338.37017|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET +1299718338.37017|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET +1299718338.37017|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET +1299718338.37017|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET +1299718338.37017|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T diff --git a/testing/btest/logging/ascii-empty.bro b/testing/btest/logging/ascii-empty.bro new file mode 100644 index 0000000000..62022107f0 --- /dev/null +++ b/testing/btest/logging/ascii-empty.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +redef LogAscii::output_to_stdout = T; +redef LogAscii::separator = "|"; +redef LogAscii::empty_field = "EMPTY"; +redef LogAscii::unset_field = "NOT-SET"; +redef LogAscii::header_prefix = "PREFIX<>"; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + b: bool &optional; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $b=T, $status="failure", $country=""]); + +} + diff --git a/testing/btest/logging/ascii-escape.bro b/testing/btest/logging/ascii-escape.bro index 1789c1bb63..f5c50c60bd 100644 --- a/testing/btest/logging/ascii-escape.bro +++ b/testing/btest/logging/ascii-escape.bro @@ -27,7 +27,6 @@ event bro_init() Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); Log::write(SSH, [$t=network_time(), $id=cid, $status="fa||ure", $country="UK"]); Log::write(SSH, [$t=network_time(), $id=cid, $status="su||ess", $country="BR"]); - Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); - + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); } From 52c54859b620e2fbaf6f97df26c56e750c6b92b3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 16:55:29 -0800 Subject: [PATCH 48/67] Bugfix: the header line was ending with a separator. Test baselines needed adaption as well. --- src/LogWriterAscii.cc | 6 +- .../logging.adapt-filter/ssh-new-default.log | 6 +- .../btest/Baseline/logging.ascii-empty/output | 12 ++-- .../Baseline/logging.ascii-escape/ssh.log | Bin 373 -> 376 bytes .../Baseline/logging.ascii-options/output | 10 +-- testing/btest/Baseline/logging.events/output | 4 +- .../btest/Baseline/logging.path-func/output | 20 +++--- .../Baseline/logging.pred/ssh.failure.log | 4 +- .../Baseline/logging.pred/ssh.success.log | 4 +- .../logging.remote/sender.ssh.failure.log | 8 +-- .../Baseline/logging.remote/sender.ssh.log | 12 ++-- .../logging.remote/sender.ssh.success.log | 6 +- .../Baseline/logging.remove/ssh.failure.log | 6 +- testing/btest/Baseline/logging.remove/ssh.log | 8 +-- .../btest/Baseline/logging.rotate-custom/out | 64 +++++++++--------- testing/btest/Baseline/logging.rotate/out | 20 +++--- testing/btest/Baseline/logging.stdout/output | 12 ++-- .../Baseline/logging.test-logging/ssh.log | 12 ++-- 18 files changed, 107 insertions(+), 107 deletions(-) diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 71bd0cf2e0..3ec22b7fa0 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -60,11 +60,11 @@ bool LogWriterAscii::DoInit(string path, int num_fields, const LogField* const * for ( int i = 0; i < num_fields; i++ ) { - const LogField* field = fields[i]; - if ( fputs(field->name.c_str(), file) == EOF ) + if ( i > 0 && fwrite(separator, separator_len, 1, file) != 1 ) goto write_error; - if ( fwrite(separator, separator_len, 1, file) != 1 ) + const LogField* field = fields[i]; + if ( fputs(field->name.c_str(), file) == EOF ) goto write_error; } diff --git a/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log index e0f76c1dfc..469f2d1991 100644 --- a/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log +++ b/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log @@ -1,3 +1,3 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299633982.99326 1.2.3.4 1234 2.3.4.5 80 success unknown -1299633982.99326 1.2.3.4 1234 2.3.4.5 80 failure US +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.40319 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718503.40319 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/logging.ascii-empty/output b/testing/btest/Baseline/logging.ascii-empty/output index 1a7ada05ae..d377ca15d7 100644 --- a/testing/btest/Baseline/logging.ascii-empty/output +++ b/testing/btest/Baseline/logging.ascii-empty/output @@ -1,6 +1,6 @@ -PREFIX<>t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b| -1299718338.37017|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET -1299718338.37017|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET -1299718338.37017|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET -1299718338.37017|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET -1299718338.37017|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T +PREFIX<>t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T diff --git a/testing/btest/Baseline/logging.ascii-escape/ssh.log b/testing/btest/Baseline/logging.ascii-escape/ssh.log index 9892e1aeee15040732851bcfff5b9d27885b195b..6797e1e302234c2b6e90ab6fcf02df8a778a3b97 100644 GIT binary patch delta 97 zcmey$^n+=FgCLiok)@@%p@pe|nVyNcv4O?J333w)#JP|K3{ZFq6JJW82*^#=V3a}; KQD^pz-~s^d5g6|P delta 97 zcmeyt^p$CXgHTNkm!XlRrMaP*xv8O^rKy3*#6VL{B#zv~bwa587gET4g~>XM637BT K4V=ur5nKTO)fpuK diff --git a/testing/btest/Baseline/logging.ascii-options/output b/testing/btest/Baseline/logging.ascii-options/output index 9a3e1f8d20..33a922cc2b 100644 --- a/testing/btest/Baseline/logging.ascii-options/output +++ b/testing/btest/Baseline/logging.ascii-options/output @@ -1,5 +1,5 @@ -1299649281.43936|1.2.3.4|1234|2.3.4.5|80|success|unknown -1299649281.43936|1.2.3.4|1234|2.3.4.5|80|failure|US -1299649281.43936|1.2.3.4|1234|2.3.4.5|80|failure|UK -1299649281.43936|1.2.3.4|1234|2.3.4.5|80|success|BR -1299649281.43936|1.2.3.4|1234|2.3.4.5|80|failure|MX +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|unknown +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|US +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|UK +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|BR +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|MX diff --git a/testing/btest/Baseline/logging.events/output b/testing/btest/Baseline/logging.events/output index 21571f5eee..c3dbf607a6 100644 --- a/testing/btest/Baseline/logging.events/output +++ b/testing/btest/Baseline/logging.events/output @@ -1,2 +1,2 @@ -[t=1298326194.80574, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=] -[t=1298326194.80574, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] +[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=] +[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] diff --git a/testing/btest/Baseline/logging.path-func/output b/testing/btest/Baseline/logging.path-func/output index 2b44fb2948..25e4ca6696 100644 --- a/testing/btest/Baseline/logging.path-func/output +++ b/testing/btest/Baseline/logging.path-func/output @@ -1,13 +1,13 @@ static-prefix-0.log static-prefix-1.log static-prefix-2.log -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 success unknown -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 success BR -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure MX3 -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure US -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure MX -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure UK -1299630368.36038 1.2.3.4 1234 2.3.4.5 80 failure MX2 +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 success BR +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX3 +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX2 diff --git a/testing/btest/Baseline/logging.pred/ssh.failure.log b/testing/btest/Baseline/logging.pred/ssh.failure.log index a5dcba878c..c46990dc65 100644 --- a/testing/btest/Baseline/logging.pred/ssh.failure.log +++ b/testing/btest/Baseline/logging.pred/ssh.failure.log @@ -1,2 +1,2 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.46042 1.2.3.4 1234 2.3.4.5 80 failure US +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.16177 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/logging.pred/ssh.success.log b/testing/btest/Baseline/logging.pred/ssh.success.log index 1402f6f06d..c6adcd86aa 100644 --- a/testing/btest/Baseline/logging.pred/ssh.success.log +++ b/testing/btest/Baseline/logging.pred/ssh.success.log @@ -1,2 +1,2 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.46042 1.2.3.4 1234 2.3.4.5 80 success - +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.16177 1.2.3.4 1234 2.3.4.5 80 success - diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log index 1a9b7f64b1..815b0366b3 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log @@ -1,4 +1,4 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure US -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure UK -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.log b/testing/btest/Baseline/logging.remote/sender.ssh.log index f780e80992..2f159ad6a7 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.log @@ -1,6 +1,6 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success - -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure US -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure UK -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success BR -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success - +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success BR +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.success.log b/testing/btest/Baseline/logging.remote/sender.ssh.success.log index d662d5af9f..2838cc68b8 100644 --- a/testing/btest/Baseline/logging.remote/sender.ssh.success.log +++ b/testing/btest/Baseline/logging.remote/sender.ssh.success.log @@ -1,3 +1,3 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success - -1299630368.80723 1.2.3.4 1234 2.3.4.5 80 success BR +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success - +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/logging.remove/ssh.failure.log b/testing/btest/Baseline/logging.remove/ssh.failure.log index 5628867595..ddbacda28e 100644 --- a/testing/btest/Baseline/logging.remove/ssh.failure.log +++ b/testing/btest/Baseline/logging.remove/ssh.failure.log @@ -1,3 +1,3 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure US -1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure UK +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/logging.remove/ssh.log b/testing/btest/Baseline/logging.remove/ssh.log index 476442eec8..123e8e3a87 100644 --- a/testing/btest/Baseline/logging.remove/ssh.log +++ b/testing/btest/Baseline/logging.remove/ssh.log @@ -1,4 +1,4 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure US -1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure UK -1299630368.57877 1.2.3.4 1234 2.3.4.5 80 failure BR +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure BR diff --git a/testing/btest/Baseline/logging.rotate-custom/out b/testing/btest/Baseline/logging.rotate-custom/out index 423b15651d..8206ec5f9a 100644 --- a/testing/btest/Baseline/logging.rotate-custom/out +++ b/testing/btest/Baseline/logging.rotate-custom/out @@ -29,106 +29,106 @@ 1st test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 2nd test2-11-03-07_04.59.55.log test2.log 11-03-07_04.59.55 11-03-07_04.59.55 1 > test-11-03-06_19.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299466805.0 10.0.0.1 20 10.0.0.2 1024 1299470395.0 10.0.0.2 20 10.0.0.3 0 > test-11-03-06_20.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299470405.0 10.0.0.1 20 10.0.0.2 1025 1299473995.0 10.0.0.2 20 10.0.0.3 1 > test-11-03-06_21.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299474005.0 10.0.0.1 20 10.0.0.2 1026 1299477595.0 10.0.0.2 20 10.0.0.3 2 > test-11-03-06_22.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299477605.0 10.0.0.1 20 10.0.0.2 1027 1299481195.0 10.0.0.2 20 10.0.0.3 3 > test-11-03-06_23.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299481205.0 10.0.0.1 20 10.0.0.2 1028 1299484795.0 10.0.0.2 20 10.0.0.3 4 > test-11-03-07_00.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299484805.0 10.0.0.1 20 10.0.0.2 1029 1299488395.0 10.0.0.2 20 10.0.0.3 5 > test-11-03-07_01.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299488405.0 10.0.0.1 20 10.0.0.2 1030 1299491995.0 10.0.0.2 20 10.0.0.3 6 > test-11-03-07_02.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299492005.0 10.0.0.1 20 10.0.0.2 1031 1299495595.0 10.0.0.2 20 10.0.0.3 7 > test-11-03-07_03.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299495605.0 10.0.0.1 20 10.0.0.2 1032 1299499195.0 10.0.0.2 20 10.0.0.3 8 > test-11-03-07_04.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299499205.0 10.0.0.1 20 10.0.0.2 1033 1299502795.0 10.0.0.2 20 10.0.0.3 9 > test.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p > test2-11-03-06_19.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299466805.0 10.0.0.1 20 10.0.0.2 1024 > test2-11-03-06_19.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299470395.0 10.0.0.2 20 10.0.0.3 0 > test2-11-03-06_20.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299470405.0 10.0.0.1 20 10.0.0.2 1025 > test2-11-03-06_20.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299473995.0 10.0.0.2 20 10.0.0.3 1 > test2-11-03-06_21.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299474005.0 10.0.0.1 20 10.0.0.2 1026 > test2-11-03-06_21.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299477595.0 10.0.0.2 20 10.0.0.3 2 > test2-11-03-06_22.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299477605.0 10.0.0.1 20 10.0.0.2 1027 > test2-11-03-06_22.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299481195.0 10.0.0.2 20 10.0.0.3 3 > test2-11-03-06_23.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299481205.0 10.0.0.1 20 10.0.0.2 1028 > test2-11-03-06_23.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299484795.0 10.0.0.2 20 10.0.0.3 4 > test2-11-03-07_00.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299484805.0 10.0.0.1 20 10.0.0.2 1029 > test2-11-03-07_00.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299488395.0 10.0.0.2 20 10.0.0.3 5 > test2-11-03-07_01.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299488405.0 10.0.0.1 20 10.0.0.2 1030 > test2-11-03-07_01.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299491995.0 10.0.0.2 20 10.0.0.3 6 > test2-11-03-07_02.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299492005.0 10.0.0.1 20 10.0.0.2 1031 > test2-11-03-07_02.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299495595.0 10.0.0.2 20 10.0.0.3 7 > test2-11-03-07_03.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299495605.0 10.0.0.1 20 10.0.0.2 1032 > test2-11-03-07_03.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299499195.0 10.0.0.2 20 10.0.0.3 8 > test2-11-03-07_04.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299499205.0 10.0.0.1 20 10.0.0.2 1033 > test2-11-03-07_04.59.55.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299502795.0 10.0.0.2 20 10.0.0.3 9 > test2.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p diff --git a/testing/btest/Baseline/logging.rotate/out b/testing/btest/Baseline/logging.rotate/out index 27bd5179cf..006cd2c851 100644 --- a/testing/btest/Baseline/logging.rotate/out +++ b/testing/btest/Baseline/logging.rotate/out @@ -9,42 +9,42 @@ test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0 test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0 test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 > test-11-03-06_19.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299466805.0 10.0.0.1 20 10.0.0.2 1024 1299470395.0 10.0.0.2 20 10.0.0.3 0 > test-11-03-06_20.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299470405.0 10.0.0.1 20 10.0.0.2 1025 1299473995.0 10.0.0.2 20 10.0.0.3 1 > test-11-03-06_21.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299474005.0 10.0.0.1 20 10.0.0.2 1026 1299477595.0 10.0.0.2 20 10.0.0.3 2 > test-11-03-06_22.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299477605.0 10.0.0.1 20 10.0.0.2 1027 1299481195.0 10.0.0.2 20 10.0.0.3 3 > test-11-03-06_23.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299481205.0 10.0.0.1 20 10.0.0.2 1028 1299484795.0 10.0.0.2 20 10.0.0.3 4 > test-11-03-07_00.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299484805.0 10.0.0.1 20 10.0.0.2 1029 1299488395.0 10.0.0.2 20 10.0.0.3 5 > test-11-03-07_01.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299488405.0 10.0.0.1 20 10.0.0.2 1030 1299491995.0 10.0.0.2 20 10.0.0.3 6 > test-11-03-07_02.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299492005.0 10.0.0.1 20 10.0.0.2 1031 1299495595.0 10.0.0.2 20 10.0.0.3 7 > test-11-03-07_03.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299495605.0 10.0.0.1 20 10.0.0.2 1032 1299499195.0 10.0.0.2 20 10.0.0.3 8 > test-11-03-07_04.00.05.log -# t id.orig_h id.orig_p id.resp_h id.resp_p +# t id.orig_h id.orig_p id.resp_h id.resp_p 1299499205.0 10.0.0.1 20 10.0.0.2 1033 1299502795.0 10.0.0.2 20 10.0.0.3 9 diff --git a/testing/btest/Baseline/logging.stdout/output b/testing/btest/Baseline/logging.stdout/output index d7dcbd4e48..4c73aed8e4 100644 --- a/testing/btest/Baseline/logging.stdout/output +++ b/testing/btest/Baseline/logging.stdout/output @@ -1,6 +1,6 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299635864.89679 1.2.3.4 1234 2.3.4.5 80 success unknown -1299635864.89679 1.2.3.4 1234 2.3.4.5 80 failure US -1299635864.89679 1.2.3.4 1234 2.3.4.5 80 failure UK -1299635864.89679 1.2.3.4 1234 2.3.4.5 80 success BR -1299635864.89679 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 failure US +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 success BR +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.test-logging/ssh.log b/testing/btest/Baseline/logging.test-logging/ssh.log index d73dc32ec0..82523b7c13 100644 --- a/testing/btest/Baseline/logging.test-logging/ssh.log +++ b/testing/btest/Baseline/logging.test-logging/ssh.log @@ -1,6 +1,6 @@ -# t id.orig_h id.orig_p id.resp_h id.resp_p status country -1299630368.69723 1.2.3.4 1234 2.3.4.5 80 success unknown -1299630368.69723 1.2.3.4 1234 2.3.4.5 80 failure US -1299630368.69723 1.2.3.4 1234 2.3.4.5 80 failure UK -1299630368.69723 1.2.3.4 1234 2.3.4.5 80 success BR -1299630368.69723 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 failure US +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 success BR +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 failure MX From c6e3174bc8686256e71c32be2e468e121dec2610 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 18:01:41 -0800 Subject: [PATCH 49/67] The logging systems now supports fields of type set[]. --- src/LogMgr.cc | 171 +++++++++++++++++-------- src/LogMgr.h | 8 +- src/LogWriterAscii.cc | 142 ++++++++++++-------- src/LogWriterAscii.h | 4 + src/logging.bif | 1 + testing/btest/logging/remote-types.bro | 90 +++++++++++++ testing/btest/logging/types.bro | 57 +++++++++ 7 files changed, 364 insertions(+), 109 deletions(-) create mode 100644 testing/btest/logging/remote-types.bro create mode 100644 testing/btest/logging/types.bro diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0873ca991e..d92493f299 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -63,6 +63,21 @@ struct LogMgr::Stream { ~Stream(); }; +LogVal::~LogVal() + { + if ( type == TYPE_STRING && present ) + delete val.string_val; + + if ( type == TYPE_TABLE && present ) + { + for ( int i = 0; i < val.set_val.size; i++ ) + delete val.set_val.vals[i]; + + delete [] val.set_val.vals; + } + } + + bool LogVal::Read(SerializationFormat* fmt) { int ty; @@ -139,6 +154,23 @@ bool LogVal::Read(SerializationFormat* fmt) return fmt->Read(val.string_val, "string"); } + case TYPE_TABLE: + { + if ( ! fmt->Read(&val.set_val.size, "set_size") ) + return false; + + val.set_val.vals = new LogVal* [val.set_val.size]; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + val.set_val.vals[i] = new LogVal; + if ( ! val.set_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + default: internal_error(::fmt("unsupported type %s in LogVal::Write", type_name(type))); } @@ -205,6 +237,20 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_STRING: return fmt->Write(*val.string_val, "string"); + case TYPE_TABLE: + { + if ( ! fmt->Write(val.set_val.size, "set_size") ) + return false; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + if ( ! val.set_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + default: internal_error(::fmt("unsupported type %s in LogVal::REad", type_name(type))); } @@ -402,14 +448,18 @@ bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, T // Recurse. if ( ! TraverseRecord(filter, t->AsRecordType(), include, exclude, new_path, new_indices) ) return false; + + continue; } - else + else if ( t->Tag() == TYPE_TABLE && t->AsTableType()->IsSet() ) { + // That's ok, handle it with all the other types below. + } + + else { run_time("unsupported field type for log column"); return false; } - - continue; } // If include fields are specified, only include if explicitly listed. @@ -701,6 +751,70 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } +LogVal* LogMgr::ValToLogVal(Val* val) + { + LogVal* lval = new LogVal(val->Type()->Tag()); + + switch ( lval->type ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_ENUM: + lval->val.int_val = val->InternalInt(); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + lval->val.uint_val = val->InternalUnsigned(); + break; + + case TYPE_PORT: + lval->val.uint_val = val->AsPortVal()->Port(); + break; + + case TYPE_SUBNET: + lval->val.subnet_val = *val->AsSubNet(); + break; + + case TYPE_NET: + case TYPE_ADDR: + { + addr_type t = val->AsAddr(); + copy_addr(&t, &lval->val.addr_val); + break; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + lval->val.double_val = val->InternalDouble(); + break; + + case TYPE_STRING: + { + const BroString* s = val->AsString(); + lval->val.string_val = new string((const char*) s->Bytes(), s->Len()); + break; + } + + case TYPE_TABLE: + { + ListVal* set = val->AsTableVal()->ConvertToPureList(); + lval->val.set_val.size = set->Length(); + lval->val.set_val.vals = new LogVal* [lval->val.set_val.size]; + + for ( int i = 0; i < lval->val.set_val.size; i++ ) + lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); + + break; + } + + default: + internal_error("unsupported type for log_write"); + } + + return lval; + } + LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) { LogVal** vals = new LogVal*[filter->num_fields]; @@ -727,55 +841,8 @@ LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) } } - if ( ! val ) - continue; - - vals[i] = new LogVal(type); - - switch ( val->Type()->Tag() ) { - case TYPE_BOOL: - case TYPE_INT: - case TYPE_ENUM: - vals[i]->val.int_val = val->InternalInt(); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - vals[i]->val.uint_val = val->InternalUnsigned(); - break; - - case TYPE_PORT: - vals[i]->val.uint_val = val->AsPortVal()->Port(); - break; - - case TYPE_SUBNET: - vals[i]->val.subnet_val = *val->AsSubNet(); - break; - - case TYPE_NET: - case TYPE_ADDR: - { - addr_type t = val->AsAddr(); - copy_addr(&t, &vals[i]->val.addr_val); - break; - } - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - vals[i]->val.double_val = val->InternalDouble(); - break; - - case TYPE_STRING: - { - const BroString* s = val->AsString(); - vals[i]->val.string_val = new string((const char*) s->Bytes(), s->Len()); - break; - } - - default: - internal_error("unsupported type for log_write"); - } + if ( val ) + vals[i] = ValToLogVal(val); } return vals; diff --git a/src/LogMgr.h b/src/LogMgr.h index 8f8d991c00..88e7ef0131 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -36,17 +36,20 @@ struct LogVal { // The following union is a subset of BroValUnion, including only the // atomic types. - union { + struct set_t { bro_int_t size; LogVal** vals; }; + + union _val { bro_int_t int_val; bro_uint_t uint_val; addr_type addr_val; subnet_type subnet_val; double double_val; string* string_val; + set_t set_val; } val; LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} - ~LogVal() { if ( type == TYPE_STRING && present ) delete val.string_val; } + ~LogVal(); bool Read(SerializationFormat* fmt); bool Write(SerializationFormat* fmt) const; @@ -97,6 +100,7 @@ private: struct WriterInfo; bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); + LogVal* ValToLogVal(Val* val); LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 3ec22b7fa0..48537173b3 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -16,6 +16,10 @@ LogWriterAscii::LogWriterAscii() separator = new char[separator_len]; memcpy(separator, BifConst::LogAscii::separator->Bytes(), separator_len); + set_separator_len = BifConst::LogAscii::set_separator->Len(); + set_separator = new char[set_separator_len]; + memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), set_separator_len); + empty_field_len = BifConst::LogAscii::empty_field->Len(); empty_field = new char[empty_field_len]; memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), empty_field_len); @@ -27,6 +31,7 @@ LogWriterAscii::LogWriterAscii() header_prefix_len = BifConst::LogAscii::header_prefix->Len(); header_prefix = new char[header_prefix_len]; memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(), header_prefix_len); + } LogWriterAscii::~LogWriterAscii() @@ -35,6 +40,7 @@ LogWriterAscii::~LogWriterAscii() fclose(file); delete [] separator; + delete [] set_separator; delete [] empty_field; delete [] unset_field; delete [] header_prefix; @@ -89,6 +95,86 @@ void LogWriterAscii::DoFinish() { } +bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) + { + if ( ! val->present ) + { + desc->AddN(unset_field, unset_field_len); + return true; + } + + switch ( val->type ) { + + case TYPE_BOOL: + desc->Add(val->val.int_val ? "T" : "F"); + break; + + case TYPE_INT: + case TYPE_ENUM: + desc->Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + desc->Add(val->val.uint_val); + break; + + case TYPE_SUBNET: + desc->Add(dotted_addr(val->val.subnet_val.net)); + desc->Add("/"); + desc->Add(val->val.subnet_val.width); + break; + + case TYPE_NET: + case TYPE_ADDR: + desc->Add(dotted_addr(val->val.addr_val)); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + desc->Add(val->val.double_val); + break; + + case TYPE_STRING: + { + int size = val->val.string_val->size(); + if ( size ) + desc->AddN(val->val.string_val->data(), val->val.string_val->size()); + else + desc->AddN(empty_field, empty_field_len); + break; + } + + case TYPE_TABLE: + { + if ( ! val->val.set_val.size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + desc->AddN(set_separator, set_separator_len); + + if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + return false; + } + + break; + } + + default: + Error(Fmt("unsupported field format %d for %s", val->type, field->name.c_str())); + return false; + } + + return true; + } + bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, LogVal** vals) { ODesc desc(DESC_READABLE); @@ -99,63 +185,9 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, Log if ( i > 0 ) desc.AddRaw(separator, separator_len); - LogVal* val = vals[i]; - const LogField* field = fields[i]; - - if ( ! val->present ) - { - desc.AddN(unset_field, unset_field_len); - continue; - } - - switch ( field->type ) { - case TYPE_BOOL: - desc.Add(val->val.int_val ? "T" : "F"); - break; - - case TYPE_INT: - case TYPE_ENUM: - desc.Add(val->val.int_val); - break; - - case TYPE_COUNT: - case TYPE_COUNTER: - case TYPE_PORT: - desc.Add(val->val.uint_val); - break; - - case TYPE_SUBNET: - desc.Add(dotted_addr(val->val.subnet_val.net)); - desc.Add("/"); - desc.Add(val->val.subnet_val.width); - break; - - case TYPE_NET: - case TYPE_ADDR: - desc.Add(dotted_addr(val->val.addr_val)); - break; - - case TYPE_DOUBLE: - case TYPE_TIME: - case TYPE_INTERVAL: - desc.Add(val->val.double_val); - break; - - case TYPE_STRING: - { - int size = val->val.string_val->size(); - if ( size ) - desc.AddN(val->val.string_val->data(), val->val.string_val->size()); - else - desc.AddN(empty_field, empty_field_len); - break; - } - - default: - Error(Fmt("unsupported field format %d for %s", field->type, field->name.c_str())); + if ( ! DoWriteOne(&desc, vals[i], fields[i]) ) return false; } - } desc.Add("\n"); diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h index 384be65f19..fee0d2f1f7 100644 --- a/src/LogWriterAscii.h +++ b/src/LogWriterAscii.h @@ -24,6 +24,7 @@ protected: private: bool IsSpecial(string path) { return path.find("/dev/") == 0; } + bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field); FILE* file; string fname; @@ -35,6 +36,9 @@ private: char* separator; int separator_len; + char* set_separator; + int set_separator_len; + char* empty_field; int empty_field_len; diff --git a/src/logging.bif b/src/logging.bif index 2eb8eea35b..fe2baae782 100644 --- a/src/logging.bif +++ b/src/logging.bif @@ -69,6 +69,7 @@ const output_to_stdout: bool; const include_header: bool; const header_prefix: string; const separator: string; +const set_separator: string; const empty_field: string; const unset_field: string; diff --git a/testing/btest/logging/remote-types.bro b/testing/btest/logging/remote-types.bro new file mode 100644 index 0000000000..57874c7c82 --- /dev/null +++ b/testing/btest/logging/remote-types.bro @@ -0,0 +1,90 @@ +# +# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-diff receiver/ssh.log +# @TEST-EXEC: cmp receiver/ssh.log sender/ssh.log + +# Remote version testing all types. + +# This is the common part loaded by both sender and receiver. + +redef LogAscii::empty_field = "EMPTY"; + +module SSH; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + n: net; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + }; +} + +global log_ssh: event(rec: Log); + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); +} + +##### + +@TEST-START-FILE sender.bro + +module SSH; + +@load listen-clear + +event remote_connection_handshake_done(p: event_peer) + { + local empty_set: set[string]; + + Log::write(SSH, [ + $b=T, + $i=-42, + $e=SSH, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $n=10.0., + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set + ]); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +@load remote + +redef Remote::destinations += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/btest/logging/types.bro b/testing/btest/logging/types.bro new file mode 100644 index 0000000000..c54a1fa9f1 --- /dev/null +++ b/testing/btest/logging/types.bro @@ -0,0 +1,57 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::empty_field = "EMPTY"; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + n: net; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local empty_set: set[string]; + + Log::write(SSH, [ + $b=T, + $i=-42, + $e=SSH, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $n=10.0., + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set + ]); +} + From 89e8ea7353ab2a68461a9ae82404f2e84d50eb7d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 18:06:50 -0800 Subject: [PATCH 50/67] Enums are now logged with their ID names, not anymore with their numerical values. --- policy/logging-ascii.bro | 5 ++++- src/LogMgr.cc | 14 ++++++++++---- src/LogWriterAscii.cc | 2 +- .../Baseline/logging.remote-types/receiver.ssh.log | 2 ++ testing/btest/Baseline/logging.types/ssh.log | 2 ++ 5 files changed, 19 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/logging.remote-types/receiver.ssh.log create mode 100644 testing/btest/Baseline/logging.types/ssh.log diff --git a/policy/logging-ascii.bro b/policy/logging-ascii.bro index 151d29de99..1699fb08a7 100644 --- a/policy/logging-ascii.bro +++ b/policy/logging-ascii.bro @@ -15,7 +15,10 @@ export { # The separator between fields. const separator = "\t" &redef; - # The string to use for empty string fields. + # The separator between set elements. + const set_separator = "," &redef; + + # The string to use for empty fields. const empty_field = "" &redef; # The string to use for an unset optional field. diff --git a/src/LogMgr.cc b/src/LogMgr.cc index d92493f299..0d84e8c233 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -65,7 +65,7 @@ struct LogMgr::Stream { LogVal::~LogVal() { - if ( type == TYPE_STRING && present ) + if ( (type == TYPE_ENUM || type == TYPE_STRING) && present ) delete val.string_val; if ( type == TYPE_TABLE && present ) @@ -93,7 +93,6 @@ bool LogVal::Read(SerializationFormat* fmt) switch ( type ) { case TYPE_BOOL: case TYPE_INT: - case TYPE_ENUM: return fmt->Read(&val.int_val, "int"); case TYPE_COUNT: @@ -148,6 +147,7 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_INTERVAL: return fmt->Read(&val.double_val, "double"); + case TYPE_ENUM: case TYPE_STRING: { val.string_val = new string; @@ -189,7 +189,6 @@ bool LogVal::Write(SerializationFormat* fmt) const switch ( type ) { case TYPE_BOOL: case TYPE_INT: - case TYPE_ENUM: return fmt->Write(val.int_val, "int"); case TYPE_COUNT: @@ -234,6 +233,7 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_INTERVAL: return fmt->Write(val.double_val, "double"); + case TYPE_ENUM: case TYPE_STRING: return fmt->Write(*val.string_val, "string"); @@ -758,10 +758,16 @@ LogVal* LogMgr::ValToLogVal(Val* val) switch ( lval->type ) { case TYPE_BOOL: case TYPE_INT: - case TYPE_ENUM: lval->val.int_val = val->InternalInt(); break; + case TYPE_ENUM: + { + const char* s = val->Type()->AsEnumType()->Lookup(val->InternalInt()); + lval->val.string_val = new string(s); + break; + } + case TYPE_COUNT: case TYPE_COUNTER: lval->val.uint_val = val->InternalUnsigned(); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 48537173b3..e959bcb150 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -110,7 +110,6 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) break; case TYPE_INT: - case TYPE_ENUM: desc->Add(val->val.int_val); break; @@ -137,6 +136,7 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) desc->Add(val->val.double_val); break; + case TYPE_ENUM: case TYPE_STRING: { int size = val->val.string_val->size(); diff --git a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log new file mode 100644 index 0000000000..5628efda11 --- /dev/null +++ b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log @@ -0,0 +1,2 @@ +# b i e c p sn n a d t iv s sc ss se +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299722788.39808 100.0 hurz 4,2,3,1 AA,BB,CC EMPTY diff --git a/testing/btest/Baseline/logging.types/ssh.log b/testing/btest/Baseline/logging.types/ssh.log new file mode 100644 index 0000000000..56bd94616d --- /dev/null +++ b/testing/btest/Baseline/logging.types/ssh.log @@ -0,0 +1,2 @@ +# b i e c p sn n a d t iv s sc ss se +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299722790.49273 100.0 hurz 2,1,4,3 BB,AA,CC EMPTY From 5beee9e45e20b5244e1e158c960dfb99aa3447c5 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 18:08:17 -0800 Subject: [PATCH 51/67] Updating TODO. Just one to go. :) --- TODO.logging | 4 ---- 1 file changed, 4 deletions(-) diff --git a/TODO.logging b/TODO.logging index 2e6c93b392..655d91510f 100644 --- a/TODO.logging +++ b/TODO.logging @@ -1,10 +1,6 @@ List of the things not implemented yet: - - Not sure if the logging does the right thing with &optional and - &default values. Needs testing. - Check the new event-value code. - - The Ascii writer doesn't escape the delimiter if it appears - within a field's value. Seems we need to do that. Notes about remote logging: From de227b8d8849d8d37ec13815eda8816b8aeac253 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 19:20:06 -0800 Subject: [PATCH 52/67] When creating a new stream, we check now that all log field have supported types. Also not reporting a run-time error anymore when logging to a stream that hasn't been created; just fail silently as this may happen due to other earlier errors. --- src/LogMgr.cc | 52 ++++++++++++++++++++++++++++++++++++++++++++++++++- src/LogMgr.h | 2 ++ 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 0d84e8c233..1fe9ff6c2b 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -77,7 +77,48 @@ LogVal::~LogVal() } } +bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) + { + if ( ! t ) + return false; + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_NET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + return true; + + case TYPE_RECORD: + return ! atomic_only; + + case TYPE_TABLE: + { + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } + + default: + return false; + } + + return false; + } + bool LogVal::Read(SerializationFormat* fmt) { int ty; @@ -302,7 +343,7 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id) if ( idx >= streams.size() || ! streams[idx] ) { - run_time(fmt("unknown log stream (%d)", id->AsEnum())); + // run_time(fmt("unknown log stream (%d)", id->AsEnum())); return 0; } @@ -338,6 +379,15 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) RecordType* columns = sval->Lookup(rtype->FieldOffset("columns"))->AsType()->AsTypeType()->Type()->AsRecordType(); + for ( int i = 0; i < columns->NumFields(); i++ ) + { + if ( ! LogVal::IsCompatibleType(columns->FieldType(i)) ) + { + run_time("type of field '%s' is not support for logging output", columns->FieldName(i)); + return false; + } + } + Val* event_val = sval->Lookup(rtype->FieldOffset("ev")); Func* event = event_val ? event_val->AsFunc() : 0; diff --git a/src/LogMgr.h b/src/LogMgr.h index 88e7ef0131..2a00ac6fb0 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -54,6 +54,8 @@ struct LogVal { bool Read(SerializationFormat* fmt); bool Write(SerializationFormat* fmt) const; + static bool IsCompatibleType(BroType* t, bool atomic_only=false); + private: LogVal(const LogVal& other) { } }; From d54c70589894f96ea08ead9222d2c274bd2e11d0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 19:25:41 -0800 Subject: [PATCH 53/67] Adding new environment variable BRO_SEED_FILE to set the seed file for the random number generator. This works like the corresponding command line option but is more convinient when writing tests as it can be set in btest.cfg. --- src/main.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main.cc b/src/main.cc index 06456f9765..22f8ee6976 100644 --- a/src/main.cc +++ b/src/main.cc @@ -196,6 +196,7 @@ void usage() fprintf(stderr, " $BROPATH | file search path (%s)\n", bro_path()); fprintf(stderr, " $BRO_PREFIXES | prefix list (%s)\n", bro_prefixes()); fprintf(stderr, " $BRO_DNS_FAKE | disable DNS lookups (%s)\n", bro_dns_fake()); + fprintf(stderr, " $BRO_SEED_FILE | file to load seeds from (not set)\n"); exit(1); } @@ -350,7 +351,7 @@ int main(int argc, char** argv) char* bst_file = 0; char* id_name = 0; char* events_file = 0; - char* seed_load_file = 0; + char* seed_load_file = getenv("BRO_SEED_FILE"); char* seed_save_file = 0; int seed = 0; int dump_cfg = false; From 170a8bd40323f789586f62139aeb5b9e9e48bf48 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 19:27:03 -0800 Subject: [PATCH 54/67] Setting random number generator seed in btest.cfg. --- testing/btest/btest.cfg | 1 + testing/btest/random.seed | 17 +++++++++++++++++ 2 files changed, 18 insertions(+) create mode 100644 testing/btest/random.seed diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index a49fc6af02..6a5cf94b86 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -8,5 +8,6 @@ IgnoreFiles = *.tmp *.swp #* *.trace [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` +BRO_SEED_FILE=%(testbase)s/random.seed PATH=%(testbase)s/../../build/src:%(testbase)s/../../aux/btest:%(default_path)s TEST_DIFF_CANONIFIER=%(testbase)s/Scripts/diff-canonifier diff --git a/testing/btest/random.seed b/testing/btest/random.seed new file mode 100644 index 0000000000..e70c8f85ef --- /dev/null +++ b/testing/btest/random.seed @@ -0,0 +1,17 @@ +2983378351 +1299727368 +0 +310447 +0 +1409073626 +3975311262 +34130240 +1450515018 +1466150520 +1342286698 +1193956778 +2188527278 +3361989254 +3912865238 +3596260151 +517973768 From 871561939b3f91d3b930a19ee9f48e691c4865dd Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 9 Mar 2011 19:32:13 -0800 Subject: [PATCH 55/67] Updating baselines. --- testing/btest/Baseline/logging.remote-types/receiver.ssh.log | 2 +- testing/btest/Baseline/logging.types/ssh.log | 2 +- testing/btest/logging/remote-types.bro | 4 +--- testing/btest/logging/remote.bro | 4 +--- 4 files changed, 4 insertions(+), 8 deletions(-) diff --git a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log index 5628efda11..bd90306440 100644 --- a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log +++ b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log @@ -1,2 +1,2 @@ # b i e c p sn n a d t iv s sc ss se -T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299722788.39808 100.0 hurz 4,2,3,1 AA,BB,CC EMPTY +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299727902.65854 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY diff --git a/testing/btest/Baseline/logging.types/ssh.log b/testing/btest/Baseline/logging.types/ssh.log index 56bd94616d..b20a2fca37 100644 --- a/testing/btest/Baseline/logging.types/ssh.log +++ b/testing/btest/Baseline/logging.types/ssh.log @@ -1,2 +1,2 @@ # b i e c p sn n a d t iv s sc ss se -T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299722790.49273 100.0 hurz 2,1,4,3 BB,AA,CC EMPTY +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299727493.47095 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY diff --git a/testing/btest/logging/remote-types.bro b/testing/btest/logging/remote-types.bro index 57874c7c82..a3dbe1f163 100644 --- a/testing/btest/logging/remote-types.bro +++ b/testing/btest/logging/remote-types.bro @@ -38,11 +38,9 @@ export { }; } -global log_ssh: event(rec: Log); - event bro_init() { - Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); + Log::create_stream(SSH, [$columns=Log]); } ##### diff --git a/testing/btest/logging/remote.bro b/testing/btest/logging/remote.bro index 18fb7e4f2e..3ff8527cc9 100644 --- a/testing/btest/logging/remote.bro +++ b/testing/btest/logging/remote.bro @@ -28,11 +28,9 @@ export { }; } -global log_ssh: event(rec: Log); - event bro_init() { - Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); + Log::create_stream(SSH, [$columns=Log]); Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); } From 45ebfbb2b8948c06fb98436a160b507f8333c11d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 10 Mar 2011 17:53:09 -0800 Subject: [PATCH 56/67] Bug fixes. - Fixing a crash with an invalid pointer. - Fixing a namespacing problem with is_ftp_data_conn() and check_relay_3(). - Fixing the do-we-have-an-event-handler-defined check. Standard test-suite passes. Seth, I think you can give it a try now ... --- policy/conn.bro | 6 ++++-- policy/firewall.bro | 2 +- policy/mime.bro | 17 +++++++++-------- src/EventHandler.cc | 5 +++++ src/EventHandler.h | 3 +-- src/Expr.cc | 5 ++++- src/Func.h | 4 +++- src/Type.cc | 4 ++-- 8 files changed, 29 insertions(+), 17 deletions(-) diff --git a/policy/conn.bro b/policy/conn.bro index 134ac9db13..b0301c3fbb 100644 --- a/policy/conn.bro +++ b/policy/conn.bro @@ -14,7 +14,9 @@ const conn_closed = { TCP_CLOSED, TCP_RESET }; global have_FTP = F; # if true, we've loaded ftp.bro global have_SMTP = F; # if true, we've loaded smtp.bro -global is_ftp_data_conn: function(c: connection): bool; + +# TODO: Do we have a nicer way of doing this? +export { global FTP::is_ftp_data_conn: function(c: connection): bool; } # Whether to include connection state history in the logs generated # by record_connection. @@ -186,7 +188,7 @@ function determine_service_non_DPD(c: connection) : string return i; # return first; } - else if ( have_FTP && is_ftp_data_conn(c) ) + else if ( have_FTP && FTP::is_ftp_data_conn(c) ) return port_names[20/tcp]; else if ( [c$id$resp_h, c$id$resp_p] in RPC_server_map ) diff --git a/policy/firewall.bro b/policy/firewall.bro index adbf7a450c..59a92206b4 100644 --- a/policy/firewall.bro +++ b/policy/firewall.bro @@ -151,7 +151,7 @@ function do_match(c: connection, r: rule): bool return F; } - if ( r$is_ftp && ! is_ftp_data_conn(c) ) + if ( r$is_ftp && ! FTP::is_ftp_data_conn(c) ) return F; return T; diff --git a/policy/mime.bro b/policy/mime.bro index 7f923aac08..73a4ecefee 100644 --- a/policy/mime.bro +++ b/policy/mime.bro @@ -74,13 +74,14 @@ function mime_header_subject(session: mime_session_info, ### This is a bit clunky. These are functions we call out to, defined # elsewhere. The way we really ought to do this is to have them passed # in during initialization. But for now, we presume knowledge of their -# names in global scope. -module GLOBAL; -global check_relay_3: - function(session: MIME::mime_session_info, msg_id: string); -global check_relay_4: - function(session: MIME::mime_session_info, content_hash: string); -module MIME; +# names. +export + { + global SMTP::check_relay_3: + function(session: MIME::mime_session_info, msg_id: string); + global SMTP::check_relay_4: + function(session: MIME::mime_session_info, content_hash: string); + } function mime_header_message_id(session: mime_session_info, name: string, arg: string) { @@ -107,7 +108,7 @@ function mime_header_message_id(session: mime_session_info, name: string, arg: s s = t[1]; if ( session$level == 1 && SMTP::process_smtp_relay ) - check_relay_3(session, s); + SMTP::check_relay_3(session, s); } redef mime_header_handler = { diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 9df18ab007..9cc5306c9c 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -23,6 +23,11 @@ EventHandler::~EventHandler() delete [] group; } +EventHandler::operator bool() const + { + return enabled && ((local && local->HasBodies()) || receivers.length()); + } + FuncType* EventHandler::FType() { if ( type ) diff --git a/src/EventHandler.h b/src/EventHandler.h index de4d1a1458..4320134d50 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -34,8 +34,7 @@ public: void Call(val_list* vl, bool no_remote = false); // Returns true if there is at least one local or remote handler. - operator bool() const - { return enabled && (local || receivers.length()); } + operator bool() const; void SetUsed() { used = true; } bool Used() { return used; } diff --git a/src/Expr.cc b/src/Expr.cc index 0f186b1f99..1a4c42cf95 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4030,7 +4030,10 @@ Val* RecordCoerceExpr::Fold(Val* v) const for ( int i = 0; i < map_size; ++i ) { if ( map[i] >= 0 ) - val->Assign(i, rv->Lookup(map[i])->Ref()); + { + Val* v = rv->Lookup(map[i]); + val->Assign(i, v ? v->Ref() : 0); + } else val->Assign(i, 0); } diff --git a/src/Func.h b/src/Func.h index f82b9ee539..b6cfab893b 100644 --- a/src/Func.h +++ b/src/Func.h @@ -15,6 +15,7 @@ class FuncType; class Stmt; class Frame; class ID; +class CallExpr; class Func : public BroObj { public: @@ -36,7 +37,8 @@ public: { return priority > other.priority; } // reverse sort }; - virtual const vector& GetBodies() const { return bodies; } + const vector& GetBodies() const { return bodies; } + bool HasBodies() const { return bodies.size(); } // virtual Val* Call(ListExpr* args) const = 0; virtual Val* Call(val_list* args, Frame* parent = 0) const = 0; diff --git a/src/Type.cc b/src/Type.cc index b201373453..810a08b443 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1463,9 +1463,9 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) int same_attrs(const Attributes* a1, const Attributes* a2) { if ( ! a1 ) - return (a2 != 0); + return (a2 == 0); - if ( ! a2 ) + if ( a2 ) return 0; return (*a1 == *a2); From 0f854315e922502aacbb21c43af711092177c102 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 10 Mar 2011 18:13:17 -0800 Subject: [PATCH 57/67] New test. --- .../Baseline/logging.empty-event/ssh.log | 6 ++++ testing/btest/logging/empty-event.bro | 33 +++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 testing/btest/Baseline/logging.empty-event/ssh.log create mode 100644 testing/btest/logging/empty-event.bro diff --git a/testing/btest/Baseline/logging.empty-event/ssh.log b/testing/btest/Baseline/logging.empty-event/ssh.log new file mode 100644 index 0000000000..7f21430ea7 --- /dev/null +++ b/testing/btest/Baseline/logging.empty-event/ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 success unknown +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 failure US +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 failure UK +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 success BR +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/logging/empty-event.bro b/testing/btest/logging/empty-event.bro new file mode 100644 index 0000000000..977450b40f --- /dev/null +++ b/testing/btest/logging/empty-event.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + }; +} + +global log_ssh: event(rec: Log); + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From 7526058071c2fe9324aef43f472ffda78278c076 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 17 Mar 2011 17:19:46 -0700 Subject: [PATCH 58/67] Fixing a bug with nested record ctors. If a record field was initialized with another record ctor, there was no record type coercion in place in case of a type mismatch. --- src/Expr.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/Expr.cc b/src/Expr.cc index 3be0bdb592..541689df5b 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3316,8 +3316,16 @@ Val* RecordConstructorExpr::InitVal(const BroType* t, Val* aggr) const break; } - else - ar->Assign(t_i, rv->Lookup(i)->Ref()); + if ( ar_t->FieldType(t_i)->Tag() == TYPE_RECORD + && ! same_type(ar_t->FieldType(t_i), rv->Lookup(i)->Type()) ) + { + Expr* rhs = new ConstExpr(rv->Lookup(i)->Ref()); + Expr* e = new RecordCoerceExpr(rhs, ar_t->FieldType(t_i)->AsRecordType()); + ar->Assign(t_i, e->Eval(0)); + break; + } + + ar->Assign(t_i, rv->Lookup(i)->Ref()); } for ( i = 0; i < ar_t->NumFields(); ++i ) From 38a1aa5a346d10de32f9b40e0869cdb48a98974b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 28 Mar 2011 17:55:41 -0700 Subject: [PATCH 59/67] &log keyword, and vector logging. The &log keyword now operates as discussed: - When associated with individual record fields, it defines them as being logged. - When associated with a complete record type, it defines all fields to be logged. - When associated with a record extension, it defines all added fields to be logged. Note that for nested record types, the inner fields must likewise be declared with &log. Consequently, conn_id is now declared with &log in bro.init. Vectors are now allowed to be logged and will be recorded as an ordered set of items. --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- policy/bro.init | 2 +- src/Attr.cc | 6 + src/Attr.h | 1 + src/ID.cc | 19 +++ src/LogMgr.cc | 114 ++++++++++++++++-- src/LogMgr.h | 16 +-- src/LogWriterAscii.cc | 20 +++ src/parse.y | 6 +- src/scan.l | 1 + testing/btest/Baseline/logging.attr/ssh.log | 6 + .../logging.remote-types/receiver.ssh.log | 4 +- .../btest/Baseline/logging.rotate-custom/out | 4 +- testing/btest/Baseline/logging.types/ssh.log | 4 +- testing/btest/Baseline/logging.vec/ssh.log | 2 + testing/btest/logging/adapt-filter.bro | 2 +- testing/btest/logging/ascii-empty.bro | 2 +- testing/btest/logging/ascii-escape.bro | 2 +- testing/btest/logging/ascii-options.bro | 2 +- testing/btest/logging/attr-extend.bro | 37 ++++++ testing/btest/logging/attr.bro | 31 +++++ testing/btest/logging/disable-stream.bro | 2 +- testing/btest/logging/empty-event.bro | 2 +- testing/btest/logging/events.bro | 2 +- testing/btest/logging/no-local.bro | 2 +- testing/btest/logging/path-func.bro | 2 +- testing/btest/logging/pred.bro | 2 +- testing/btest/logging/remote-types.bro | 9 +- testing/btest/logging/remote.bro | 2 +- testing/btest/logging/remove.bro | 2 +- testing/btest/logging/rotate-custom.bro | 4 +- testing/btest/logging/rotate.bro | 2 +- testing/btest/logging/stdout.bro | 2 +- testing/btest/logging/test-logging.bro | 2 +- testing/btest/logging/types.bro | 9 +- testing/btest/logging/vec.bro | 27 +++++ 40 files changed, 307 insertions(+), 55 deletions(-) create mode 100644 testing/btest/Baseline/logging.attr/ssh.log create mode 100644 testing/btest/Baseline/logging.vec/ssh.log create mode 100644 testing/btest/logging/attr-extend.bro create mode 100644 testing/btest/logging/attr.bro create mode 100644 testing/btest/logging/vec.bro diff --git a/aux/binpac b/aux/binpac index c3c7ef0dfd..26d02716f9 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit c3c7ef0dfddb0746d3762e41086ba42928e68483 +Subproject commit 26d02716f9090651f319a4bfdf8ede49b3a7b53a diff --git a/aux/bro-aux b/aux/bro-aux index 98f92eeb40..7e50bac938 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 98f92eeb40281045159097764abddc428fb49bf2 +Subproject commit 7e50bac938af1831ecf9660159145a3c2e77e13d diff --git a/aux/broccoli b/aux/broccoli index 48d473398e..9332ab3467 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 48d473398e577893b6c7f77d605ccdf266a2f93b +Subproject commit 9332ab3467191ac22be09d6941ebd469e7a334d0 diff --git a/aux/broctl b/aux/broctl index 532dcd5aa5..06b74a0f23 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 532dcd5aa51c8b29b2d71cd37e1d7c21e33cc715 +Subproject commit 06b74a0f23767c8345ed146657120aba812f6764 diff --git a/aux/btest b/aux/btest index a2b04952ae..acf25f34ab 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit a2b04952ae91dcd27d5e68a42d5d26c291ecb1f5 +Subproject commit acf25f34ab48568f5db919c9e3505ed319daa4e5 diff --git a/policy/bro.init b/policy/bro.init index b3f78f0689..94f9b519db 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -21,7 +21,7 @@ type conn_id: record { orig_p: port; resp_h: addr; resp_p: port; -}; +} &log; type icmp_conn: record { orig_h: addr; diff --git a/src/Attr.cc b/src/Attr.cc index 98be9a9465..47cf1e9180 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -7,6 +7,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" +#include "LogMgr.h" const char* attr_name(attr_tag t) { @@ -332,6 +333,11 @@ void Attributes::CheckAttr(Attr* a) } break; + case ATTR_LOG: + if ( ! LogVal::IsCompatibleType(type) ) + Error("&log applied to a type that cannot be logged"); + break; + default: BadTag("Attributes::CheckAttr", attr_name(a->Tag())); } diff --git a/src/Attr.h b/src/Attr.h index 26231baeb4..b8602bd745 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_MERGEABLE, ATTR_PRIORITY, ATTR_GROUP, + ATTR_LOG, ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; diff --git a/src/ID.cc b/src/ID.cc index c908a8c3c4..0155d50f67 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -235,6 +235,25 @@ void ID::UpdateValAttrs() } } } + + if ( Type()->Tag() == TYPE_RECORD ) + { + Attr* attr = attrs->FindAttr(ATTR_LOG); + if ( attr ) + { + // Apply &log to all record fields. + RecordType* rt = Type()->AsRecordType(); + for ( int i = 0; i < rt->NumFields(); ++i ) + { + TypeDecl* fd = rt->FieldDecl(i); + + if ( ! fd->attrs ) + fd->attrs = new Attributes(new attr_list, rt->FieldType(i)); + + fd->attrs->AddAttr(new Attr(ATTR_LOG)); + } + } + } } void ID::AddAttrs(Attributes* a) diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 1fe9ff6c2b..14637f3978 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -75,6 +75,14 @@ LogVal::~LogVal() delete [] val.set_val.vals; } + + if ( type == TYPE_VECTOR && present ) + { + for ( int i = 0; i < val.vector_val.size; i++ ) + delete val.vector_val.vals[i]; + + delete [] val.vector_val.vals; + } } bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) @@ -112,6 +120,14 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) return IsCompatibleType(t->AsSetType()->Indices()->PureType()); } + case TYPE_VECTOR: + { + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType()); + } + default: return false; } @@ -149,7 +165,7 @@ bool LogVal::Read(SerializationFormat* fmt) && fmt->Read(&net[2], "net2") && fmt->Read(&net[3], "net3") && fmt->Read(&val.subnet_val.width, "width")) ) - return false; + return false; #ifdef BROv6 val.subnet_val.net[0] = net[0]; @@ -212,8 +228,25 @@ bool LogVal::Read(SerializationFormat* fmt) return true; } + case TYPE_VECTOR: + { + if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) + return false; + + val.vector_val.vals = new LogVal* [val.vector_val.size]; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + val.vector_val.vals[i] = new LogVal; + if ( ! val.vector_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + default: - internal_error(::fmt("unsupported type %s in LogVal::Write", type_name(type))); + internal_error("unsupported type %s in LogVal::Write", type_name(type)); } return false; @@ -292,8 +325,22 @@ bool LogVal::Write(SerializationFormat* fmt) const return true; } + case TYPE_VECTOR: + { + if ( ! fmt->Write(val.vector_val.size, "vector_size") ) + return false; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + if ( ! val.vector_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + default: - internal_error(::fmt("unsupported type %s in LogVal::REad", type_name(type))); + internal_error("unsupported type %s in LogVal::REad", type_name(type)); } return false; @@ -379,13 +426,26 @@ bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) RecordType* columns = sval->Lookup(rtype->FieldOffset("columns"))->AsType()->AsTypeType()->Type()->AsRecordType(); + bool log_attr_present = false; + for ( int i = 0; i < columns->NumFields(); i++ ) { + if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) + continue; + if ( ! LogVal::IsCompatibleType(columns->FieldType(i)) ) { run_time("type of field '%s' is not support for logging output", columns->FieldName(i)); return false; } + + log_attr_present = true; + } + + if ( ! log_attr_present ) + { + run_time("logged record type does not have any &log attributes"); + return false; } Val* event_val = sval->Lookup(rtype->FieldOffset("ev")); @@ -473,12 +533,16 @@ bool LogMgr::DisableStream(EnumVal* id) } // Helper for recursive record field unrolling. -bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) +bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) { for ( int i = 0; i < rt->NumFields(); ++i ) { BroType* t = rt->FieldType(i); + // Ignore if &log not specified. + if ( ! rt->FieldDecl(i)->FindAttr(ATTR_LOG) ) + continue; + list new_indices = indices; new_indices.push_back(i); @@ -496,16 +560,22 @@ bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, T if ( t->Tag() == TYPE_RECORD ) { // Recurse. - if ( ! TraverseRecord(filter, t->AsRecordType(), include, exclude, new_path, new_indices) ) + if ( ! TraverseRecord(stream, filter, t->AsRecordType(), include, exclude, new_path, new_indices) ) return false; continue; } + else if ( t->Tag() == TYPE_TABLE && t->AsTableType()->IsSet() ) { // That's ok, handle it with all the other types below. } + else if ( t->Tag() == TYPE_VECTOR ) + { + // That's ok, handle it with all the other types below. + } + else { run_time("unsupported field type for log column"); return false; @@ -595,7 +665,7 @@ bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) filter->num_fields = 0; filter->fields = 0; - if ( ! TraverseRecord(filter, stream->columns, include ? include->AsTableVal() : 0, exclude ? exclude->AsTableVal() : 0, "", list()) ) + if ( ! TraverseRecord(stream, filter, stream->columns, include ? include->AsTableVal() : 0, exclude ? exclude->AsTableVal() : 0, "", list()) ) return false; // Get the path for the filter. @@ -780,7 +850,7 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) // Alright, can do the write now. - LogVal** vals = RecordToFilterVals(filter, columns); + LogVal** vals = RecordToFilterVals(stream, filter, columns); if ( filter->remote ) remote_serializer->SendLogWrite(stream->id, filter->writer, path, filter->num_fields, vals); @@ -801,9 +871,15 @@ bool LogMgr::Write(EnumVal* id, RecordVal* columns) return true; } -LogVal* LogMgr::ValToLogVal(Val* val) +LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) { - LogVal* lval = new LogVal(val->Type()->Tag()); + if ( ! ty ) + ty = val->Type(); + + if ( ! val ) + return new LogVal(ty->Tag(), false); + + LogVal* lval = new LogVal(ty->Tag()); switch ( lval->type ) { case TYPE_BOOL: @@ -864,14 +940,26 @@ LogVal* LogMgr::ValToLogVal(Val* val) break; } - default: - internal_error("unsupported type for log_write"); + case TYPE_VECTOR: + { + VectorVal* vec = val->AsVectorVal(); + lval->val.vector_val.size = vec->Size(); + lval->val.vector_val.vals = new LogVal* [lval->val.vector_val.size]; + + for ( int i = 0; i < lval->val.vector_val.size; i++ ) + lval->val.vector_val.vals[i] = ValToLogVal(vec->Lookup(VECTOR_MIN + i), vec->Type()->YieldType()); + + break; } + default: + internal_error("unsupported type for log_write"); + } + return lval; } -LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns) +LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns) { LogVal** vals = new LogVal*[filter->num_fields]; @@ -921,7 +1009,7 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, int n // Need to instantiate a new writer. - LogWriterDefinition* ld = log_writers; + LogWriterDefinition* ld = log_writers; while ( true ) { diff --git a/src/LogMgr.h b/src/LogMgr.h index 2a00ac6fb0..8875541549 100644 --- a/src/LogMgr.h +++ b/src/LogMgr.h @@ -37,15 +37,17 @@ struct LogVal { // The following union is a subset of BroValUnion, including only the // atomic types. struct set_t { bro_int_t size; LogVal** vals; }; + typedef set_t vec_t; union _val { bro_int_t int_val; bro_uint_t uint_val; addr_type addr_val; subnet_type subnet_val; - double double_val; + double double_val; string* string_val; set_t set_val; + vec_t vector_val; } val; LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) : type(arg_type), present(arg_present) {} @@ -75,11 +77,11 @@ public: bool EnableStream(EnumVal* id); bool DisableStream(EnumVal* id); bool AddFilter(EnumVal* id, RecordVal* filter); - bool RemoveFilter(EnumVal* id, StringVal* name); + bool RemoveFilter(EnumVal* id, StringVal* name); bool RemoveFilter(EnumVal* id, string name); bool Write(EnumVal* id, RecordVal* columns); - bool SetBuf(EnumVal* id, bool enabled); // Changes the state for all writers for that stream. - bool Flush(EnumVal* id); // Flushes all writers for the stream. + bool SetBuf(EnumVal* id, bool enabled); // Changes the state for all writers for that stream. + bool Flush(EnumVal* id); // Flushes all writers for the stream. protected: friend class LogWriter; @@ -101,9 +103,9 @@ private: struct Stream; struct WriterInfo; - bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); - LogVal* ValToLogVal(Val* val); - LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns); + bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices); + LogVal* ValToLogVal(Val* val, BroType* ty = 0); + LogVal** RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns); Stream* FindStream(EnumVal* id); void RemoveDisabledWriters(Stream* stream); void InstallRotationTimer(WriterInfo* winfo); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index e959bcb150..034fdac015 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -167,6 +167,26 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) break; } + case TYPE_VECTOR: + { + if ( ! val->val.vector_val.size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + desc->AddN(set_separator, set_separator_len); + + if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + return false; + } + + break; + } + default: Error(Fmt("unsupported field format %d for %s", val->type, field->name.c_str())); return false; diff --git a/src/parse.y b/src/parse.y index 300c338b5c..18ccfff202 100644 --- a/src/parse.y +++ b/src/parse.y @@ -3,7 +3,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 71 +%expect 74 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,7 +24,7 @@ %token TOK_ATTR_EXPIRE_CREATE TOK_ATTR_EXPIRE_READ TOK_ATTR_EXPIRE_WRITE %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE -%token TOK_ATTR_PRIORITY TOK_ATTR_GROUP +%token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG %token TOK_DEBUG @@ -1034,6 +1034,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_LOG + { $$ = new Attr(ATTR_LOG); } ; stmt: diff --git a/src/scan.l b/src/scan.l index 9dc4d828e0..63ebb49e3d 100644 --- a/src/scan.l +++ b/src/scan.l @@ -196,6 +196,7 @@ when return TOK_WHEN; &encrypt return TOK_ATTR_ENCRYPT; &expire_func return TOK_ATTR_EXPIRE_FUNC; &group return TOK_ATTR_GROUP; +&log return TOK_ATTR_LOG; &mergeable return TOK_ATTR_MERGEABLE; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; diff --git a/testing/btest/Baseline/logging.attr/ssh.log b/testing/btest/Baseline/logging.attr/ssh.log new file mode 100644 index 0000000000..c4355d2fd5 --- /dev/null +++ b/testing/btest/Baseline/logging.attr/ssh.log @@ -0,0 +1,6 @@ +# status country +success unknown +failure US +failure UK +success BR +failure MX diff --git a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log index bd90306440..a7818428a7 100644 --- a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log +++ b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log @@ -1,2 +1,2 @@ -# b i e c p sn n a d t iv s sc ss se -T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299727902.65854 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY +# b i e c p sn n a d t iv s sc ss se vc ve +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1301360085.98852 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/logging.rotate-custom/out b/testing/btest/Baseline/logging.rotate-custom/out index 8206ec5f9a..e270b6a773 100644 --- a/testing/btest/Baseline/logging.rotate-custom/out +++ b/testing/btest/Baseline/logging.rotate-custom/out @@ -68,8 +68,6 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p 1299499205.0 10.0.0.1 20 10.0.0.2 1033 1299502795.0 10.0.0.2 20 10.0.0.3 9 -> test.log -# t id.orig_h id.orig_p id.resp_h id.resp_p > test2-11-03-06_19.00.05.log # t id.orig_h id.orig_p id.resp_h id.resp_p 1299466805.0 10.0.0.1 20 10.0.0.2 1024 @@ -132,3 +130,5 @@ 1299502795.0 10.0.0.2 20 10.0.0.3 9 > test2.log # t id.orig_h id.orig_p id.resp_h id.resp_p +> test.log +# t id.orig_h id.orig_p id.resp_h id.resp_p diff --git a/testing/btest/Baseline/logging.types/ssh.log b/testing/btest/Baseline/logging.types/ssh.log index b20a2fca37..ee657ab462 100644 --- a/testing/btest/Baseline/logging.types/ssh.log +++ b/testing/btest/Baseline/logging.types/ssh.log @@ -1,2 +1,2 @@ -# b i e c p sn n a d t iv s sc ss se -T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1299727493.47095 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY +# b i e c p sn n a d t iv s sc ss se vc ve +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1301359781.8203 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/logging.vec/ssh.log b/testing/btest/Baseline/logging.vec/ssh.log new file mode 100644 index 0000000000..1602f7d1c0 --- /dev/null +++ b/testing/btest/Baseline/logging.vec/ssh.log @@ -0,0 +1,2 @@ +# vec +-,2,-,-,5 diff --git a/testing/btest/logging/adapt-filter.bro b/testing/btest/logging/adapt-filter.bro index 393f46eba9..6312e4c106 100644 --- a/testing/btest/logging/adapt-filter.bro +++ b/testing/btest/logging/adapt-filter.bro @@ -18,7 +18,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/ascii-empty.bro b/testing/btest/logging/ascii-empty.bro index 62022107f0..9e91ebc089 100644 --- a/testing/btest/logging/ascii-empty.bro +++ b/testing/btest/logging/ascii-empty.bro @@ -19,7 +19,7 @@ export { status: string &optional; country: string &default="unknown"; b: bool &optional; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/ascii-escape.bro b/testing/btest/logging/ascii-escape.bro index f5c50c60bd..7acea46250 100644 --- a/testing/btest/logging/ascii-escape.bro +++ b/testing/btest/logging/ascii-escape.bro @@ -14,7 +14,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/ascii-options.bro b/testing/btest/logging/ascii-options.bro index 747249342b..3f33c17b96 100644 --- a/testing/btest/logging/ascii-options.bro +++ b/testing/btest/logging/ascii-options.bro @@ -16,7 +16,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/attr-extend.bro b/testing/btest/logging/attr-extend.bro new file mode 100644 index 0000000000..f6964a749e --- /dev/null +++ b/testing/btest/logging/attr-extend.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; + status: string &optional &log; + country: string &default="unknown" &log; + }; +} + +redef Log += record { + a1: count &log; + a2: count; +}; + +redef Log += record { + b1: count; + b2: count; +} &log; + + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", a1=1, a2=2, a3=3, a4=4]); +} + diff --git a/testing/btest/logging/attr.bro b/testing/btest/logging/attr.bro new file mode 100644 index 0000000000..658dcae04b --- /dev/null +++ b/testing/btest/logging/attr.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; + status: string &optional &log; + country: string &default="unknown" &log; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/disable-stream.bro b/testing/btest/logging/disable-stream.bro index 97b47298e9..472766d8e0 100644 --- a/testing/btest/logging/disable-stream.bro +++ b/testing/btest/logging/disable-stream.bro @@ -12,7 +12,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/empty-event.bro b/testing/btest/logging/empty-event.bro index 977450b40f..3e343b75a2 100644 --- a/testing/btest/logging/empty-event.bro +++ b/testing/btest/logging/empty-event.bro @@ -12,7 +12,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } global log_ssh: event(rec: Log); diff --git a/testing/btest/logging/events.bro b/testing/btest/logging/events.bro index 605ae0d5b6..101b13b095 100644 --- a/testing/btest/logging/events.bro +++ b/testing/btest/logging/events.bro @@ -17,7 +17,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } global ssh_log: event(rec: Log); diff --git a/testing/btest/logging/no-local.bro b/testing/btest/logging/no-local.bro index 43eccce2c5..eb6cb60151 100644 --- a/testing/btest/logging/no-local.bro +++ b/testing/btest/logging/no-local.bro @@ -12,7 +12,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } redef Log::enable_local_logging = F; diff --git a/testing/btest/logging/path-func.bro b/testing/btest/logging/path-func.bro index 3efa9d55a4..b744057a84 100644 --- a/testing/btest/logging/path-func.bro +++ b/testing/btest/logging/path-func.bro @@ -18,7 +18,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } global c = -1; diff --git a/testing/btest/logging/pred.bro b/testing/btest/logging/pred.bro index 38a7a53548..eb1a02cf08 100644 --- a/testing/btest/logging/pred.bro +++ b/testing/btest/logging/pred.bro @@ -18,7 +18,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } function fail(rec: Log): bool diff --git a/testing/btest/logging/remote-types.bro b/testing/btest/logging/remote-types.bro index a3dbe1f163..ecf3e96bc5 100644 --- a/testing/btest/logging/remote-types.bro +++ b/testing/btest/logging/remote-types.bro @@ -35,7 +35,9 @@ export { sc: set[count]; ss: set[string]; se: set[string]; - }; + vc: vector of count; + ve: vector of string; + } &log; } event bro_init() @@ -54,6 +56,7 @@ module SSH; event remote_connection_handshake_done(p: event_peer) { local empty_set: set[string]; + local empty_vector: vector of string; Log::write(SSH, [ $b=T, @@ -70,7 +73,9 @@ event remote_connection_handshake_done(p: event_peer) $s="hurz", $sc=set(1,2,3,4), $ss=set("AA", "BB", "CC"), - $se=empty_set + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector ]); } @TEST-END-FILE diff --git a/testing/btest/logging/remote.bro b/testing/btest/logging/remote.bro index 3ff8527cc9..cd0a361b7c 100644 --- a/testing/btest/logging/remote.bro +++ b/testing/btest/logging/remote.bro @@ -25,7 +25,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/remove.bro b/testing/btest/logging/remove.bro index a3fd16d760..488b21408f 100644 --- a/testing/btest/logging/remove.bro +++ b/testing/btest/logging/remove.bro @@ -16,7 +16,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/rotate-custom.bro b/testing/btest/logging/rotate-custom.bro index e77a0eced9..66e90de8c3 100644 --- a/testing/btest/logging/rotate-custom.bro +++ b/testing/btest/logging/rotate-custom.bro @@ -1,6 +1,6 @@ # # @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out -# @TEST-EXEC: for i in test*.log; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out # @TEST-EXEC: btest-diff out module Test; @@ -14,7 +14,7 @@ export { type Log: record { t: time; id: conn_id; # Will be rolled out into individual columns. - }; + } &log; } redef Log::default_rotation_interval = 1hr; diff --git a/testing/btest/logging/rotate.bro b/testing/btest/logging/rotate.bro index 96d5068db7..dc7cd79d56 100644 --- a/testing/btest/logging/rotate.bro +++ b/testing/btest/logging/rotate.bro @@ -14,7 +14,7 @@ export { type Log: record { t: time; id: conn_id; # Will be rolled out into individual columns. - }; + } &log; } redef Log::default_rotation_interval = 1hr; diff --git a/testing/btest/logging/stdout.bro b/testing/btest/logging/stdout.bro index 15fd071b58..a482f742a0 100644 --- a/testing/btest/logging/stdout.bro +++ b/testing/btest/logging/stdout.bro @@ -13,7 +13,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/test-logging.bro b/testing/btest/logging/test-logging.bro index 5011ecd26c..8443bc2236 100644 --- a/testing/btest/logging/test-logging.bro +++ b/testing/btest/logging/test-logging.bro @@ -12,7 +12,7 @@ export { id: conn_id; # Will be rolled out into individual columns. status: string &optional; country: string &default="unknown"; - }; + } &log; } event bro_init() diff --git a/testing/btest/logging/types.bro b/testing/btest/logging/types.bro index c54a1fa9f1..8cd59192bd 100644 --- a/testing/btest/logging/types.bro +++ b/testing/btest/logging/types.bro @@ -27,7 +27,9 @@ export { sc: set[count]; ss: set[string]; se: set[string]; - }; + vc: vector of count; + ve: vector of string; + } &log; } event bro_init() @@ -35,6 +37,7 @@ event bro_init() Log::create_stream(SSH, [$columns=Log]); local empty_set: set[string]; + local empty_vector: vector of string; Log::write(SSH, [ $b=T, @@ -51,7 +54,9 @@ event bro_init() $s="hurz", $sc=set(1,2,3,4), $ss=set("AA", "BB", "CC"), - $se=empty_set + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector ]); } diff --git a/testing/btest/logging/vec.bro b/testing/btest/logging/vec.bro new file mode 100644 index 0000000000..8d882d53bb --- /dev/null +++ b/testing/btest/logging/vec.bro @@ -0,0 +1,27 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + vec: vector of string &log; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local v: vector of string; + + v[2] = "2"; + v[5] = "5"; + + Log::write(SSH, [$vec=v]); +} + + From 0a97a9e82a4b80350839582582bd2830f75e4091 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 5 Apr 2011 16:17:12 -0700 Subject: [PATCH 60/67] Fixing attributes to allow &default in tables/sets to be associated with the field. This works now: type X: record { a: table[string] of bool &default=table( ["foo"] = T ); b: table[string] of bool &default=table(); c: set[string] &default=set("A", "B", "C"); d: set[string] &default=set(); }; I think previously the intend was to associate &default with the table/set (i.e., define the default value for non-existing indices). However, that was already not working: the error checking was reporting type mismatches. So, this shouldn't break anything and make things more consistent. --- src/Attr.cc | 35 +++++++++++++------ src/Attr.h | 3 +- src/Expr.cc | 4 +-- src/StateAccess.cc | 2 +- src/Type.cc | 4 +-- src/Type.h | 2 +- src/Var.cc | 4 +-- src/parse.y | 9 ++--- .../language.rec-table-default/output | 14 ++++++++ testing/btest/language/rec-table-default.bro | 19 ++++++++++ 10 files changed, 72 insertions(+), 24 deletions(-) create mode 100644 testing/btest/Baseline/language.rec-table-default/output create mode 100644 testing/btest/language/rec-table-default.bro diff --git a/src/Attr.cc b/src/Attr.cc index 5a83d0501b..472fbdffe2 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -57,10 +57,11 @@ void Attr::AddTag(ODesc* d) const d->Add(attr_name(Tag())); } -Attributes::Attributes(attr_list* a, BroType* t) +Attributes::Attributes(attr_list* a, BroType* t, bool arg_in_record) { attrs = new attr_list(a->length()); type = t->Ref(); + in_record = arg_in_record; SetLocationInfo(&start_location, &end_location); @@ -199,7 +200,7 @@ void Attributes::CheckAttr(Attr* a) { BroType* atype = a->AttrExpr()->Type(); - if ( type->Tag() != TYPE_TABLE || type->IsSet() ) + if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) ) { if ( ! same_type(atype, type) ) a->AttrExpr()->Error("&default value has inconsistent type", type); @@ -208,18 +209,30 @@ void Attributes::CheckAttr(Attr* a) TableType* tt = type->AsTableType(); - if ( ! same_type(atype, tt->YieldType()) ) + if ( ! in_record ) { - // It can still be a default function. - if ( atype->Tag() == TYPE_FUNC ) + // &default applies to the type itself. + if ( ! same_type(atype, tt->YieldType()) ) { - FuncType* f = atype->AsFuncType(); - if ( ! f->CheckArgs(tt->IndexTypes()) || - ! same_type(f->YieldType(), tt->YieldType()) ) - Error("&default function type clash"); + // It can still be a default function. + if ( atype->Tag() == TYPE_FUNC ) + { + FuncType* f = atype->AsFuncType(); + if ( ! f->CheckArgs(tt->IndexTypes()) || + ! same_type(f->YieldType(), tt->YieldType()) ) + Error("&default function type clash"); + } + else + Error("&default value has inconsistent type"); } - else - Error("&default value has inconsistent type"); + } + else + { + // &default applies to record field. + if ( ! same_type(atype, type) && + ! (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) ) + Error("&default value has inconsistent type"); + break; } } break; diff --git a/src/Attr.h b/src/Attr.h index 73fb101841..4191ffa911 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -62,7 +62,7 @@ protected: // Manages a collection of attributes. class Attributes : public BroObj { public: - Attributes(attr_list* a, BroType* t); + Attributes(attr_list* a, BroType* t, bool in_record); ~Attributes(); void AddAttr(Attr* a); @@ -87,6 +87,7 @@ protected: BroType* type; attr_list* attrs; + bool in_record; }; #endif diff --git a/src/Expr.cc b/src/Expr.cc index dbfca7c9cb..1c26039a6d 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3400,7 +3400,7 @@ TableConstructorExpr::TableConstructorExpr(ListExpr* constructor_list, SetError("values in table(...) constructor do not specify a table"); } - attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0; } Val* TableConstructorExpr::Eval(Frame* f) const @@ -3466,7 +3466,7 @@ SetConstructorExpr::SetConstructorExpr(ListExpr* constructor_list, else if ( type->Tag() != TYPE_TABLE || ! type->AsTableType()->IsSet() ) SetError("values in set(...) constructor do not specify a set"); - attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0; } Val* SetConstructorExpr::Eval(Frame* f) const diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 17f702f07e..74481a155a 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -935,7 +935,7 @@ void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) attr_list* a = new attr_list; Attr* attr = new Attr(ATTR_TRACKED); a->append(attr); - id->SetAttrs(new Attributes(a, id->Type())); + id->SetAttrs(new Attributes(a, id->Type(), false)); Unref(attr); } diff --git a/src/Type.cc b/src/Type.cc index ce3bbc52af..81994a19a8 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -680,10 +680,10 @@ bool FuncType::DoUnserialize(UnserialInfo* info) return UNSERIALIZE(&is_event); } -TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs) +TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_record) { type = t; - attrs = arg_attrs ? new Attributes(arg_attrs, t) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, t, in_record) : 0; id = i; } diff --git a/src/Type.h b/src/Type.h index fe4f7a5eb5..bf037e2ed6 100644 --- a/src/Type.h +++ b/src/Type.h @@ -361,7 +361,7 @@ protected: class TypeDecl { public: - TypeDecl(BroType* t, const char* i, attr_list* attrs = 0); + TypeDecl(BroType* t, const char* i, attr_list* attrs = 0, bool in_record = false); ~TypeDecl(); const Attr* FindAttr(attr_tag a) const diff --git a/src/Var.cc b/src/Var.cc index 04ef78c14c..0c2be25711 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -107,7 +107,7 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, id->SetType(t); if ( attr ) - id->AddAttrs(new Attributes(attr, t)); + id->AddAttrs(new Attributes(attr, t, false)); if ( id->FindAttr(ATTR_PERSISTENT) || id->FindAttr(ATTR_SYNCHRONIZED) ) { @@ -221,7 +221,7 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) id->MakeType(); if ( attr ) - id->SetAttrs(new Attributes(attr, t)); + id->SetAttrs(new Attributes(attr, t, false)); } void begin_func(ID* id, const char* module_name, function_flavor flavor, diff --git a/src/parse.y b/src/parse.y index a982935b2b..370d1e0571 100644 --- a/src/parse.y +++ b/src/parse.y @@ -100,6 +100,7 @@ extern Expr* g_curr_debug_expr; Expr* bro_this = 0; int in_init = 0; +int in_record = 0; bool in_debug = false; bool resolving_global_ID = false; @@ -704,10 +705,10 @@ type: $$ = new SetType($3, 0); } - | TOK_RECORD '{' type_decl_list '}' + | TOK_RECORD '{' { ++in_record; } type_decl_list { --in_record; } '}' { - set_location(@1, @4); - $$ = new RecordType($3); + set_location(@1, @5); + $$ = new RecordType($4); } | TOK_UNION '{' type_list '}' @@ -805,7 +806,7 @@ type_decl: TOK_ID ':' type opt_attr ';' { set_location(@1, @5); - $$ = new TypeDecl($3, $1, $4); + $$ = new TypeDecl($3, $1, $4, (in_record > 0)); } ; diff --git a/testing/btest/Baseline/language.rec-table-default/output b/testing/btest/Baseline/language.rec-table-default/output new file mode 100644 index 0000000000..a1531d06ee --- /dev/null +++ b/testing/btest/Baseline/language.rec-table-default/output @@ -0,0 +1,14 @@ +{ +[foo] = T +} +{ + +} +{ +A, +B, +C +} +{ + +} diff --git a/testing/btest/language/rec-table-default.bro b/testing/btest/language/rec-table-default.bro new file mode 100644 index 0000000000..1473933e6a --- /dev/null +++ b/testing/btest/language/rec-table-default.bro @@ -0,0 +1,19 @@ + +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type X: record { + a: table[string] of bool &default=table( ["foo"] = T ); + b: table[string] of bool &default=table(); + c: set[string] &default=set("A", "B", "C"); + d: set[string] &default=set(); +}; + +global x: X; +global y: table[string] of bool &default=T; + +print x$a; +print x$b; +print x$c; +print x$d; + From 0257bd304e8ffcb6c8356962722b365c2af49d12 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 5 Apr 2011 16:23:49 -0700 Subject: [PATCH 61/67] Adding language directory to btest.cfg. --- testing/btest/btest.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 0aca59fd64..560ff61306 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,6 +1,6 @@ [btest] -TestDirs = +TestDirs = language TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp From 5629359a87da6c43a8404d726f368707de7f27f6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 5 Apr 2011 16:28:55 -0700 Subject: [PATCH 62/67] Test commit. --- testing/btest/btest.cfg | 1 - 1 file changed, 1 deletion(-) diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 560ff61306..ea24fda9df 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,4 +1,3 @@ - [btest] TestDirs = language TmpDir = %(testbase)s/.tmp From 58f86ae55dc6d24e9e101f7e05ada9b64e34e412 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 17 Apr 2011 10:42:51 -0700 Subject: [PATCH 63/67] Fixing bug with records sometimes unnecessarily coerced on assignment. --- aux/btest | 2 +- src/Type.cc | 4 ++-- .../btest/Baseline/language.record-ref-assign/output | 1 + testing/btest/language/record-ref-assign.bro | 12 ++++++++++++ 4 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/language.record-ref-assign/output create mode 100644 testing/btest/language/record-ref-assign.bro diff --git a/aux/btest b/aux/btest index acf25f34ab..409bda3a00 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit acf25f34ab48568f5db919c9e3505ed319daa4e5 +Subproject commit 409bda3a003b18c4736ef168595f20118f4d0038 diff --git a/src/Type.cc b/src/Type.cc index 85e057137f..8048c864d0 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1504,8 +1504,8 @@ int same_attrs(const Attributes* a1, const Attributes* a2) if ( ! a1 ) return (a2 == 0); - if ( a2 ) - return 0; + if ( ! a2 ) + return (a1 == 0); return (*a1 == *a2); } diff --git a/testing/btest/Baseline/language.record-ref-assign/output b/testing/btest/Baseline/language.record-ref-assign/output new file mode 100644 index 0000000000..4caf2f8fa8 --- /dev/null +++ b/testing/btest/Baseline/language.record-ref-assign/output @@ -0,0 +1 @@ +XXX, XXX diff --git a/testing/btest/language/record-ref-assign.bro b/testing/btest/language/record-ref-assign.bro new file mode 100644 index 0000000000..f71bc3890c --- /dev/null +++ b/testing/btest/language/record-ref-assign.bro @@ -0,0 +1,12 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +type State: record { + host: string &default="NOT SET"; +}; + +global session: State; +global s: State; +s = session; +s$host = "XXX"; +print s$host, session$host; From 09d37b202673a223b4f9e3acbb75be5726e5f5d4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 17 Apr 2011 11:14:07 -0700 Subject: [PATCH 64/67] Fixing logging filter "include" and "exclude" options. --- src/LogMgr.cc | 20 +++++++---- .../btest/Baseline/logging.exclude/ssh.log | 6 ++++ .../btest/Baseline/logging.include/ssh.log | 6 ++++ testing/btest/btest.cfg | 2 +- testing/btest/logging/exclude.bro | 34 +++++++++++++++++++ testing/btest/logging/include.bro | 34 +++++++++++++++++++ 6 files changed, 95 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/logging.exclude/ssh.log create mode 100644 testing/btest/Baseline/logging.include/ssh.log create mode 100644 testing/btest/logging/exclude.bro create mode 100644 testing/btest/logging/include.bro diff --git a/src/LogMgr.cc b/src/LogMgr.cc index 14637f3978..bd14cf17db 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -553,8 +553,6 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, Tabl else new_path = path + "." + rt->FieldName(i); - StringVal* new_path_val = new StringVal(path.c_str()); - if ( t->InternalType() == TYPE_INTERNAL_OTHER ) { if ( t->Tag() == TYPE_RECORD ) @@ -585,15 +583,25 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, Tabl // If include fields are specified, only include if explicitly listed. if ( include ) { - if ( ! include->Lookup(new_path_val) ) - return true; + StringVal* new_path_val = new StringVal(new_path.c_str()); + bool result = include->Lookup(new_path_val); + + Unref(new_path_val); + + if ( ! result ) + continue; } // If exclude fields are specified, do not only include if listed. if ( exclude ) { - if ( exclude->Lookup(new_path_val) ) - return true; + StringVal* new_path_val = new StringVal(new_path.c_str()); + bool result = exclude->Lookup(new_path_val); + + Unref(new_path_val); + + if ( result ) + continue; } // Alright, we want this field. diff --git a/testing/btest/Baseline/logging.exclude/ssh.log b/testing/btest/Baseline/logging.exclude/ssh.log new file mode 100644 index 0000000000..4defa5ced1 --- /dev/null +++ b/testing/btest/Baseline/logging.exclude/ssh.log @@ -0,0 +1,6 @@ +# id.orig_p id.resp_h id.resp_p status country +1234 2.3.4.5 80 success unknown +1234 2.3.4.5 80 failure US +1234 2.3.4.5 80 failure UK +1234 2.3.4.5 80 success BR +1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.include/ssh.log b/testing/btest/Baseline/logging.include/ssh.log new file mode 100644 index 0000000000..881704257e --- /dev/null +++ b/testing/btest/Baseline/logging.include/ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 6a5cf94b86..3f79ae0e59 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,6 +1,6 @@ [btest] -TestDirs = logging +TestDirs = logging language TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp diff --git a/testing/btest/logging/exclude.bro b/testing/btest/logging/exclude.bro new file mode 100644 index 0000000000..46603d3202 --- /dev/null +++ b/testing/btest/logging/exclude.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + Log::remove_default_filter(SSH); + Log::add_filter(SSH, [$name="f1", $exclude=set("t", "id.orig_h")]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/include.bro b/testing/btest/logging/include.bro new file mode 100644 index 0000000000..f1fac64bbd --- /dev/null +++ b/testing/btest/logging/include.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + Log::remove_default_filter(SSH); + Log::add_filter(SSH, [$name="default", $include=set("t", "id.orig_h")]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + From 29b0d0d1d9ec1bfdbd39a9a68424730b37ef2b2c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 17 Apr 2011 20:52:18 -0700 Subject: [PATCH 65/67] Files can now be logged; their filename will be written out. --- src/LogMgr.cc | 17 ++++++++++++++- src/LogWriterAscii.cc | 1 + src/Type.cc | 4 +++- testing/btest/Baseline/logging.file/ssh.log | 2 ++ testing/btest/logging/file.bro | 23 +++++++++++++++++++++ 5 files changed, 45 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/logging.file/ssh.log create mode 100644 testing/btest/logging/file.bro diff --git a/src/LogMgr.cc b/src/LogMgr.cc index bd14cf17db..e8e6660643 100644 --- a/src/LogMgr.cc +++ b/src/LogMgr.cc @@ -65,7 +65,7 @@ struct LogMgr::Stream { LogVal::~LogVal() { - if ( (type == TYPE_ENUM || type == TYPE_STRING) && present ) + if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE) && present ) delete val.string_val; if ( type == TYPE_TABLE && present ) @@ -104,6 +104,7 @@ bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) case TYPE_INTERVAL: case TYPE_ENUM: case TYPE_STRING: + case TYPE_FILE: return true; case TYPE_RECORD: @@ -206,6 +207,7 @@ bool LogVal::Read(SerializationFormat* fmt) case TYPE_ENUM: case TYPE_STRING: + case TYPE_FILE: { val.string_val = new string; return fmt->Read(val.string_val, "string"); @@ -309,6 +311,7 @@ bool LogVal::Write(SerializationFormat* fmt) const case TYPE_ENUM: case TYPE_STRING: + case TYPE_FILE: return fmt->Write(*val.string_val, "string"); case TYPE_TABLE: @@ -574,6 +577,11 @@ bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, Tabl // That's ok, handle it with all the other types below. } + else if ( t->Tag() == TYPE_FILE ) + { + // That's ok, handle it with all the other types below. + } + else { run_time("unsupported field type for log column"); return false; @@ -936,6 +944,13 @@ LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) break; } + case TYPE_FILE: + { + const BroFile* f = val->AsFile(); + lval->val.string_val = new string(f->Name()); + break; + } + case TYPE_TABLE: { ListVal* set = val->AsTableVal()->ConvertToPureList(); diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc index 034fdac015..eaf0e9abde 100644 --- a/src/LogWriterAscii.cc +++ b/src/LogWriterAscii.cc @@ -138,6 +138,7 @@ bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) case TYPE_ENUM: case TYPE_STRING: + case TYPE_FILE: { int size = val->val.string_val->size(); if ( size ) diff --git a/src/Type.cc b/src/Type.cc index df1420d99c..61d8b6a8dd 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -748,6 +748,8 @@ RecordType::RecordType(TypeList* arg_base, type_decl_list* refinements) void RecordType::Init(TypeList* arg_base) { + assert(false); // Is this ever used? + base = arg_base; if ( ! base ) @@ -914,7 +916,7 @@ const char* RecordType::AddFields(type_decl_list* others, attr_list* attr) log = true; } } - + loop_over_list(*others, i) { TypeDecl* td = (*others)[i]; diff --git a/testing/btest/Baseline/logging.file/ssh.log b/testing/btest/Baseline/logging.file/ssh.log new file mode 100644 index 0000000000..49115ab1df --- /dev/null +++ b/testing/btest/Baseline/logging.file/ssh.log @@ -0,0 +1,2 @@ +# t f +1303098703.62603 Foo.log diff --git a/testing/btest/logging/file.bro b/testing/btest/logging/file.bro new file mode 100644 index 0000000000..6d73ec52dd --- /dev/null +++ b/testing/btest/logging/file.bro @@ -0,0 +1,23 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + f: file; + } &log; +} + +const foo_log = open_log_file("Foo") &redef; + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::write(SSH, [$t=network_time(), $f=foo_log]); +} + From b3b5a731138c4d117d16179106691e8d034d7e8b Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Apr 2011 16:21:13 -0700 Subject: [PATCH 66/67] Fixing problem with optional fields. --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- src/Val.cc | 1 - .../Baseline/language.rec-nested-opt/output | 3 +++ .../btest/Baseline/logging.rotate-custom/out | 4 +-- testing/btest/language/rec-nested-opt.bro | 25 +++++++++++++++++++ 9 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/language.rec-nested-opt/output create mode 100644 testing/btest/language/rec-nested-opt.bro diff --git a/aux/binpac b/aux/binpac index c3c7ef0dfd..26d02716f9 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit c3c7ef0dfddb0746d3762e41086ba42928e68483 +Subproject commit 26d02716f9090651f319a4bfdf8ede49b3a7b53a diff --git a/aux/bro-aux b/aux/bro-aux index 98f92eeb40..7e50bac938 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 98f92eeb40281045159097764abddc428fb49bf2 +Subproject commit 7e50bac938af1831ecf9660159145a3c2e77e13d diff --git a/aux/broccoli b/aux/broccoli index 48d473398e..9332ab3467 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 48d473398e577893b6c7f77d605ccdf266a2f93b +Subproject commit 9332ab3467191ac22be09d6941ebd469e7a334d0 diff --git a/aux/broctl b/aux/broctl index 532dcd5aa5..06b74a0f23 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 532dcd5aa51c8b29b2d71cd37e1d7c21e33cc715 +Subproject commit 06b74a0f23767c8345ed146657120aba812f6764 diff --git a/aux/btest b/aux/btest index 409bda3a00..b29e2214cc 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 409bda3a003b18c4736ef168595f20118f4d0038 +Subproject commit b29e2214cc959b6b6c841497b7881df369331561 diff --git a/src/Val.cc b/src/Val.cc index 2b086ccec3..ce639e3d0d 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2955,7 +2955,6 @@ RecordVal* RecordVal::CoerceTo(const RecordType* t, Val* aggr) const Expr* rhs = new ConstExpr(Lookup(i)->Ref()); Expr* e = new RecordCoerceExpr(rhs, ar_t->FieldType(t_i)->AsRecordType()); ar->Assign(t_i, e->Eval(0)); - break; } ar->Assign(t_i, Lookup(i)->Ref()); diff --git a/testing/btest/Baseline/language.rec-nested-opt/output b/testing/btest/Baseline/language.rec-nested-opt/output new file mode 100644 index 0000000000..e41358b71d --- /dev/null +++ b/testing/btest/Baseline/language.rec-nested-opt/output @@ -0,0 +1,3 @@ +{ +[Wget/1.9+cvs-stable (Red Hat modified)] = [name=Wget, version=[major=1, minor=9, addl=+cvs], host=0.0.0.0, ts=0.0] +} diff --git a/testing/btest/Baseline/logging.rotate-custom/out b/testing/btest/Baseline/logging.rotate-custom/out index e270b6a773..8206ec5f9a 100644 --- a/testing/btest/Baseline/logging.rotate-custom/out +++ b/testing/btest/Baseline/logging.rotate-custom/out @@ -68,6 +68,8 @@ # t id.orig_h id.orig_p id.resp_h id.resp_p 1299499205.0 10.0.0.1 20 10.0.0.2 1033 1299502795.0 10.0.0.2 20 10.0.0.3 9 +> test.log +# t id.orig_h id.orig_p id.resp_h id.resp_p > test2-11-03-06_19.00.05.log # t id.orig_h id.orig_p id.resp_h id.resp_p 1299466805.0 10.0.0.1 20 10.0.0.2 1024 @@ -130,5 +132,3 @@ 1299502795.0 10.0.0.2 20 10.0.0.3 9 > test2.log # t id.orig_h id.orig_p id.resp_h id.resp_p -> test.log -# t id.orig_h id.orig_p id.resp_h id.resp_p diff --git a/testing/btest/language/rec-nested-opt.bro b/testing/btest/language/rec-nested-opt.bro new file mode 100644 index 0000000000..eb7375541b --- /dev/null +++ b/testing/btest/language/rec-nested-opt.bro @@ -0,0 +1,25 @@ + +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type Version: record { + major: count &optional; ##< Major version number + minor: count &optional; ##< Minor version number + addl: string &optional; ##< Additional version string (e.g. "beta42") +} &log; + +type Info: record { + name: string; + version: Version; + host: addr; + ts: time; +}; + + +# Important thing to note here is that $minor2 is not include in the $version field. +global matched_software: table[string] of Info = { + ["Wget/1.9+cvs-stable (Red Hat modified)"] = + [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=0.0.0.0, $ts=network_time()], +}; + +print matched_software; From 5a6311d360c2cf50436faccfe212d4efe8601369 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Apr 2011 16:58:18 -0700 Subject: [PATCH 67/67] Implementing a VectorCoerceExpr. Turns out we didn't have that yet. I don't remember who implemented vectors originally, but he does owe us all round at Jupiter ... --- src/Expr.cc | 70 +++++++++++++++++-- src/Expr.h | 16 ++++- src/SerialTypes.h | 1 + src/Type.cc | 6 ++ src/Type.h | 4 ++ .../language.vector-coerce-expr/output | 4 ++ testing/btest/language/vector-coerce-expr.bro | 20 ++++++ 7 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 testing/btest/Baseline/language.vector-coerce-expr/output create mode 100644 testing/btest/language/vector-coerce-expr.bro diff --git a/src/Expr.cc b/src/Expr.cc index d384bb27e0..0de68b1f46 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2497,12 +2497,7 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, bool AssignExpr::TypeCheck() { TypeTag bt1 = op1->Type()->Tag(); - if ( IsVector(bt1) ) - bt1 = op1->Type()->AsVectorType()->YieldType()->Tag(); - TypeTag bt2 = op2->Type()->Tag(); - if ( IsVector(bt2) ) - bt2 = op2->Type()->AsVectorType()->YieldType()->Tag(); if ( bt1 == TYPE_LIST && bt2 == TYPE_ANY ) // This is ok because we cannot explicitly declare lists on @@ -2531,6 +2526,13 @@ bool AssignExpr::TypeCheck() return true; } + if ( bt1 == TYPE_VECTOR && bt2 == bt1 && + op2->Type()->AsVectorType()->IsUnspecifiedVector() ) + { + op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType()); + return true; + } + if ( op1->Type()->Tag() == TYPE_RECORD && op2->Type()->Tag() == TYPE_RECORD ) { @@ -3506,6 +3508,13 @@ VectorConstructorExpr::VectorConstructorExpr(ListExpr* constructor_list) if ( IsError() ) return; + if ( constructor_list->Exprs().length() == 0 ) + { + // vector(). + SetType(new ::VectorType(base_type(TYPE_ANY))); + return; + } + BroType* t = merge_type_list(constructor_list); if ( t ) { @@ -4122,6 +4131,50 @@ bool TableCoerceExpr::DoUnserialize(UnserialInfo* info) return true; } +VectorCoerceExpr::VectorCoerceExpr(Expr* op, VectorType* v) +: UnaryExpr(EXPR_VECTOR_COERCE, op) + { + if ( IsError() ) + return; + + SetType(v->Ref()); + + if ( Type()->Tag() != TYPE_VECTOR ) + ExprError("coercion to non-vector"); + + else if ( op->Type()->Tag() != TYPE_VECTOR ) + ExprError("coercion of non-vector to vector"); + } + + +VectorCoerceExpr::~VectorCoerceExpr() + { + } + +Val* VectorCoerceExpr::Fold(Val* v) const + { + VectorVal* vv = v->AsVectorVal(); + + if ( vv->Size() > 0 ) + Internal("coercion of non-empty vector"); + + return new VectorVal(Type()->Ref()->AsVectorType()); + } + +IMPLEMENT_SERIAL(VectorCoerceExpr, SER_VECTOR_COERCE_EXPR); + +bool VectorCoerceExpr::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_VECTOR_COERCE_EXPR, UnaryExpr); + return true; + } + +bool VectorCoerceExpr::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(UnaryExpr); + return true; + } + FlattenExpr::FlattenExpr(Expr* arg_op) : UnaryExpr(EXPR_FLATTEN, arg_op) { @@ -5337,6 +5390,13 @@ int check_and_promote_expr(Expr*& e, BroType* t) return 1; } + if ( t->Tag() == TYPE_VECTOR && et->Tag() == TYPE_VECTOR && + et->AsVectorType()->IsUnspecifiedVector() ) + { + e = new VectorCoerceExpr(e, t->AsVectorType()); + return 1; + } + t->Error("type clash", e); return 0; } diff --git a/src/Expr.h b/src/Expr.h index 9c338a0f8a..933c42daa2 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -40,7 +40,7 @@ typedef enum { EXPR_CALL, EXPR_EVENT, EXPR_SCHEDULE, - EXPR_ARITH_COERCE, EXPR_RECORD_COERCE, EXPR_TABLE_COERCE, + EXPR_ARITH_COERCE, EXPR_RECORD_COERCE, EXPR_TABLE_COERCE, EXPR_VECTOR_COERCE, EXPR_SIZE, EXPR_FLATTEN, #define NUM_EXPRS (int(EXPR_FLATTEN) + 1) @@ -895,6 +895,20 @@ protected: DECLARE_SERIAL(TableCoerceExpr); }; +class VectorCoerceExpr : public UnaryExpr { +public: + VectorCoerceExpr(Expr* op, VectorType* v); + ~VectorCoerceExpr(); + +protected: + friend class Expr; + VectorCoerceExpr() { } + + Val* Fold(Val* v) const; + + DECLARE_SERIAL(VectorCoerceExpr); +}; + // An internal operator for flattening array indices that are records // into a list of individual values. class FlattenExpr : public UnaryExpr { diff --git a/src/SerialTypes.h b/src/SerialTypes.h index a33b026fae..4f9ab4585b 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -146,6 +146,7 @@ SERIAL_EXPR(TABLE_CONSTRUCTOR_EXPR, 40) SERIAL_EXPR(SET_CONSTRUCTOR_EXPR, 41) SERIAL_EXPR(VECTOR_CONSTRUCTOR_EXPR, 42) SERIAL_EXPR(TABLE_COERCE_EXPR, 43) +SERIAL_EXPR(VECTOR_COERCE_EXPR, 44) #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT) SERIAL_STMT(STMT, 1) diff --git a/src/Type.cc b/src/Type.cc index 61d8b6a8dd..4bcd31cc4d 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -1290,6 +1290,11 @@ int VectorType::MatchesIndex(ListExpr*& index) const MATCHES_INDEX_SCALAR : DOES_NOT_MATCH_INDEX; } +bool VectorType::IsUnspecifiedVector() const + { + return yield_type->Tag() == TYPE_ANY; + } + IMPLEMENT_SERIAL(VectorType, SER_VECTOR_TYPE); bool VectorType::DoSerialize(SerialInfo* info) const @@ -1651,6 +1656,7 @@ BroType* merge_types(const BroType* t1, const BroType* t2) case TYPE_ADDR: case TYPE_NET: case TYPE_SUBNET: + case TYPE_BOOL: case TYPE_ANY: case TYPE_ERROR: return base_type(tg1); diff --git a/src/Type.h b/src/Type.h index ee991a13a3..05ca3adad1 100644 --- a/src/Type.h +++ b/src/Type.h @@ -525,6 +525,10 @@ public: int MatchesIndex(ListExpr*& index) const; + // Returns true if this table type is "unspecified", which is what one + // gets using an empty "vector()" constructor. + bool IsUnspecifiedVector() const; + protected: VectorType() { yield_type = 0; } diff --git a/testing/btest/Baseline/language.vector-coerce-expr/output b/testing/btest/Baseline/language.vector-coerce-expr/output new file mode 100644 index 0000000000..baabf9d89d --- /dev/null +++ b/testing/btest/Baseline/language.vector-coerce-expr/output @@ -0,0 +1,4 @@ +[] +[1, 2, 3] +[T, F, T] +[] diff --git a/testing/btest/language/vector-coerce-expr.bro b/testing/btest/language/vector-coerce-expr.bro new file mode 100644 index 0000000000..d58417f226 --- /dev/null +++ b/testing/btest/language/vector-coerce-expr.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type X: record { + a: vector of bool &default=vector(T, F, T); + b: vector of bool &default=vector(); +}; + +global x: X; + +global a: vector of count; + +a = vector(); +print a; + +a = vector(1,2,3); +print a; + +print x$a; +print x$b;