diff --git a/CHANGES b/CHANGES index 8ea82df983..2e83e651d2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,55 @@ +1.6-dev.88 Wed Apr 20 20:43:48 PDT 2011 + +- Implementation of Bro's new logging framework. We will document this + separately. (Robin Sommer) + +- Already defined record types can now be further extended via the + '+=' operator. The added fields must be either &optional or have a + &default value. (Robin Sommer) + + Example: + + type Foo: record { + a: count; + b: count &optional; + }; + + redef record Foo += { + c: count &default=42; + d: count &optional; + }; + + global f: Foo = [$a=21]; + + print f; + + Output: + + [a=21, b=, c=42, d=] + +- Enabling assignment of empty vectors ("vector()"). (Robin Sommer) + +- Fixing attributes to allow &default attributes to be associated with + records fields of type tables/sets/vector. (Robin Sommer) + +- '[]' is now a valid record constructor. (Robin Sommer) + +- A instance of a record type A is now coercable into one of type B if + the fields of type A are a subset of those of type B. (Robin Sommer) + +- A number of bug fixes and enhancements for record/set/table/vector + coercion. (Robin Sommer) + +- Fixing a problem with records that have optional fields when used as + table/set indices. Addresses #367. (Robin Sommer) + +- Fixing an off-by-one error in join_string_vec(). (Seth Hall) + +- Updating to_count() to cope with 64bit ints. (Seth Hall) + +- A new BiF count_to_v4_addr() to turn a count into an IPv4 address. + (Seth Hall) + 1.6-dev.80 Mon Apr 18 14:50:54 PDT 2011 - New framework for generating documentation from Bro scripts. (Jon diff --git a/VERSION b/VERSION index 43119b8d6f..f11ca0fe8a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.6-dev.80 +1.6-dev.88 diff --git a/policy/bro.init b/policy/bro.init index ba180e5045..43601fa8b3 100644 --- a/policy/bro.init +++ b/policy/bro.init @@ -21,7 +21,7 @@ type conn_id: record { orig_p: port; resp_h: addr; resp_p: port; -}; +} &log; type icmp_conn: record { orig_h: addr; @@ -277,6 +277,9 @@ type entropy_test_result: record { @load strings.bif.bro @load bro.bif.bro +@load logging # sic! Not logging.bif. +@load logging-ascii + global bro_alarm_file: file &redef; global alarm_hook: function(msg: string): bool &redef; global log_file_name: function(tag: string): string &redef; @@ -1375,7 +1378,7 @@ const suppress_local_output = F &redef; # Holds the filename of the trace file given with -w (empty if none). const trace_output_file = ""; - + # If a trace file is given, dump *all* packets seen by Bro into it. # By default, Bro applies (very few) heuristics to reduce the volume. # A side effect of setting this to true is that we can write the diff --git a/policy/conn.bro b/policy/conn.bro index 134ac9db13..a5d620d4c8 100644 --- a/policy/conn.bro +++ b/policy/conn.bro @@ -14,7 +14,9 @@ const conn_closed = { TCP_CLOSED, TCP_RESET }; global have_FTP = F; # if true, we've loaded ftp.bro global have_SMTP = F; # if true, we've loaded smtp.bro -global is_ftp_data_conn: function(c: connection): bool; + +# TODO: Do we have a nicer way of defining this prototype? +export { global FTP::is_ftp_data_conn: function(c: connection): bool; } # Whether to include connection state history in the logs generated # by record_connection. @@ -186,7 +188,7 @@ function determine_service_non_DPD(c: connection) : string return i; # return first; } - else if ( have_FTP && is_ftp_data_conn(c) ) + else if ( have_FTP && FTP::is_ftp_data_conn(c) ) return port_names[20/tcp]; else if ( [c$id$resp_h, c$id$resp_p] in RPC_server_map ) diff --git a/policy/firewall.bro b/policy/firewall.bro index adbf7a450c..59a92206b4 100644 --- a/policy/firewall.bro +++ b/policy/firewall.bro @@ -151,7 +151,7 @@ function do_match(c: connection, r: rule): bool return F; } - if ( r$is_ftp && ! is_ftp_data_conn(c) ) + if ( r$is_ftp && ! FTP::is_ftp_data_conn(c) ) return F; return T; diff --git a/policy/logging-ascii.bro b/policy/logging-ascii.bro new file mode 100644 index 0000000000..ad59e8dcf5 --- /dev/null +++ b/policy/logging-ascii.bro @@ -0,0 +1,29 @@ +##! Interface for the ascii log writer. + +module LogAscii; + +export { + ## If true, output everything to stdout rather than + ## into files. This is primarily for debugging purposes. + const output_to_stdout = F &redef; + + ## If true, include a header line with column names. + const include_header = T &redef; + + # Prefix for the header line if included. + const header_prefix = "# " &redef; + + ## Separator between fields. + const separator = "\t" &redef; + + ## Separator between set elements. + const set_separator = "," &redef; + + ## String to use for empty fields. + const empty_field = "" &redef; + + ## String to use for an unset &optional field. + const unset_field = "-" &redef; +} + + diff --git a/policy/logging.bro b/policy/logging.bro new file mode 100644 index 0000000000..ed126f9950 --- /dev/null +++ b/policy/logging.bro @@ -0,0 +1,209 @@ +##! The Bro logging interface. +##! +##! See XXX for a introduction to Bro's logging framework. + +module Log; + +# Log::ID and Log::Writer are defined in bro.init due to circular dependencies. + +export { + ## If true, is local logging is by default enabled for all filters. + const enable_local_logging = T &redef; + + ## If true, is remote logging is by default enabled for all filters. + const enable_remote_logging = T &redef; + + ## Default writer to use if a filter does not specify + ## anything else. + const default_writer = WRITER_ASCII &redef; + + ## Type defining the content of a logging stream. + type Stream: record { + ## A record type defining the log's columns. + columns: any; + + ## Event that will be raised once for each log entry. + ## The event receives a single same parameter, an instance of type ``columns``. + ev: any &optional; + }; + + ## Filter customizing logging. + type Filter: record { + ## Descriptive name to reference this filter. + name: string; + + ## The writer to use. + writer: Writer &default=default_writer; + + ## Predicate indicating whether a log entry should be recorded. + ## If not given, all entries are recorded. + ## + ## rec: An instance of the streams's ``columns`` type with its + ## fields set to the values to logged. + ## + ## Returns: True if the entry is to be recorded. + pred: function(rec: any): bool &optional; + + ## Output path for recording entries matching this + ## filter. + ## + ## The specific interpretation of the string is up to + ## the used writer, and may for example be the destination + ## file name. Generally, filenames are expected to given + ## without any extensions; writers will add appropiate + ## extensions automatically. + path: string &optional; + + ## A function returning the output path for recording entries + ## matching this filter. This is similar to ``path`` yet allows + ## to compute the string dynamically. It is ok to return + ## different strings for separate calls, but be careful: it's + ## easy to flood the disk by returning a new string for each + ## connection ... + path_func: function(id: ID, path: string): string &optional; + + ## Subset of column names to record. If not given, all + ## columns are recorded. + include: set[string] &optional; + + ## Subset of column names to exclude from recording. If not given, + ## all columns are recorded. + exclude: set[string] &optional; + + ## If true, entries are recorded locally. + log_local: bool &default=enable_local_logging; + + ## If true, entries are passed on to remote peers. + log_remote: bool &default=enable_remote_logging; + }; + + # Log rotation support. + + ## Information passed into rotation callback functions. + type RotationInfo: record { + writer: Writer; ##> Writer. + path: string; ##> Original path value. + open: time; ##> Time when opened. + close: time; ##> Time when closed. + }; + + ## Default rotation interval. Zero disables rotation. + const default_rotation_interval = 0secs &redef; + + ## Default naming suffix format. Uses a strftime() style. + const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef; + + ## Default postprocessor for writers outputting into files. + const default_rotation_postprocessor = "" &redef; + + ## Default function to construct the name of a rotated output file. + ## The default implementation appends info$date_fmt to the original + ## file name. + ## + ## info: Meta-data about the file to be rotated. + global default_rotation_path_func: function(info: RotationInfo) : string &redef; + + ## Type for controlling file rotation. + type RotationControl: record { + ## Rotation interval. + interv: interval &default=default_rotation_interval; + ## Format for timestamps embedded into rotated file names. + date_fmt: string &default=default_rotation_date_format; + ## Postprocessor process to run on rotate file. + postprocessor: string &default=default_rotation_postprocessor; + }; + + ## Specifies rotation parameters per ``(id, path)`` tuple. + ## If a pair is not found in this table, default values defined in + ## ``RotationControl`` are used. + const rotation_control: table[Writer, string] of RotationControl &default=[] &redef; + + ## Sentinel value for indicating that a filter was not found when looked up. + const no_filter: Filter = [$name=""]; # Sentinel. + + # TODO: Document. + global create_stream: function(id: ID, stream: Stream) : bool; + global enable_stream: function(id: ID) : bool; + global disable_stream: function(id: ID) : bool; + global add_filter: function(id: ID, filter: Filter) : bool; + global remove_filter: function(id: ID, name: string) : bool; + global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found. + global write: function(id: ID, columns: any) : bool; + global set_buf: function(id: ID, buffered: bool): bool; + global flush: function(id: ID): bool; + global add_default_filter: function(id: ID) : bool; + global remove_default_filter: function(id: ID) : bool; +} + +# We keep a script-level copy of all filters so that we can manipulate them. +global filters: table[ID, string] of Filter; + +@load logging.bif # Needs Filter and Stream defined. + +module Log; + +function default_rotation_path_func(info: RotationInfo) : string + { + local date_fmt = rotation_control[info$writer, info$path]$date_fmt; + return fmt("%s-%s", info$path, strftime(date_fmt, info$open)); + } + +function create_stream(id: ID, stream: Stream) : bool + { + if ( ! __create_stream(id, stream) ) + return F; + + return add_default_filter(id); + } + +function disable_stream(id: ID) : bool + { + if ( ! __disable_stream(id) ) + return F; + } + +function add_filter(id: ID, filter: Filter) : bool + { + filters[id, filter$name] = filter; + return __add_filter(id, filter); + } + +function remove_filter(id: ID, name: string) : bool + { + delete filters[id, name]; + return __remove_filter(id, name); + } + +function get_filter(id: ID, name: string) : Filter + { + if ( [id, name] in filters ) + return filters[id, name]; + + return no_filter; + } + +function write(id: ID, columns: any) : bool + { + return __write(id, columns); + } + +function set_buf(id: ID, buffered: bool): bool + { + return __set_buf(id, buffered); + } + +function flush(id: ID): bool + { + return __flush(id); + } + +function add_default_filter(id: ID) : bool + { + return add_filter(id, [$name="default"]); + } + +function remove_default_filter(id: ID) : bool + { + return remove_filter(id, "default"); + } + diff --git a/policy/mime.bro b/policy/mime.bro index 7f923aac08..73a4ecefee 100644 --- a/policy/mime.bro +++ b/policy/mime.bro @@ -74,13 +74,14 @@ function mime_header_subject(session: mime_session_info, ### This is a bit clunky. These are functions we call out to, defined # elsewhere. The way we really ought to do this is to have them passed # in during initialization. But for now, we presume knowledge of their -# names in global scope. -module GLOBAL; -global check_relay_3: - function(session: MIME::mime_session_info, msg_id: string); -global check_relay_4: - function(session: MIME::mime_session_info, content_hash: string); -module MIME; +# names. +export + { + global SMTP::check_relay_3: + function(session: MIME::mime_session_info, msg_id: string); + global SMTP::check_relay_4: + function(session: MIME::mime_session_info, content_hash: string); + } function mime_header_message_id(session: mime_session_info, name: string, arg: string) { @@ -107,7 +108,7 @@ function mime_header_message_id(session: mime_session_info, name: string, arg: s s = t[1]; if ( session$level == 1 && SMTP::process_smtp_relay ) - check_relay_3(session, s); + SMTP::check_relay_3(session, s); } redef mime_header_handler = { diff --git a/policy/remote.bro b/policy/remote.bro index fad2beb900..294c8fcd1e 100644 --- a/policy/remote.bro +++ b/policy/remote.bro @@ -43,6 +43,9 @@ export { # Whether to perform state synchronization with peer. sync: bool &default = T; + # Whether to request logs from the peer. + request_logs: bool &default = F; + # When performing state synchronization, whether we consider # our state to be authoritative. If so, we will send the peer # our current set when the connection is set up. @@ -176,6 +179,12 @@ function setup_peer(p: event_peer, dst: Destination) request_remote_sync(p, dst$auth); } + if ( dst$request_logs ) + { + do_script_log(p, "requesting logs"); + request_remote_logs(p); + } + dst$peer = p; dst$connected = T; connected_peers[p$id] = dst; diff --git a/src/Attr.cc b/src/Attr.cc index cb3f21db92..5cf36a643a 100644 --- a/src/Attr.cc +++ b/src/Attr.cc @@ -7,6 +7,7 @@ #include "Attr.h" #include "Expr.h" #include "Serializer.h" +#include "LogMgr.h" const char* attr_name(attr_tag t) { @@ -88,10 +89,11 @@ void Attr::AddTag(ODesc* d) const d->Add(attr_name(Tag())); } -Attributes::Attributes(attr_list* a, BroType* t) +Attributes::Attributes(attr_list* a, BroType* t, bool arg_in_record) { attrs = new attr_list(a->length()); type = t->Ref(); + in_record = arg_in_record; SetLocationInfo(&start_location, &end_location); @@ -241,27 +243,72 @@ void Attributes::CheckAttr(Attr* a) { BroType* atype = a->AttrExpr()->Type(); - if ( type->Tag() != TYPE_TABLE || type->IsSet() ) + if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) ) { - if ( ! same_type(atype, type) ) - a->AttrExpr()->Error("&default value has inconsistent type", type); - break; + if ( same_type(atype, type) ) + // Ok. + break; + + // Record defaults may be promotable. + if ( (type->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD && + record_promotion_compatible(atype->AsRecordType(), + type->AsRecordType())) ) + // Ok. + break; + + a->AttrExpr()->Error("&default value has inconsistent type", type); } TableType* tt = type->AsTableType(); + BroType* ytype = tt->YieldType(); - if ( ! same_type(atype, tt->YieldType()) ) + if ( ! in_record ) { - // It can still be a default function. - if ( atype->Tag() == TYPE_FUNC ) + // &default applies to the type itself. + if ( ! same_type(atype, ytype) ) { - FuncType* f = atype->AsFuncType(); - if ( ! f->CheckArgs(tt->IndexTypes()) || - ! same_type(f->YieldType(), tt->YieldType()) ) - Error("&default function type clash"); + // It can still be a default function. + if ( atype->Tag() == TYPE_FUNC ) + { + FuncType* f = atype->AsFuncType(); + if ( ! f->CheckArgs(tt->IndexTypes()) || + ! same_type(f->YieldType(), ytype) ) + Error("&default function type clash"); + + // Ok. + break; + } + + // Table defaults may be promotable. + if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD && + record_promotion_compatible(atype->AsRecordType(), + ytype->AsRecordType())) ) + // Ok. + break; + + Error("&default value has inconsistent type 2"); } - else - Error("&default value has inconsistent type"); + + // Ok. + break; + } + + else + { + // &default applies to record field. + + if ( same_type(atype, type) || + (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) ) + // Ok. + break; + + // Table defaults may be promotable. + if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD && + record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) ) + // Ok. + break; + + Error("&default value has inconsistent type"); } } break; @@ -358,10 +405,12 @@ void Attributes::CheckAttr(Attr* a) case ATTR_GROUP: if ( type->Tag() != TYPE_FUNC || ! type->AsFuncType()->IsEvent() ) - { Error("&group only applicable to events"); - break; - } + break; + + case ATTR_LOG: + if ( ! LogVal::IsCompatibleType(type) ) + Error("&log applied to a type that cannot be logged"); break; default: @@ -369,6 +418,41 @@ void Attributes::CheckAttr(Attr* a) } } +bool Attributes::operator==(const Attributes& other) const + { + if ( ! attrs ) + return other.attrs; + + if ( ! other.attrs ) + return false; + + loop_over_list(*attrs, i) + { + Attr* a = (*attrs)[i]; + Attr* o = other.FindAttr(a->Tag()); + + if ( ! o ) + return false; + + if ( ! (*a == *o) ) + return false; + } + + loop_over_list(*other.attrs, j) + { + Attr* o = (*other.attrs)[j]; + Attr* a = FindAttr(o->Tag()); + + if ( ! a ) + return false; + + if ( ! (*a == *o) ) + return false; + } + + return true; + } + bool Attributes::Serialize(SerialInfo* info) const { return SerialObj::Serialize(info); diff --git a/src/Attr.h b/src/Attr.h index ccc6b0f5d1..bdde98b4a7 100644 --- a/src/Attr.h +++ b/src/Attr.h @@ -35,6 +35,7 @@ typedef enum { ATTR_MERGEABLE, ATTR_PRIORITY, ATTR_GROUP, + ATTR_LOG, ATTR_TRACKED, // hidden attribute, tracked by NotifierRegistry #define NUM_ATTRS (int(ATTR_TRACKED) + 1) } attr_tag; @@ -53,6 +54,20 @@ public: void Describe(ODesc* d) const; void DescribeReST(ODesc* d) const; + bool operator==(const Attr& other) const + { + if ( tag != other.tag ) + return false; + + if ( expr || other.expr ) + // If any has an expression and they aren't the same object, we + // declare them unequal, as we can't really find out if the two + // expressions are equivalent. + return (expr == other.expr); + + return true; + } + protected: void AddTag(ODesc* d) const; @@ -63,7 +78,7 @@ protected: // Manages a collection of attributes. class Attributes : public BroObj { public: - Attributes(attr_list* a, BroType* t); + Attributes(attr_list* a, BroType* t, bool in_record); ~Attributes(); void AddAttr(Attr* a); @@ -81,6 +96,8 @@ public: bool Serialize(SerialInfo* info) const; static Attributes* Unserialize(UnserialInfo* info); + bool operator==(const Attributes& other) const; + protected: Attributes() { type = 0; attrs = 0; } void CheckAttr(Attr* attr); @@ -89,6 +106,7 @@ protected: BroType* type; attr_list* attrs; + bool in_record; }; #endif diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b2e89033ad..ac87ae4dd4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -132,6 +132,7 @@ endmacro(GET_BIF_OUTPUT_FILES) set(BIF_SRCS bro.bif + logging.bif event.bif const.bif types.bif @@ -338,6 +339,9 @@ set(bro_SRCS IRC.cc List.cc Logger.cc + LogMgr.cc + LogWriter.cc + LogWriterAscii.cc Login.cc MIME.cc NCP.cc diff --git a/src/CompHash.cc b/src/CompHash.cc index 2e0870303c..69b3d9c38d 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -65,11 +65,22 @@ CompositeHash::~CompositeHash() // Computes the piece of the hash for Val*, returning the new kp. char* CompositeHash::SingleValHash(int type_check, char* kp0, - BroType* bt, Val* v) const + BroType* bt, Val* v, bool optional) const { char* kp1 = 0; InternalTypeTag t = bt->InternalType(); + if ( optional ) + { + // Add a marker saying whether the optional field is set. + char* kp = AlignAndPadType(kp0); + *kp = ( v ? 1 : 0); + kp0 = reinterpret_cast(kp+1); + + if ( ! v ) + return kp0; + } + if ( type_check ) { InternalTypeTag vt = v->Type()->InternalType(); @@ -163,12 +174,16 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0, for ( int i = 0; i < num_fields; ++i ) { Val* rv_i = rv->Lookup(i); - if ( ! rv_i ) + + Attributes* a = rt->FieldDecl(i)->attrs; + bool optional = (a && a->FindAttr(ATTR_OPTIONAL)); + + if ( ! (rv_i || optional) ) return 0; if ( ! (kp = SingleValHash(type_check, kp, rt->FieldType(i), - rv_i)) ) + rv_i, optional)) ) return 0; } @@ -248,7 +263,7 @@ HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const char* kp = k; loop_over_list(*tl, i) { - kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i]); + kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i], false); if ( ! kp ) return 0; } @@ -315,10 +330,13 @@ HashKey* CompositeHash::ComputeSingletonHash(const Val* v, int type_check) const } int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v, - int type_check, int sz) const + int type_check, int sz, bool optional) const { InternalTypeTag t = bt->InternalType(); + if ( optional ) + sz = SizeAlign(sz, sizeof(char)); + if ( type_check && v ) { InternalTypeTag vt = v->Type()->InternalType(); @@ -369,9 +387,12 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v, for ( int i = 0; i < num_fields; ++i ) { + Attributes* a = rt->FieldDecl(i)->attrs; + bool optional = (a && a->FindAttr(ATTR_OPTIONAL)); + sz = SingleTypeKeySize(rt->FieldType(i), - rv ? rv->Lookup(i) : 0, - type_check, sz); + rv ? rv->Lookup(i) : 0, + type_check, sz, optional); if ( ! sz ) return 0; } @@ -386,7 +407,7 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v, case TYPE_INTERNAL_STRING: if ( ! v ) - return 0; + return optional ? sz : 0; // Factor in length field. sz = SizeAlign(sz, sizeof(int)); @@ -418,7 +439,7 @@ int CompositeHash::ComputeKeySize(const Val* v, int type_check) const loop_over_list(*tl, i) { sz = SingleTypeKeySize((*tl)[i], v ? v->AsListVal()->Index(i) : 0, - type_check, sz); + type_check, sz, false); if ( ! sz ) return 0; } @@ -495,20 +516,20 @@ ListVal* CompositeHash::RecoverVals(const HashKey* k) const loop_over_list(*tl, i) { Val* v; - kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v); + kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v, false); ASSERT(v); l->Append(v); } if ( kp != k_end ) - internal_error("under-ran key in CompositeHash::DescribeKey"); + internal_error("under-ran key in CompositeHash::DescribeKey %ld", k_end - kp); return l; } const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, const char* const k_end, BroType* t, - Val*& pval) const + Val*& pval, bool optional) const { // k->Size() == 0 for a single empty string. if ( kp0 >= k_end && k->Size() > 0 ) @@ -516,9 +537,20 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, TypeTag tag = t->Tag(); InternalTypeTag it = t->InternalType(); - const char* kp1 = 0; + if ( optional ) + { + const char* kp = AlignType(kp0); + kp0 = kp1 = reinterpret_cast(kp+1); + + if ( ! *kp ) + { + pval = 0; + return kp0; + } + } + switch ( it ) { case TYPE_INTERNAL_INT: { @@ -647,9 +679,13 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, for ( i = 0; i < num_fields; ++i ) { Val* v; + + Attributes* a = rt->FieldDecl(i)->attrs; + bool optional = (a && a->FindAttr(ATTR_OPTIONAL)); + kp = RecoverOneVal(k, kp, k_end, - rt->FieldType(i), v); - if ( ! v ) + rt->FieldType(i), v, optional); + if ( ! (v || optional) ) { internal_error("didn't recover expected number of fields from HashKey"); pval = 0; diff --git a/src/CompHash.h b/src/CompHash.h index a0632e1bfe..d3bde770b8 100644 --- a/src/CompHash.h +++ b/src/CompHash.h @@ -29,8 +29,8 @@ protected: // Computes the piece of the hash for Val*, returning the new kp. // Used as a helper for ComputeHash in the non-singleton case. - char* SingleValHash(int type_check, char* kp, - BroType* bt, Val* v) const; + char* SingleValHash(int type_check, char* kp, BroType* bt, Val* v, + bool optional) const; // Recovers just one Val of possibly many; called from RecoverVals. // Upon return, pval will point to the recovered Val of type t. @@ -38,7 +38,7 @@ protected: // upon errors, so there is no return value for invalid input. const char* RecoverOneVal(const HashKey* k, const char* kp, const char* const k_end, - BroType* t, Val*& pval) const; + BroType* t, Val*& pval, bool optional) const; // Rounds the given pointer up to the nearest multiple of the // given size, if not already a multiple. @@ -77,7 +77,7 @@ protected: int ComputeKeySize(const Val* v = 0, int type_check = 1) const; int SingleTypeKeySize(BroType*, const Val*, - int type_check, int sz) const; + int type_check, int sz, bool optional) const; TypeList* type; char* key; // space for composite key diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 9ab0bde380..3c9dbad363 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -17,6 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "compressor", 0, false }, {"string", 0, false }, { "notifiers", 0, false }, { "main-loop", 0, false }, { "dpd", 0, false }, { "tm", 0, false }, + { "logging", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 3d8c1ac83f..49c875a5c4 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -20,11 +20,12 @@ enum DebugStream { DBG_STATE, // StateAccess logging DBG_CHUNKEDIO, // ChunkedIO logging DBG_COMPRESSOR, // Connection compressor - DBG_STRING, // String code - DBG_NOTIFIERS, // Notifiers (see StateAccess.h) + DBG_STRING, // String code + DBG_NOTIFIERS, // Notifiers (see StateAccess.h) DBG_MAINLOOP, // Main IOSource loop DBG_DPD, // Dynamic application detection framework - DBG_TM, // Time-machine packet input via Brocolli + DBG_TM, // Time-machine packet input via Brocolli + DBG_LOGGING, // Logging streams NUM_DBGS // Has to be last }; diff --git a/src/Desc.cc b/src/Desc.cc index 86ae395aca..15db73d7c7 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -42,6 +42,8 @@ ODesc::ODesc(desc_type t, BroFile* arg_f) do_flush = 1; include_stats = 0; indent_with_spaces = 0; + escape = 0; + escape_len = 0; } ODesc::~ODesc() @@ -55,6 +57,12 @@ ODesc::~ODesc() free(base); } +void ODesc::SetEscape(const char* arg_escape, int len) + { + escape = arg_escape; + escape_len = len; + } + void ODesc::PushIndent() { ++indent_level; @@ -199,8 +207,44 @@ void ODesc::Indent() } } +static const char hex_chars[] = "0123456789ABCDEF"; void ODesc::AddBytes(const void* bytes, unsigned int n) + { + if ( ! escape ) + return AddBytesRaw(bytes, n); + + const char* s = (const char*) bytes; + const char* e = (const char*) bytes + n; + + while ( s < e ) + { + const char* t = (const char*) memchr(s, escape[0], e - s); + + if ( ! t ) + break; + + if ( memcmp(t, escape, escape_len) != 0 ) + break; + + AddBytesRaw(s, t - s); + + for ( int i = 0; i < escape_len; ++i ) + { + char hex[5] = "\\x00"; + hex[2] = hex_chars[(*t) >> 4]; + hex[3] = hex_chars[(*t) & 0x0f]; + AddBytesRaw(hex, sizeof(hex)); + ++t; + } + + s = t; + } + + AddBytesRaw(s, e - s); + } + +void ODesc::AddBytesRaw(const void* bytes, unsigned int n) { if ( n == 0 ) return; diff --git a/src/Desc.h b/src/Desc.h index ff7e6e864d..f6e2756efc 100644 --- a/src/Desc.h +++ b/src/Desc.h @@ -49,6 +49,9 @@ public: void SetFlush(int arg_do_flush) { do_flush = arg_do_flush; } + // The string passed in must remain valid as long as this object lives. + void SetEscape(const char* escape, int len); + void PushIndent(); void PopIndent(); void PopIndentNoNL(); @@ -97,6 +100,9 @@ public: Add("\n", 0); } + // Bypasses the escaping enabled via SetEscape(). + void AddRaw(const char* s, int len) { AddBytesRaw(s, len); } + // Returns the description as a string. const char* Description() const { return (const char*) base; } @@ -119,6 +125,7 @@ protected: void Indent(); void AddBytes(const void* bytes, unsigned int n); + void AddBytesRaw(const void* bytes, unsigned int n); // Make buffer big enough for n bytes beyond bufp. void Grow(unsigned int n); @@ -132,6 +139,9 @@ protected: unsigned int offset; // where we are in the buffer unsigned int size; // size of buffer in bytes + int escape_len; // number of bytes in to escape sequence + const char* escape; // bytes to escape on output + BroFile* f; // or the file we're using. int indent_level; diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 9df18ab007..9cc5306c9c 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -23,6 +23,11 @@ EventHandler::~EventHandler() delete [] group; } +EventHandler::operator bool() const + { + return enabled && ((local && local->HasBodies()) || receivers.length()); + } + FuncType* EventHandler::FType() { if ( type ) diff --git a/src/EventHandler.h b/src/EventHandler.h index de4d1a1458..4320134d50 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -34,8 +34,7 @@ public: void Call(val_list* vl, bool no_remote = false); // Returns true if there is at least one local or remote handler. - operator bool() const - { return enabled && (local || receivers.length()); } + operator bool() const; void SetUsed() { used = true; } bool Used() { return used; } diff --git a/src/Expr.cc b/src/Expr.cc index dbfca7c9cb..71b94c7aa7 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -284,7 +284,7 @@ Val* NameExpr::Eval(Frame* f) const Val* v; if ( id->AsType() ) - RunTime("cannot evaluate type name"); + return new Val(id->AsType(), true); if ( id->IsGlobal() ) v = id->ID_Val(); @@ -2497,12 +2497,7 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init, bool AssignExpr::TypeCheck() { TypeTag bt1 = op1->Type()->Tag(); - if ( IsVector(bt1) ) - bt1 = op1->Type()->AsVectorType()->YieldType()->Tag(); - TypeTag bt2 = op2->Type()->Tag(); - if ( IsVector(bt2) ) - bt2 = op2->Type()->AsVectorType()->YieldType()->Tag(); if ( bt1 == TYPE_LIST && bt2 == TYPE_ANY ) // This is ok because we cannot explicitly declare lists on @@ -2531,16 +2526,42 @@ bool AssignExpr::TypeCheck() return true; } + if ( bt1 == TYPE_VECTOR && bt2 == bt1 && + op2->Type()->AsVectorType()->IsUnspecifiedVector() ) + { + op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType()); + return true; + } + + if ( op1->Type()->Tag() == TYPE_RECORD && + op2->Type()->Tag() == TYPE_RECORD ) + { + if ( same_type(op1->Type(), op2->Type()) ) + { + RecordType* rt1 = op1->Type()->AsRecordType(); + RecordType* rt2 = op2->Type()->AsRecordType(); + + // Make sure the attributes match as well. + for ( int i = 0; i < rt1->NumFields(); ++i ) + { + const TypeDecl* td1 = rt1->FieldDecl(i); + const TypeDecl* td2 = rt2->FieldDecl(i); + + if ( same_attrs(td1->attrs, td2->attrs) ) + // Everything matches. + return true; + } + } + + // Need to coerce. + op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType()); + return true; + } + if ( ! same_type(op1->Type(), op2->Type()) ) { - if ( op1->Type()->Tag() == TYPE_RECORD && - op2->Type()->Tag() == TYPE_RECORD ) - op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType()); - else - { - ExprError("type clash in assignment"); - return false; - } + ExprError("type clash in assignment"); + return false; } return true; @@ -3290,48 +3311,12 @@ RecordConstructorExpr::RecordConstructorExpr(ListExpr* constructor_list) Val* RecordConstructorExpr::InitVal(const BroType* t, Val* aggr) const { - if ( ! aggr ) - aggr = new RecordVal(const_cast(t->AsRecordType())); + RecordVal* rv = Eval(0)->AsRecordVal(); + RecordVal* ar = rv->CoerceTo(t->AsRecordType(), aggr); - if ( record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) ) + if ( ar ) { - RecordVal* ar = aggr->AsRecordVal(); - RecordType* ar_t = aggr->Type()->AsRecordType(); - - RecordVal* rv = Eval(0)->AsRecordVal(); - RecordType* rv_t = rv->Type()->AsRecordType(); - - int i; - for ( i = 0; i < rv_t->NumFields(); ++i ) - { - int t_i = ar_t->FieldOffset(rv_t->FieldName(i)); - - if ( t_i < 0 ) - { - char buf[512]; - safe_snprintf(buf, sizeof(buf), - "orphan field \"%s\" in initialization", - rv_t->FieldName(i)); - Error(buf); - break; - } - - else - ar->Assign(t_i, rv->Lookup(i)->Ref()); - } - - for ( i = 0; i < ar_t->NumFields(); ++i ) - if ( ! ar->Lookup(i) && - ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) - { - char buf[512]; - safe_snprintf(buf, sizeof(buf), - "non-optional field \"%s\" missing in initialization", ar_t->FieldName(i)); - Error(buf); - } - Unref(rv); - return ar; } @@ -3400,7 +3385,7 @@ TableConstructorExpr::TableConstructorExpr(ListExpr* constructor_list, SetError("values in table(...) constructor do not specify a table"); } - attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0; } Val* TableConstructorExpr::Eval(Frame* f) const @@ -3466,7 +3451,7 @@ SetConstructorExpr::SetConstructorExpr(ListExpr* constructor_list, else if ( type->Tag() != TYPE_TABLE || ! type->AsTableType()->IsSet() ) SetError("values in set(...) constructor do not specify a set"); - attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0; } Val* SetConstructorExpr::Eval(Frame* f) const @@ -3523,6 +3508,13 @@ VectorConstructorExpr::VectorConstructorExpr(ListExpr* constructor_list) if ( IsError() ) return; + if ( constructor_list->Exprs().length() == 0 ) + { + // vector(). + SetType(new ::VectorType(base_type(TYPE_ANY))); + return; + } + BroType* t = merge_type_list(constructor_list); if ( t ) { @@ -3994,15 +3986,8 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r) { int t_i = t_r->FieldOffset(sub_r->FieldName(i)); if ( t_i < 0 ) - { - // Same as in RecordConstructorExpr::InitVal. - char buf[512]; - safe_snprintf(buf, sizeof(buf), - "orphan record field \"%s\"", - sub_r->FieldName(i)); - Error(buf); + // Orphane field in rhs, that's ok. continue; - } BroType* sub_t_i = sub_r->FieldType(i); BroType* sup_t_i = t_r->FieldType(t_i); @@ -4047,7 +4032,23 @@ Val* RecordCoerceExpr::Fold(Val* v) const for ( int i = 0; i < map_size; ++i ) { if ( map[i] >= 0 ) - val->Assign(i, rv->Lookup(map[i])->Ref()); + { + Val* rhs = rv->Lookup(map[i]); + if ( ! rhs ) + { + const Attr* def = rv->Type()->AsRecordType()->FieldDecl( + map[i])->FindAttr(ATTR_DEFAULT); + + if ( def ) + rhs = def->AttrExpr()->Eval(0); + } + + if ( rhs ) + rhs = rhs->Ref(); + + assert(rhs || Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_OPTIONAL)); + val->Assign(i, rhs); + } else val->Assign(i, 0); } @@ -4132,6 +4133,50 @@ bool TableCoerceExpr::DoUnserialize(UnserialInfo* info) return true; } +VectorCoerceExpr::VectorCoerceExpr(Expr* op, VectorType* v) +: UnaryExpr(EXPR_VECTOR_COERCE, op) + { + if ( IsError() ) + return; + + SetType(v->Ref()); + + if ( Type()->Tag() != TYPE_VECTOR ) + ExprError("coercion to non-vector"); + + else if ( op->Type()->Tag() != TYPE_VECTOR ) + ExprError("coercion of non-vector to vector"); + } + + +VectorCoerceExpr::~VectorCoerceExpr() + { + } + +Val* VectorCoerceExpr::Fold(Val* v) const + { + VectorVal* vv = v->AsVectorVal(); + + if ( vv->Size() > 0 ) + Internal("coercion of non-empty vector"); + + return new VectorVal(Type()->Ref()->AsVectorType()); + } + +IMPLEMENT_SERIAL(VectorCoerceExpr, SER_VECTOR_COERCE_EXPR); + +bool VectorCoerceExpr::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_VECTOR_COERCE_EXPR, UnaryExpr); + return true; + } + +bool VectorCoerceExpr::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(UnaryExpr); + return true; + } + FlattenExpr::FlattenExpr(Expr* arg_op) : UnaryExpr(EXPR_FLATTEN, arg_op) { @@ -5308,27 +5353,52 @@ int check_and_promote_expr(Expr*& e, BroType* t) return 1; } - else if ( ! same_type(t, et) ) + if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD ) { - if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD ) - { - RecordType* t_r = t->AsRecordType(); - RecordType* et_r = et->AsRecordType(); + RecordType* t_r = t->AsRecordType(); + RecordType* et_r = et->AsRecordType(); - if ( record_promotion_compatible(t_r, et_r) ) + if ( same_type(t, et) ) + { + // Make sure the attributes match as well. + for ( int i = 0; i < t_r->NumFields(); ++i ) { - e = new RecordCoerceExpr(e, t_r); - return 1; + const TypeDecl* td1 = t_r->FieldDecl(i); + const TypeDecl* td2 = et_r->FieldDecl(i); + + if ( same_attrs(td1->attrs, td2->attrs) ) + // Everything matches perfectly. + return 1; } } - else if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE && + if ( record_promotion_compatible(t_r, et_r) ) + { + e = new RecordCoerceExpr(e, t_r); + return 1; + } + + t->Error("incompatible record types", e); + return 0; + } + + + if ( ! same_type(t, et) ) + { + if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE && et->AsTableType()->IsUnspecifiedTable() ) { e = new TableCoerceExpr(e, t->AsTableType()); return 1; } + if ( t->Tag() == TYPE_VECTOR && et->Tag() == TYPE_VECTOR && + et->AsVectorType()->IsUnspecifiedVector() ) + { + e = new VectorCoerceExpr(e, t->AsVectorType()); + return 1; + } + t->Error("type clash", e); return 0; } diff --git a/src/Expr.h b/src/Expr.h index 9c338a0f8a..3d07f68df1 100644 --- a/src/Expr.h +++ b/src/Expr.h @@ -40,7 +40,10 @@ typedef enum { EXPR_CALL, EXPR_EVENT, EXPR_SCHEDULE, - EXPR_ARITH_COERCE, EXPR_RECORD_COERCE, EXPR_TABLE_COERCE, + EXPR_ARITH_COERCE, + EXPR_RECORD_COERCE, + EXPR_TABLE_COERCE, + EXPR_VECTOR_COERCE, EXPR_SIZE, EXPR_FLATTEN, #define NUM_EXPRS (int(EXPR_FLATTEN) + 1) @@ -895,6 +898,20 @@ protected: DECLARE_SERIAL(TableCoerceExpr); }; +class VectorCoerceExpr : public UnaryExpr { +public: + VectorCoerceExpr(Expr* op, VectorType* v); + ~VectorCoerceExpr(); + +protected: + friend class Expr; + VectorCoerceExpr() { } + + Val* Fold(Val* v) const; + + DECLARE_SERIAL(VectorCoerceExpr); +}; + // An internal operator for flattening array indices that are records // into a list of individual values. class FlattenExpr : public UnaryExpr { diff --git a/src/Func.cc b/src/Func.cc index 6a9f00f45e..0f34da58a5 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -239,11 +239,15 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits, : Func(BRO_FUNC) { id = arg_id; - Body b; - b.stmts = AddInits(arg_body, aggr_inits); - b.priority = 0; - bodies.push_back(b); frame_size = arg_frame_size; + + if ( arg_body ) + { + Body b; + b.stmts = AddInits(arg_body, aggr_inits); + b.priority = 0; + bodies.push_back(b); + } } BroFunc::~BroFunc() @@ -267,6 +271,13 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const #ifdef PROFILE_BRO_FUNCTIONS DEBUG_MSG("Function: %s\n", id->Name()); #endif + if ( ! bodies.size() ) + { + // Can only happen for events. + assert(IsEvent()); + return 0 ; + } + SegmentProfiler(segment_logger, location); Frame* f = new Frame(frame_size, this, args); @@ -497,9 +508,11 @@ void builtin_run_time(const char* msg, BroObj* arg) } #include "bro.bif.func_h" +#include "logging.bif.func_h" #include "strings.bif.func_h" #include "bro.bif.func_def" +#include "logging.bif.func_def" #include "strings.bif.func_def" void init_builtin_funcs() @@ -511,6 +524,7 @@ void init_builtin_funcs() gap_info = internal_type("gap_info")->AsRecordType(); #include "bro.bif.func_init" +#include "logging.bif.func_init" #include "strings.bif.func_init" did_builtin_init = true; diff --git a/src/Func.h b/src/Func.h index f82b9ee539..b6cfab893b 100644 --- a/src/Func.h +++ b/src/Func.h @@ -15,6 +15,7 @@ class FuncType; class Stmt; class Frame; class ID; +class CallExpr; class Func : public BroObj { public: @@ -36,7 +37,8 @@ public: { return priority > other.priority; } // reverse sort }; - virtual const vector& GetBodies() const { return bodies; } + const vector& GetBodies() const { return bodies; } + bool HasBodies() const { return bodies.size(); } // virtual Val* Call(ListExpr* args) const = 0; virtual Val* Call(val_list* args, Frame* parent = 0) const = 0; diff --git a/src/ID.cc b/src/ID.cc index 8ba1913b87..eac4f4e916 100644 --- a/src/ID.cc +++ b/src/ID.cc @@ -235,6 +235,25 @@ void ID::UpdateValAttrs() } } } + + if ( Type()->Tag() == TYPE_RECORD ) + { + Attr* attr = attrs->FindAttr(ATTR_LOG); + if ( attr ) + { + // Apply &log to all record fields. + RecordType* rt = Type()->AsRecordType(); + for ( int i = 0; i < rt->NumFields(); ++i ) + { + TypeDecl* fd = rt->FieldDecl(i); + + if ( ! fd->attrs ) + fd->attrs = new Attributes(new attr_list, rt->FieldType(i), true); + + fd->attrs->AddAttr(new Attr(ATTR_LOG)); + } + } + } } void ID::AddAttrs(Attributes* a) diff --git a/src/LogMgr.cc b/src/LogMgr.cc new file mode 100644 index 0000000000..62e74668a9 --- /dev/null +++ b/src/LogMgr.cc @@ -0,0 +1,1419 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "LogMgr.h" +#include "Event.h" +#include "EventHandler.h" +#include "NetVar.h" +#include "Net.h" + +#include "LogWriterAscii.h" + +// Structure describing a log writer type. +struct LogWriterDefinition { + bro_int_t type; // The type. + const char *name; // Descriptive name for error messages. + bool (*init)(); // An optional one-time initialization function. + LogWriter* (*factory)(); // A factory function creating instances. +}; + +// Static table defining all availabel log writers. +LogWriterDefinition log_writers[] = { + { BifEnum::Log::WRITER_ASCII, "Ascii", 0, LogWriterAscii::Instantiate }, + + // End marker, don't touch. + { BifEnum::Log::WRITER_DEFAULT, "None", 0, (LogWriter* (*)())0 } +}; + +struct LogMgr::Filter { + string name; + EnumVal* id; + Func* pred; + Func* path_func; + string path; + Val* path_val; + EnumVal* writer; + bool local; + bool remote; + + int num_fields; + LogField** fields; + + // Vector indexed by field number. Each element is a list of record + // indices defining a path leading to the value across potential + // sub-records. + vector > indices; + + ~Filter(); +}; + +struct LogMgr::WriterInfo { + EnumVal* type; + double open_time; + Timer* rotation_timer; + LogWriter *writer; + }; + +struct LogMgr::Stream { + EnumVal* id; + bool enabled; + string name; + RecordType* columns; + EventHandlerPtr event; + list filters; + + typedef pair WriterPathPair; + + typedef map WriterMap; + + WriterMap writers; // Writers indexed by id/path pair. + + ~Stream(); + }; + +bool LogField::Read(SerializationFormat* fmt) + { + int t; + + bool success = (fmt->Read(&name, "name") && fmt->Read(&t, "type")); + type = (TypeTag) t; + + return success; + } + +bool LogField::Write(SerializationFormat* fmt) const + { + return (fmt->Write(name, "name") && fmt->Write((int)type, "type")); + } + +LogVal::~LogVal() + { + if ( (type == TYPE_ENUM || type == TYPE_STRING || type == TYPE_FILE) + && present ) + delete val.string_val; + + if ( type == TYPE_TABLE && present ) + { + for ( int i = 0; i < val.set_val.size; i++ ) + delete val.set_val.vals[i]; + + delete [] val.set_val.vals; + } + + if ( type == TYPE_VECTOR && present ) + { + for ( int i = 0; i < val.vector_val.size; i++ ) + delete val.vector_val.vals[i]; + + delete [] val.vector_val.vals; + } + } + +bool LogVal::IsCompatibleType(BroType* t, bool atomic_only) + { + if ( ! t ) + return false; + + switch ( t->Tag() ) { + case TYPE_BOOL: + case TYPE_INT: + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + case TYPE_SUBNET: + case TYPE_NET: + case TYPE_ADDR: + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + return true; + + case TYPE_RECORD: + return ! atomic_only; + + case TYPE_TABLE: + { + if ( atomic_only ) + return false; + + if ( ! t->IsSet() ) + return false; + + return IsCompatibleType(t->AsSetType()->Indices()->PureType()); + } + + case TYPE_VECTOR: + { + if ( atomic_only ) + return false; + + return IsCompatibleType(t->AsVectorType()->YieldType()); + } + + default: + return false; + } + + return false; + } + +bool LogVal::Read(SerializationFormat* fmt) + { + int ty; + + if ( ! (fmt->Read(&ty, "type") && fmt->Read(&present, "present")) ) + return false; + + type = (TypeTag)(ty); + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Read(&val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Read(&val.uint_val, "uint"); + + case TYPE_SUBNET: + { + uint32 net[4]; + if ( ! (fmt->Read(&net[0], "net0") && + fmt->Read(&net[1], "net1") && + fmt->Read(&net[2], "net2") && + fmt->Read(&net[3], "net3") && + fmt->Read(&val.subnet_val.width, "width")) ) + return false; + +#ifdef BROv6 + val.subnet_val.net[0] = net[0]; + val.subnet_val.net[1] = net[1]; + val.subnet_val.net[2] = net[2]; + val.subnet_val.net[3] = net[3]; +#else + val.subnet_val.net = net[0]; +#endif + return true; + } + + case TYPE_NET: + case TYPE_ADDR: + { + uint32 addr[4]; + if ( ! (fmt->Read(&addr[0], "addr0") && + fmt->Read(&addr[1], "addr1") && + fmt->Read(&addr[2], "addr2") && + fmt->Read(&addr[3], "addr3")) ) + return false; + +#ifdef BROv6 + val.addr_val[0] = addr[0]; + val.addr_val[1] = addr[1]; + val.addr_val[2] = addr[2]; + val.addr_val[3] = addr[3]; +#else + val.addr_val = addr[0]; +#endif + return true; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Read(&val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + { + val.string_val = new string; + return fmt->Read(val.string_val, "string"); + } + + case TYPE_TABLE: + { + if ( ! fmt->Read(&val.set_val.size, "set_size") ) + return false; + + val.set_val.vals = new LogVal* [val.set_val.size]; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + val.set_val.vals[i] = new LogVal; + + if ( ! val.set_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Read(&val.vector_val.size, "vector_size") ) + return false; + + val.vector_val.vals = new LogVal* [val.vector_val.size]; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + val.vector_val.vals[i] = new LogVal; + + if ( ! val.vector_val.vals[i]->Read(fmt) ) + return false; + } + + return true; + } + + default: + internal_error("unsupported type %s in LogVal::Write", type_name(type)); + } + + return false; + } + +bool LogVal::Write(SerializationFormat* fmt) const + { + if ( ! (fmt->Write((int)type, "type") && + fmt->Write(present, "present")) ) + return false; + + if ( ! present ) + return true; + + switch ( type ) { + case TYPE_BOOL: + case TYPE_INT: + return fmt->Write(val.int_val, "int"); + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + return fmt->Write(val.uint_val, "uint"); + + case TYPE_SUBNET: + { +#ifdef BROv6 + uint32* net = val.subnet_val; +#else + uint32 net[4]; + net[0] = val.subnet_val.net; + net[1] = net[2] = net[3] = 0; +#endif + return fmt->Write(net[0], "net0") && + fmt->Write(net[1], "net1") && + fmt->Write(net[2], "net2") && + fmt->Write(net[3], "net3") && + fmt->Write(val.subnet_val.width, "width"); + } + + case TYPE_NET: + case TYPE_ADDR: + { +#ifdef BROv6 + uint32* addr = val.addr_val; +#else + uint32 addr[4]; + addr[0] = val.addr_val; + addr[1] = addr[2] = addr[3] = 0; +#endif + return fmt->Write(addr[0], "addr0") && + fmt->Write(addr[1], "addr1") && + fmt->Write(addr[2], "addr2") && + fmt->Write(addr[3], "addr3"); + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + return fmt->Write(val.double_val, "double"); + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + return fmt->Write(*val.string_val, "string"); + + case TYPE_TABLE: + { + if ( ! fmt->Write(val.set_val.size, "set_size") ) + return false; + + for ( int i = 0; i < val.set_val.size; ++i ) + { + if ( ! val.set_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + case TYPE_VECTOR: + { + if ( ! fmt->Write(val.vector_val.size, "vector_size") ) + return false; + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + if ( ! val.vector_val.vals[i]->Write(fmt) ) + return false; + } + + return true; + } + + default: + internal_error("unsupported type %s in LogVal::REad", type_name(type)); + } + + return false; + } + +LogMgr::Filter::~Filter() + { + for ( int i = 0; i < num_fields; ++i ) + delete fields[i]; + + Unref(path_val); + } + +LogMgr::Stream::~Stream() + { + Unref(columns); + + for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ ) + { + WriterInfo* winfo = i->second; + + if ( winfo->rotation_timer ) + timer_mgr->Cancel(winfo->rotation_timer); + + Unref(winfo->type); + + delete winfo->writer; + delete i->second; + } + + for ( list::iterator f = filters.begin(); f != filters.end(); ++f ) + delete *f; + } + +LogMgr::LogMgr() + { + } + +LogMgr::~LogMgr() + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + delete *s; + } + +LogMgr::Stream* LogMgr::FindStream(EnumVal* id) + { + unsigned int idx = id->AsEnum(); + + if ( idx >= streams.size() || ! streams[idx] ) + return 0; + + return streams[idx]; + } + +void LogMgr::RemoveDisabledWriters(Stream* stream) + { + list disabled; + + for ( Stream::WriterMap::iterator j = stream->writers.begin(); j != stream->writers.end(); j++ ) + { + if ( j->second->writer->Disabled() ) + { + delete j->second; + disabled.push_back(j->first); + } + } + + for ( list::iterator j = disabled.begin(); j != disabled.end(); j++ ) + stream->writers.erase(*j); + } + +bool LogMgr::CreateStream(EnumVal* id, RecordVal* sval) + { + RecordType* rtype = sval->Type()->AsRecordType(); + + if ( ! same_type(rtype, BifType::Record::Log::Stream, 0) ) + { + run_time("sval argument not of right type"); + return false; + } + + RecordType* columns = sval->Lookup(rtype->FieldOffset("columns")) + ->AsType()->AsTypeType()->Type()->AsRecordType(); + + bool log_attr_present = false; + + for ( int i = 0; i < columns->NumFields(); i++ ) + { + if ( ! (columns->FieldDecl(i)->FindAttr(ATTR_LOG)) ) + continue; + + if ( ! LogVal::IsCompatibleType(columns->FieldType(i)) ) + { + run_time("type of field '%s' is not support for logging output", + columns->FieldName(i)); + + return false; + } + + log_attr_present = true; + } + + if ( ! log_attr_present ) + { + run_time("logged record type does not have any &log attributes"); + return false; + } + + Val* event_val = sval->Lookup(rtype->FieldOffset("ev")); + Func* event = event_val ? event_val->AsFunc() : 0; + + if ( event ) + { + // Make sure the event is prototyped as expected. + FuncType* etype = event->FType()->AsFuncType(); + + if ( ! etype->IsEvent() ) + { + run_time("stream event is a function, not an event"); + return false; + } + + const type_list* args = etype->ArgTypes()->Types(); + + if ( args->length() != 1 ) + { + run_time("stream event must take a single argument"); + return false; + } + + if ( ! same_type((*args)[0], columns) ) + { + run_time("stream event's argument type does not match column record type"); + return new Val(0, TYPE_BOOL); + } + } + + // Make sure the vector has an entries for all streams up to the one + // given. + + unsigned int idx = id->AsEnum(); + + while ( idx >= streams.size() ) + streams.push_back(0); + + if ( streams[idx] ) + // We already know this one, delete the previous definition. + delete streams[idx]; + + // Create new stream. + streams[idx] = new Stream; + streams[idx]->id = id->Ref()->AsEnumVal(); + streams[idx]->enabled = true; + streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); + streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + streams[idx]->columns = columns->Ref()->AsRecordType(); + + DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", + streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : ""); + + return true; + } + +bool LogMgr::EnableStream(EnumVal* id) + { + Stream* stream = FindStream(id); + + if ( ! stream ) + return false; + + if ( stream->enabled ) + return true; + + stream->enabled = true; + + DBG_LOG(DBG_LOGGING, "Reenabled logging stream '%s'", stream->name.c_str()); + return true; + } + +bool LogMgr::DisableStream(EnumVal* id) + { + Stream* stream = FindStream(id); + + if ( ! stream ) + return false; + + if ( ! stream->enabled ) + return true; + + stream->enabled = false; + + DBG_LOG(DBG_LOGGING, "Disabled logging stream '%s'", stream->name.c_str()); + return true; + } + +// Helper for recursive record field unrolling. +bool LogMgr::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, + TableVal* include, TableVal* exclude, string path, list indices) + { + for ( int i = 0; i < rt->NumFields(); ++i ) + { + BroType* t = rt->FieldType(i); + + // Ignore if &log not specified. + if ( ! rt->FieldDecl(i)->FindAttr(ATTR_LOG) ) + continue; + + list new_indices = indices; + new_indices.push_back(i); + + // Build path name. + string new_path; + + if ( ! path.size() ) + new_path = rt->FieldName(i); + else + new_path = path + "." + rt->FieldName(i); + + if ( t->InternalType() == TYPE_INTERNAL_OTHER ) + { + if ( t->Tag() == TYPE_RECORD ) + { + // Recurse. + if ( ! TraverseRecord(stream, filter, + t->AsRecordType(), + include, + exclude, + new_path, + new_indices) ) + return false; + + continue; + } + + else if ( t->Tag() == TYPE_TABLE && + t->AsTableType()->IsSet() ) + { + // That's ok, we handle it below. + } + + else if ( t->Tag() == TYPE_VECTOR ) + { + // That's ok, we handle it below. + } + + else if ( t->Tag() == TYPE_FILE ) + { + // That's ok, we handle it below. + } + + else + { + run_time("unsupported field type for log column"); + return false; + } + } + + // If include fields are specified, only include if explicitly listed. + if ( include ) + { + StringVal* new_path_val = new StringVal(new_path.c_str()); + bool result = include->Lookup(new_path_val); + + Unref(new_path_val); + + if ( ! result ) + continue; + } + + // If exclude fields are specified, do not only include if listed. + if ( exclude ) + { + StringVal* new_path_val = new StringVal(new_path.c_str()); + bool result = exclude->Lookup(new_path_val); + + Unref(new_path_val); + + if ( result ) + continue; + } + + // Alright, we want this field. + + filter->indices.push_back(new_indices); + + filter->fields = (LogField**) + realloc(filter->fields, + sizeof(LogField) * ++filter->num_fields); + + if ( ! filter->fields ) + { + run_time("out of memory in add_filter"); + return false; + } + + LogField* field = new LogField(); + field->name = new_path; + field->type = t->Tag(); + filter->fields[filter->num_fields - 1] = field; + } + + return true; + } + +bool LogMgr::AddFilter(EnumVal* id, RecordVal* fval) + { + RecordType* rtype = fval->Type()->AsRecordType(); + + if ( ! same_type(rtype, BifType::Record::Log::Filter, 0) ) + { + run_time("filter argument not of right type"); + return false; + } + + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + // Find the right writer type. + int idx = rtype->FieldOffset("writer"); + EnumVal* writer = fval->LookupWithDefault(idx)->AsEnumVal(); + + // Create a new Filter instance. + + Val* name = fval->LookupWithDefault(rtype->FieldOffset("name")); + Val* pred = fval->LookupWithDefault(rtype->FieldOffset("pred")); + Val* path_func = fval->LookupWithDefault(rtype->FieldOffset("path_func")); + Val* log_local = fval->LookupWithDefault(rtype->FieldOffset("log_local")); + Val* log_remote = fval->LookupWithDefault(rtype->FieldOffset("log_remote")); + + Filter* filter = new Filter; + filter->name = name->AsString()->CheckString(); + filter->id = id->Ref()->AsEnumVal(); + filter->pred = pred ? pred->AsFunc() : 0; + filter->path_func = path_func ? path_func->AsFunc() : 0; + filter->writer = writer->Ref()->AsEnumVal(); + filter->local = log_local->AsBool(); + filter->remote = log_remote->AsBool(); + + Unref(name); + Unref(pred); + Unref(path_func); + Unref(log_local); + Unref(log_remote); + + // Build the list of fields that the filter wants included, including + // potentially rolling out fields. + Val* include = fval->Lookup(rtype->FieldOffset("include")); + Val* exclude = fval->Lookup(rtype->FieldOffset("exclude")); + + filter->num_fields = 0; + filter->fields = 0; + if ( ! TraverseRecord(stream, filter, stream->columns, + include ? include->AsTableVal() : 0, + exclude ? exclude->AsTableVal() : 0, + "", list()) ) + return false; + + // Get the path for the filter. + Val* path_val = fval->Lookup(rtype->FieldOffset("path")); + + if ( path_val ) + { + filter->path = path_val->AsString()->CheckString(); + filter->path_val = path_val->Ref(); + } + + else + { + // If no path is given, use the Stream ID's namespace as the default + // if it has one, and it ID itself otherwise. + const char* s = stream->name.c_str(); + const char* e = strstr(s, "::"); + + if ( ! e ) + e = s + strlen(s); + + string path(s, e); + std::transform(path.begin(), path.end(), path.begin(), ::tolower); + + filter->path = path; + filter->path_val = new StringVal(path.c_str()); + } + + // Remove any filter with the same name we might already have. + RemoveFilter(id, filter->name); + + // Add the new one. + stream->filters.push_back(filter); + +#ifdef DEBUG + ODesc desc; + writer->Describe(&desc); + + DBG_LOG(DBG_LOGGING, "Created new filter '%s' for stream '%s'", + filter->name.c_str(), stream->name.c_str()); + + DBG_LOG(DBG_LOGGING, " writer : %s", desc.Description()); + DBG_LOG(DBG_LOGGING, " path : %s", filter->path.c_str()); + DBG_LOG(DBG_LOGGING, " path_func : %s", (filter->path_func ? "set" : "not set")); + DBG_LOG(DBG_LOGGING, " pred : %s", (filter->pred ? "set" : "not set")); + + for ( int i = 0; i < filter->num_fields; i++ ) + { + LogField* field = filter->fields[i]; + DBG_LOG(DBG_LOGGING, " field %10s: %s", + field->name.c_str(), type_name(field->type)); + } +#endif + + return true; + } + +bool LogMgr::RemoveFilter(EnumVal* id, StringVal* name) + { + return RemoveFilter(id, name->AsString()->CheckString()); + } + +bool LogMgr::RemoveFilter(EnumVal* id, string name) + { + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + for ( list::iterator i = stream->filters.begin(); + i != stream->filters.end(); ++i ) + { + if ( (*i)->name == name ) + { + Filter* filter = *i; + stream->filters.erase(i); + DBG_LOG(DBG_LOGGING, "Removed filter '%s' from stream '%s'", + filter->name.c_str(), stream->name.c_str()); + delete filter; + return true; + } + } + + // If we don't find the filter, we don't treat that as an error. + DBG_LOG(DBG_LOGGING, "No filter '%s' for removing from stream '%s'", + name.c_str(), stream->name.c_str()); + + return true; + } + +bool LogMgr::Write(EnumVal* id, RecordVal* columns) + { + bool error = false; + + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + if ( ! stream->enabled ) + return true; + + columns = columns->CoerceTo(stream->columns); + + if ( ! columns ) + { + run_time("incompatible log record type"); + return false; + } + + // Raise the log event. + if ( stream->event ) + { + val_list* vl = new val_list(1); + vl->append(columns->Ref()); + mgr.QueueEvent(stream->event, vl, SOURCE_LOCAL); + } + + // Send to each of our filters. + for ( list::iterator i = stream->filters.begin(); + i != stream->filters.end(); ++i ) + { + Filter* filter = *i; + string path = filter->path; + + if ( filter->pred ) + { + // See whether the predicates indicates that we want + // to log this record. + val_list vl(1); + vl.append(columns->Ref()); + Val* v = filter->pred->Call(&vl); + int result = v->AsBool(); + Unref(v); + + if ( ! result ) + continue; + } + + if ( filter->path_func ) + { + val_list vl(2); + vl.append(id->Ref()); + vl.append(filter->path_val->Ref()); + Val* v = filter->path_func->Call(&vl); + + if ( ! v->Type()->Tag() == TYPE_STRING ) + { + run_time("path_func did not return string"); + Unref(v); + return false; + } + + path = v->AsString()->CheckString(); + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Path function for filter '%s' on stream '%s' return '%s'", + filter->name.c_str(), stream->name.c_str(), path.c_str()); +#endif + } + + // See if we already have a writer for this path. + Stream::WriterMap::iterator w = + stream->writers.find(Stream::WriterPathPair(filter->writer->AsEnum(), path)); + + LogWriter* writer = 0; + + if ( w != stream->writers.end() ) + // We have a writer already. + writer = w->second->writer; + + else + { + // No, need to create one. + + // Copy the fields for LogWriter::Init() as it will take + // ownership. + LogField** arg_fields = new LogField*[filter->num_fields]; + + for ( int j = 0; j < filter->num_fields; ++j ) + arg_fields[j] = new LogField(*filter->fields[j]); + + if ( filter->local ) + { + writer = CreateWriter(stream->id, filter->writer, + path, filter->num_fields, + arg_fields); + + if ( ! writer ) + { + Unref(columns); + return false; + } + } + + if ( filter->remote ) + remote_serializer->SendLogCreateWriter(stream->id, + filter->writer, + path, + filter->num_fields, + arg_fields); + } + + // Alright, can do the write now. + + LogVal** vals = RecordToFilterVals(stream, filter, columns); + + if ( filter->remote ) + remote_serializer->SendLogWrite(stream->id, + filter->writer, + path, + filter->num_fields, + vals); + + if ( filter->local && ! writer->Write(filter->num_fields, vals) ) + error = true; + +#ifdef DEBUG + DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", + filter->name.c_str(), stream->name.c_str()); +#endif + } + + Unref(columns); + + if ( error ) + RemoveDisabledWriters(stream); + + return true; + } + +LogVal* LogMgr::ValToLogVal(Val* val, BroType* ty) + { + if ( ! ty ) + ty = val->Type(); + + if ( ! val ) + return new LogVal(ty->Tag(), false); + + LogVal* lval = new LogVal(ty->Tag()); + + switch ( lval->type ) { + case TYPE_BOOL: + case TYPE_INT: + lval->val.int_val = val->InternalInt(); + break; + + case TYPE_ENUM: + { + const char* s = + val->Type()->AsEnumType()->Lookup(val->InternalInt()); + + lval->val.string_val = new string(s); + break; + } + + case TYPE_COUNT: + case TYPE_COUNTER: + lval->val.uint_val = val->InternalUnsigned(); + break; + + case TYPE_PORT: + lval->val.uint_val = val->AsPortVal()->Port(); + break; + + case TYPE_SUBNET: + lval->val.subnet_val = *val->AsSubNet(); + break; + + case TYPE_NET: + case TYPE_ADDR: + { + addr_type t = val->AsAddr(); + copy_addr(&t, &lval->val.addr_val); + break; + } + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + lval->val.double_val = val->InternalDouble(); + break; + + case TYPE_STRING: + { + const BroString* s = val->AsString(); + lval->val.string_val = + new string((const char*) s->Bytes(), s->Len()); + break; + } + + case TYPE_FILE: + { + const BroFile* f = val->AsFile(); + lval->val.string_val = new string(f->Name()); + break; + } + + case TYPE_TABLE: + { + ListVal* set = val->AsTableVal()->ConvertToPureList(); + lval->val.set_val.size = set->Length(); + lval->val.set_val.vals = new LogVal* [lval->val.set_val.size]; + + for ( int i = 0; i < lval->val.set_val.size; i++ ) + lval->val.set_val.vals[i] = ValToLogVal(set->Index(i)); + + break; + } + + case TYPE_VECTOR: + { + VectorVal* vec = val->AsVectorVal(); + lval->val.vector_val.size = vec->Size(); + lval->val.vector_val.vals = + new LogVal* [lval->val.vector_val.size]; + + for ( int i = 0; i < lval->val.vector_val.size; i++ ) + { + lval->val.vector_val.vals[i] = + ValToLogVal(vec->Lookup(VECTOR_MIN + i), + vec->Type()->YieldType()); + } + + break; + } + + default: + internal_error("unsupported type for log_write"); + } + + return lval; + } + +LogVal** LogMgr::RecordToFilterVals(Stream* stream, Filter* filter, + RecordVal* columns) + { + LogVal** vals = new LogVal*[filter->num_fields]; + + for ( int i = 0; i < filter->num_fields; ++i ) + { + TypeTag type = TYPE_ERROR; + Val* val = columns; + + // For each field, first find the right value, which can + // potentially be nested inside other records. + list& indices = filter->indices[i]; + + for ( list::iterator j = indices.begin(); j != indices.end(); ++j ) + { + type = val->Type()->AsRecordType()->FieldType(*j)->Tag(); + val = val->AsRecordVal()->Lookup(*j); + + if ( ! val ) + { + // Value, or any of its parents, is not set. + vals[i] = new LogVal(type, false); + break; + } + } + + if ( val ) + vals[i] = ValToLogVal(val); + } + + return vals; + } + +LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, LogField** fields) + { + Stream* stream = FindStream(id); + + if ( ! stream ) + // Don't know this stream. + return false; + + Stream::WriterMap::iterator w = + stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); + + if ( w != stream->writers.end() ) + // If we already have a writer for this. That's fine, we just + // return it. + return w->second->writer; + + // Need to instantiate a new writer. + + LogWriterDefinition* ld = log_writers; + + while ( true ) + { + if ( ld->type == BifEnum::Log::WRITER_DEFAULT ) + { + run_time("unknow writer when creating writer"); + return 0; + } + + if ( ld->type == writer->AsEnum() ) + break; + + if ( ! ld->factory ) + // Oops, we can't instantiate this guy. + return 0; + + // If the writer has an init function, call it. + if ( ld->init ) + { + if ( (*ld->init)() ) + // Clear the init function so that we won't + // call it again later. + ld->init = 0; + else + // Init failed, disable by deleting factory + // function. + ld->factory = 0; + + DBG_LOG(DBG_LOGGING, "failed to init writer class %s", + ld->name); + + return false; + } + + ++ld; + } + + assert(ld->factory); + LogWriter* writer_obj = (*ld->factory)(); + + if ( ! writer_obj->Init(path, num_fields, fields) ) + { + DBG_LOG(DBG_LOGGING, "failed to init instance of writer %s", + ld->name); + + return 0; + } + + WriterInfo* winfo = new WriterInfo; + winfo->type = writer->Ref()->AsEnumVal(); + winfo->writer = writer_obj; + winfo->open_time = network_time; + winfo->rotation_timer = 0; + InstallRotationTimer(winfo); + + stream->writers.insert( + Stream::WriterMap::value_type(Stream::WriterPathPair(writer->AsEnum(), path), + winfo)); + + return writer_obj; + } + +bool LogMgr::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, + LogVal** vals) + { + Stream* stream = FindStream(id); + + if ( ! stream ) + { + // Don't know this stream. +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_LOGGING, "unknown stream %s in LogMgr::Write()", + desc.Description()); +#endif + return false; + } + + if ( ! stream->enabled ) + return true; + + Stream::WriterMap::iterator w = + stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), path)); + + if ( w == stream->writers.end() ) + { + // Don't know this writer. +#ifdef DEBUG + ODesc desc; + id->Describe(&desc); + DBG_LOG(DBG_LOGGING, "unknown writer %s in LogMgr::Write()", + desc.Description()); +#endif + return false; + } + + bool success = w->second->writer->Write(num_fields, vals); + + DBG_LOG(DBG_LOGGING, + "Wrote pre-filtered record to path '%s' on stream '%s' [%s]", + path.c_str(), stream->name.c_str(), (success ? "ok" : "error")); + + return success; + } + +void LogMgr::SendAllWritersTo(RemoteSerializer::PeerID peer) + { + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + { + Stream* stream = (*s); + + if ( ! stream ) + continue; + + for ( Stream::WriterMap::iterator i = stream->writers.begin(); + i != stream->writers.end(); i++ ) + { + LogWriter* writer = i->second->writer; + EnumVal writer_val(i->first.first, BifType::Enum::Log::Writer); + remote_serializer->SendLogCreateWriter(peer, (*s)->id, + &writer_val, + i->first.second, + writer->NumFields(), + writer->Fields()); + } + } + } + +bool LogMgr::SetBuf(EnumVal* id, bool enabled) + { + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + for ( Stream::WriterMap::iterator i = stream->writers.begin(); + i != stream->writers.end(); i++ ) + i->second->writer->SetBuf(enabled); + + RemoveDisabledWriters(stream); + + return true; + } + +bool LogMgr::Flush(EnumVal* id) + { + Stream* stream = FindStream(id); + if ( ! stream ) + return false; + + if ( ! stream->enabled ) + return true; + + for ( Stream::WriterMap::iterator i = stream->writers.begin(); + i != stream->writers.end(); i++ ) + i->second->writer->Flush(); + + RemoveDisabledWriters(stream); + + return true; + } + +void LogMgr::Error(LogWriter* writer, const char* msg) + { + run_time(fmt("error with writer for %s: %s", + writer->Path().c_str(), msg)); + } + +// Timer which on dispatching rotates the filter. +class RotationTimer : public Timer { +public: + RotationTimer(double t, LogMgr::WriterInfo* arg_winfo, bool arg_rotate) + : Timer(t, TIMER_ROTATE) + { + winfo = arg_winfo; + rotate = arg_rotate; + } + + ~RotationTimer(); + + void Dispatch(double t, int is_expire); + +protected: + LogMgr::WriterInfo* winfo; + bool rotate; +}; + +RotationTimer::~RotationTimer() + { + if ( winfo->rotation_timer == this ) + winfo->rotation_timer = 0; + } + +void RotationTimer::Dispatch(double t, int is_expire) + { + winfo->rotation_timer = 0; + + if ( rotate ) + log_mgr->Rotate(winfo); + + if ( ! is_expire ) + { + winfo->open_time = network_time; + log_mgr->InstallRotationTimer(winfo); + } + } + +RecordVal* LogMgr::LookupRotationControl(EnumVal* writer, string path) + { + TableVal* rc = BifConst::Log::rotation_control->AsTableVal(); + + ListVal* index = new ListVal(TYPE_ANY); + index->Append(writer->Ref()); + index->Append(new StringVal(path.c_str())); + + Val* r = rc->Lookup(index); + assert(r); + + Unref(index); + + return r->AsRecordVal(); + } + +void LogMgr::InstallRotationTimer(WriterInfo* winfo) + { + if ( terminating ) + return; + + if ( winfo->rotation_timer ) + { + timer_mgr->Cancel(winfo->rotation_timer); + winfo->rotation_timer = 0; + } + + RecordVal* rc = + LookupRotationControl(winfo->type, winfo->writer->Path()); + + int idx = rc->Type()->AsRecordType()->FieldOffset("interv"); + double rotation_interval = rc->LookupWithDefault(idx)->AsInterval(); + + if ( rotation_interval ) + { + // When this is called for the first time, network_time can still be + // zero. If so, we set a timer which fires immediately but doesn't + // rotate when it expires. + if ( ! network_time ) + winfo->rotation_timer = new RotationTimer(1, winfo, false); + else + { + if ( ! winfo->open_time ) + winfo->open_time = network_time; + + const char* base_time = log_rotate_base_time ? + log_rotate_base_time->AsString()->CheckString() : 0; + + double delta_t = + calc_next_rotate(rotation_interval, base_time); + + winfo->rotation_timer = + new RotationTimer(network_time + delta_t, winfo, true); + } + + timer_mgr->Add(winfo->rotation_timer); + + DBG_LOG(DBG_LOGGING, "Scheduled rotation timer for %s to %.6f", + winfo->writer->Path().c_str(), winfo->rotation_timer->Time()); + } + } + +void LogMgr::Rotate(WriterInfo* winfo) + { + DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f", + winfo->writer->Path().c_str(), network_time); + + // Create the RotationInfo record. + RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo); + info->Assign(0, winfo->type->Ref()); + info->Assign(1, new StringVal(winfo->writer->Path().c_str())); + info->Assign(2, new Val(winfo->open_time, TYPE_TIME)); + info->Assign(3, new Val(network_time, TYPE_TIME)); + + // Call the function building us the new path. + + Func* rotation_path_func = + internal_func("Log::default_rotation_path_func"); + + RecordVal* rc = + LookupRotationControl(winfo->type, winfo->writer->Path()); + + int idx = rc->Type()->AsRecordType()->FieldOffset("postprocessor"); + + string rotation_postprocessor = + rc->LookupWithDefault(idx)->AsString()->CheckString(); + + val_list vl(1); + vl.append(info); + Val* result = rotation_path_func->Call(&vl); + string new_path = result->AsString()->CheckString(); + Unref(result); + + winfo->writer->Rotate(new_path, rotation_postprocessor, + winfo->open_time, network_time, terminating); + } + + diff --git a/src/LogMgr.h b/src/LogMgr.h new file mode 100644 index 0000000000..ca5e2b5c49 --- /dev/null +++ b/src/LogMgr.h @@ -0,0 +1,133 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// A class managing log writers and filters. + +#ifndef LOGMGR_H +#define LOGMGR_H + +#include "Val.h" +#include "EventHandler.h" +#include "RemoteSerializer.h" + +class SerializationFormat; + +// Description of a log field. +struct LogField { + string name; + TypeTag type; + + LogField() { } + LogField(const LogField& other) + : name(other.name), type(other.type) { } + + // (Un-)serialize. + bool Read(SerializationFormat* fmt); + bool Write(SerializationFormat* fmt) const; +}; + +// Values as logged by a writer. +struct LogVal { + TypeTag type; + bool present; // False for unset fields. + + // The following union is a subset of BroValUnion, including only the + // types we can log directly. + struct set_t { bro_int_t size; LogVal** vals; }; + typedef set_t vec_t; + + union _val { + bro_int_t int_val; + bro_uint_t uint_val; + addr_type addr_val; + subnet_type subnet_val; + double double_val; + string* string_val; + set_t set_val; + vec_t vector_val; + } val; + + LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true) + : type(arg_type), present(arg_present) {} + ~LogVal(); + + // (Un-)serialize. + bool Read(SerializationFormat* fmt); + bool Write(SerializationFormat* fmt) const; + + // Returns true if the type can be logged the framework. If + // `atomic_only` is true, will not permit composite types. + static bool IsCompatibleType(BroType* t, bool atomic_only=false); + +private: + LogVal(const LogVal& other) { } +}; + +class LogWriter; +class RemoteSerializer; +class RotationTimer; + +class LogMgr { +public: + LogMgr(); + ~LogMgr(); + + // These correspond to the BiFs visible on the scripting layer. The + // actual BiFs just forward here. + bool CreateStream(EnumVal* id, RecordVal* stream); + bool EnableStream(EnumVal* id); + bool DisableStream(EnumVal* id); + bool AddFilter(EnumVal* id, RecordVal* filter); + bool RemoveFilter(EnumVal* id, StringVal* name); + bool RemoveFilter(EnumVal* id, string name); + bool Write(EnumVal* id, RecordVal* columns); + bool SetBuf(EnumVal* id, bool enabled); // Adjusts all writers. + bool Flush(EnumVal* id); // Flushes all writers.. + +protected: + friend class LogWriter; + friend class RemoteSerializer; + friend class RotationTimer; + + //// Function also used by the RemoteSerializer. + + // Takes ownership of fields. + LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path, + int num_fields, LogField** fields); + + // Takes ownership of values.. + bool Write(EnumVal* id, EnumVal* writer, string path, + int num_fields, LogVal** vals); + + // Announces all instantiated writers to peer. + void SendAllWritersTo(RemoteSerializer::PeerID peer); + + //// Functions safe to use by writers. + + // Reports an error for the given writer. + void Error(LogWriter* writer, const char* msg); + +private: + struct Filter; + struct Stream; + struct WriterInfo; + + bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, + TableVal* include, TableVal* exclude, string path, list indices); + + LogVal** RecordToFilterVals(Stream* stream, Filter* filter, + RecordVal* columns); + + LogVal* ValToLogVal(Val* val, BroType* ty = 0); + Stream* FindStream(EnumVal* id); + void RemoveDisabledWriters(Stream* stream); + void InstallRotationTimer(WriterInfo* winfo); + void Rotate(WriterInfo* info); + RecordVal* LookupRotationControl(EnumVal* writer, string path); + Filter* FindFilter(EnumVal* id, StringVal* filter); + + vector streams; // Indexed by stream enum. +}; + +extern LogMgr* log_mgr; + +#endif diff --git a/src/LogWriter.cc b/src/LogWriter.cc new file mode 100644 index 0000000000..6e12e3a6ed --- /dev/null +++ b/src/LogWriter.cc @@ -0,0 +1,188 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "util.h" +#include "LogWriter.h" + +LogWriter::LogWriter() + { + buf = 0; + buf_len = 1024; + buffering = true; + disabled = false; + } + +LogWriter::~LogWriter() + { + if ( buf ) + free(buf); + + delete [] fields; + } + +bool LogWriter::Init(string arg_path, int arg_num_fields, + const LogField* const * arg_fields) + { + path = arg_path; + num_fields = arg_num_fields; + fields = arg_fields; + + if ( ! DoInit(arg_path, arg_num_fields, arg_fields) ) + { + disabled = true; + return false; + } + + return true; + } + +bool LogWriter::Write(int arg_num_fields, LogVal** vals) + { + // Double-check that the arguments match. If we get this from remote, + // something might be mixed up. + if ( num_fields != arg_num_fields ) + { + DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)", + arg_num_fields, num_fields); + + return false; + } + + for ( int i = 0; i < num_fields; ++i ) + { + if ( vals[i]->type != fields[i]->type ) + { + DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)", + vals[i]->type, fields[i]->type); + return false; + } + } + + bool result = DoWrite(num_fields, fields, vals); + + DeleteVals(vals); + + if ( ! result ) + disabled = true; + + return result; + } + +bool LogWriter::SetBuf(bool enabled) + { + if ( enabled == buffering ) + // No change. + return true; + + buffering = enabled; + + if ( ! DoSetBuf(enabled) ) + { + disabled = true; + return false; + } + + return true; + } + +bool LogWriter::Rotate(string rotated_path, string postprocessor, double open, + double close, bool terminating) + { + if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) ) + { + disabled = true; + return false; + } + + return true; + } + +bool LogWriter::Flush() + { + if ( ! DoFlush() ) + { + disabled = true; + return false; + } + + return true; + } + +void LogWriter::Finish() + { + DoFinish(); + } + +const char* LogWriter::Fmt(const char* format, ...) + { + if ( ! buf ) + buf = (char*) malloc(buf_len); + + va_list al; + va_start(al, format); + int n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + + if ( (unsigned int) n >= buf_len ) + { // Not enough room, grow the buffer. + buf_len = n + 32; + buf = (char*) realloc(buf, buf_len); + + // Is it portable to restart? + va_start(al, format); + n = safe_vsnprintf(buf, buf_len, format, al); + va_end(al); + } + + return buf; + } + +void LogWriter::Error(const char *msg) + { + log_mgr->Error(this, msg); + } + +void LogWriter::DeleteVals(LogVal** vals) + { + for ( int i = 0; i < num_fields; i++ ) + delete vals[i]; + } + +bool LogWriter::RunPostProcessor(string fname, string postprocessor, + string old_name, double open, double close, + bool terminating) + { + // This function operates in a way that is backwards-compatible with + // the old Bro log rotation scheme. + + if ( ! postprocessor.size() ) + return true; + + const char* const fmt = "%y-%m-%d_%H.%M.%S"; + + struct tm tm1; + struct tm tm2; + + time_t tt1 = (time_t)open; + time_t tt2 = (time_t)close; + + localtime_r(&tt1, &tm1); + localtime_r(&tt2, &tm2); + + char buf1[128]; + char buf2[128]; + + strftime(buf1, sizeof(buf1), fmt, &tm1); + strftime(buf2, sizeof(buf2), fmt, &tm2); + + string cmd = postprocessor; + cmd += " " + fname; + cmd += " " + old_name; + cmd += " " + string(buf1); + cmd += " " + string(buf2); + cmd += " " + string(terminating ? "1" : "0"); + cmd += " &"; + + system(cmd.c_str()); + + return true; + } diff --git a/src/LogWriter.h b/src/LogWriter.h new file mode 100644 index 0000000000..8dcd05a67f --- /dev/null +++ b/src/LogWriter.h @@ -0,0 +1,188 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Interface API for a log writer backend. The LogMgr creates a separate +// writer instance of pair of (writer type, output path). +// +// Note thay classes derived from LogWriter must be fully thread-safe and not +// use any non-thread-safe Bro functionality (which includes almost +// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!. +// +// The one exception to this rule is the constructor: it is guaranteed to be +// executed inside the main thread and can thus in particular access global +// script variables. + +#ifndef LOGWRITER_H +#define LOGWRITER_H + +#include "LogMgr.h" +#include "BroString.h" + +class LogWriter { +public: + LogWriter(); + virtual ~LogWriter(); + + //// Interface methods to interact with the writer. Note that these + //// methods are not necessarily thread-safe and must be called only + //// from the main thread (which will typically mean only from the + //// LogMgr). In particular, they must not be called from the + //// writer's derived implementation. + + // One-time initialization of the writer to define the logged fields. + // Interpretation of "path" is left to the writer, and will be + // corresponding the value configured on the script-level. + // + // Returns false if an error occured, in which case the writer must + // not be used further. + // + // The new instance takes ownership of "fields", and will delete them + // when done. + bool Init(string path, int num_fields, const LogField* const * fields); + + // Writes one log entry. The method takes ownership of "vals" and + // will return immediately after queueing the write request, which is + // potentially before output has actually been written out. + // + // num_fields and the types of the LogVals must match what was passed + // to Init(). + // + // Returns false if an error occured, in which case the writer must + // not be used any further. + bool Write(int num_fields, LogVal** vals); + + // Sets the buffering status for the writer, if the writer supports + // that. (If not, it will be ignored). + bool SetBuf(bool enabled); + + // Flushes any currently buffered output, if the writer supports + // that. (If not, it will be ignored). + bool Flush(); + + // Triggers rotation, if the writer supports that. (If not, it will + // be ignored). + bool Rotate(string rotated_path, string postprocessor, double open, + double close, bool terminating); + + // Finishes writing to this logger regularly. Must not be called if + // an error has been indicated earlier. After calling this, no + // further writing must be performed. + void Finish(); + + //// Thread-safe methods that may be called from the writer + //// implementation. + + // The following methods return the information as passed to Init(). + const string Path() const { return path; } + int NumFields() const { return num_fields; } + const LogField* const * Fields() const { return fields; } + +protected: + + // Methods for writers to override. If any of these returs false, it + // will be assumed that a fatal error has occured that prevents the + // writer from further operation. It will then be disabled and + // deleted. When return false, the writer should also report the + // error via Error(). Note that even if a writer does not support the + // functionality for one these methods (like rotation), it must still + // return true if that is not to be considered a fatal error. + // + // Called once for initialization of the writer. + virtual bool DoInit(string path, int num_fields, + const LogField* const * fields) = 0; + + // Called once per log entry to record. + virtual bool DoWrite(int num_fields, const LogField* const * fields, + LogVal** vals) = 0; + + // Called when the buffering status for this writer is changed. If + // buffering is disabled, the writer should attempt to write out + // information as quickly as possible even if doing so may have a + // performance impact. If enabled (which is the default), it may + // buffer data as helpful and write it out later in a way optimized + // for performance. The current buffering state can be queried via + // IsBuf(). + // + // A writer may ignore buffering changes if it doesn't fit with its + // semantics (but must still return true in that case). + virtual bool DoSetBuf(bool enabled) = 0; + + // Called to flush any currently buffered output. + // + // A writer may ignore flush requests if it doesn't fit with its + // semantics (but must still return true in that case). + virtual bool DoFlush() = 0; + + // Called when a log output is to be rotated. Most directly this only + // applies to writers writing into files, which should then close the + // current file and open a new one. However, a writer may also + // trigger other apppropiate actions if semantics are similar. + // + // "rotate_path" reflects the path to where the rotated output is to + // be moved, with specifics depending on the writer. It should + // generally be interpreted in a way consistent with that of "path" + // as passed into DoInit(). As an example, for file-based output, + // "rotate_path" could be the original filename extended with a + // timestamp indicating the time of the rotation. + + // "postprocessor" is the name of a command to execute on the rotated + // file. If empty, no postprocessing should take place; if given but + // the writer doesn't support postprocessing, it can be ignored (but + // the method must still return true in that case). + + // "open" and "close" are the network time's when the *current* file + // was opened and closed, respectively. + // + // "terminating" indicated whether the rotation request occurs due + // the main Bro prcoess terminating (and not because we've reach a + // regularly scheduled time for rotation). + // + // A writer may ignore rotation requests if it doesn't fit with its + // semantics (but must still return true in that case). + virtual bool DoRotate(string rotated_path, string postprocessor, + double open, double close, bool terminating) = 0; + + // Called once on termination. Not called when any of the other + // methods has previously signaled an error, i.e., executing this + // method signals a regular shutdown of the writer. + virtual void DoFinish() = 0; + + //// Methods for writers to use. These are thread-safe. + + // A thread-safe version of fmt(). + const char* Fmt(const char* format, ...); + + // Returns the current buffering state. + bool IsBuf() { return buffering; } + + // Reports an error to the user. + void Error(const char *msg); + + // Runs a post-processor on the given file. Parameters correspond to + // those of DoRotate(). + bool RunPostProcessor(string fname, string postprocessor, + string old_name, double open, double close, + bool terminating); + +private: + friend class LogMgr; + + // When an error occurs, we call this method to set a flag marking + // the writer as disabled. The LogMgr will check the flag later and + // remove the writer. + bool Disabled() { return disabled; } + + // Deletes the values as passed into Write(). + void DeleteVals(LogVal** vals); + + string path; + int num_fields; + const LogField* const * fields; + bool buffering; + bool disabled; + + // For implementing Fmt(). + char* buf; + unsigned int buf_len; +}; + +#endif diff --git a/src/LogWriterAscii.cc b/src/LogWriterAscii.cc new file mode 100644 index 0000000000..4c54e76715 --- /dev/null +++ b/src/LogWriterAscii.cc @@ -0,0 +1,266 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include +#include + +#include "LogWriterAscii.h" +#include "NetVar.h" + +LogWriterAscii::LogWriterAscii() + { + file = 0; + + output_to_stdout = BifConst::LogAscii::output_to_stdout; + include_header = BifConst::LogAscii::include_header; + + separator_len = BifConst::LogAscii::separator->Len(); + separator = new char[separator_len]; + memcpy(separator, BifConst::LogAscii::separator->Bytes(), + separator_len); + + set_separator_len = BifConst::LogAscii::set_separator->Len(); + set_separator = new char[set_separator_len]; + memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(), + set_separator_len); + + empty_field_len = BifConst::LogAscii::empty_field->Len(); + empty_field = new char[empty_field_len]; + memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(), + empty_field_len); + + unset_field_len = BifConst::LogAscii::unset_field->Len(); + unset_field = new char[unset_field_len]; + memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(), + unset_field_len); + + header_prefix_len = BifConst::LogAscii::header_prefix->Len(); + header_prefix = new char[header_prefix_len]; + memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(), + header_prefix_len); + + } + +LogWriterAscii::~LogWriterAscii() + { + if ( file ) + fclose(file); + + delete [] separator; + delete [] set_separator; + delete [] empty_field; + delete [] unset_field; + delete [] header_prefix; + } + +bool LogWriterAscii::DoInit(string path, int num_fields, + const LogField* const * fields) + { + if ( output_to_stdout ) + path = "/dev/stdout"; + + fname = IsSpecial(path) ? path : path + ".log"; + + if ( ! (file = fopen(fname.c_str(), "w")) ) + { + Error(Fmt("cannot open %s: %s", fname.c_str(), + strerror(errno))); + + return false; + } + + if ( include_header ) + { + if ( fwrite(header_prefix, header_prefix_len, 1, file) != 1 ) + goto write_error; + + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 && + fwrite(separator, separator_len, 1, file) != 1 ) + goto write_error; + + const LogField* field = fields[i]; + + if ( fputs(field->name.c_str(), file) == EOF ) + goto write_error; + } + + if ( fputc('\n', file) == EOF ) + goto write_error; + } + + return true; + +write_error: + Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno))); + return false; + } + +bool LogWriterAscii::DoFlush() + { + fflush(file); + return true; + } + +void LogWriterAscii::DoFinish() + { + } + +bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field) + { + if ( ! val->present ) + { + desc->AddN(unset_field, unset_field_len); + return true; + } + + switch ( val->type ) { + + case TYPE_BOOL: + desc->Add(val->val.int_val ? "T" : "F"); + break; + + case TYPE_INT: + desc->Add(val->val.int_val); + break; + + case TYPE_COUNT: + case TYPE_COUNTER: + case TYPE_PORT: + desc->Add(val->val.uint_val); + break; + + case TYPE_SUBNET: + desc->Add(dotted_addr(val->val.subnet_val.net)); + desc->Add("/"); + desc->Add(val->val.subnet_val.width); + break; + + case TYPE_NET: + case TYPE_ADDR: + desc->Add(dotted_addr(val->val.addr_val)); + break; + + case TYPE_DOUBLE: + case TYPE_TIME: + case TYPE_INTERVAL: + desc->Add(val->val.double_val); + break; + + case TYPE_ENUM: + case TYPE_STRING: + case TYPE_FILE: + { + int size = val->val.string_val->size(); + if ( size ) + desc->AddN(val->val.string_val->data(), val->val.string_val->size()); + else + desc->AddN(empty_field, empty_field_len); + break; + } + + case TYPE_TABLE: + { + if ( ! val->val.set_val.size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + for ( int j = 0; j < val->val.set_val.size; j++ ) + { + if ( j > 0 ) + desc->AddN(set_separator, set_separator_len); + + if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) ) + return false; + } + + break; + } + + case TYPE_VECTOR: + { + if ( ! val->val.vector_val.size ) + { + desc->AddN(empty_field, empty_field_len); + break; + } + + for ( int j = 0; j < val->val.vector_val.size; j++ ) + { + if ( j > 0 ) + desc->AddN(set_separator, set_separator_len); + + if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) ) + return false; + } + + break; + } + + default: + Error(Fmt("unsupported field format %d for %s", val->type, + field->name.c_str())); + return false; + } + + return true; + } + +bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields, + LogVal** vals) + { + ODesc desc(DESC_READABLE); + desc.SetEscape(separator, separator_len); + + for ( int i = 0; i < num_fields; i++ ) + { + if ( i > 0 ) + desc.AddRaw(separator, separator_len); + + if ( ! DoWriteOne(&desc, vals[i], fields[i]) ) + return false; + } + + desc.Add("\n"); + + if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 ) + { + Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno))); + return false; + } + + if ( IsBuf() ) + fflush(file); + + return true; + } + +bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open, + double close, bool terminating) + { + if ( IsSpecial(Path()) ) + // Don't rotate special files. + return true; + + fclose(file); + + string nname = rotated_path + ".log"; + rename(fname.c_str(), nname.c_str()); + + if ( postprocessor.size() && + ! RunPostProcessor(nname, postprocessor, fname.c_str(), + open, close, terminating) ) + return false; + + return DoInit(Path(), NumFields(), Fields()); + } + +bool LogWriterAscii::DoSetBuf(bool enabled) + { + // Nothing to do. + return true; + } + + diff --git a/src/LogWriterAscii.h b/src/LogWriterAscii.h new file mode 100644 index 0000000000..fecbd9e94c --- /dev/null +++ b/src/LogWriterAscii.h @@ -0,0 +1,55 @@ +// See the file "COPYING" in the main distribution directory for copyright. +// +// Log writer for delimiter-separated ASCII logs. + +#ifndef LOGWRITERASCII_H +#define LOGWRITERASCII_H + +#include "LogWriter.h" + +class LogWriterAscii : public LogWriter { +public: + LogWriterAscii(); + ~LogWriterAscii(); + + static LogWriter* Instantiate() { return new LogWriterAscii; } + +protected: + virtual bool DoInit(string path, int num_fields, + const LogField* const * fields); + virtual bool DoWrite(int num_fields, const LogField* const * fields, + LogVal** vals); + virtual bool DoSetBuf(bool enabled); + virtual bool DoRotate(string rotated_path, string postprocessr, + double open, double close, bool terminating); + virtual bool DoFlush(); + virtual void DoFinish(); + +private: + bool IsSpecial(string path) { return path.find("/dev/") == 0; } + bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field); + + FILE* file; + string fname; + + // Options set from the script-level. + bool output_to_stdout; + bool include_header; + + char* separator; + int separator_len; + + char* set_separator; + int set_separator_len; + + char* empty_field; + int empty_field_len; + + char* unset_field; + int unset_field_len; + + char* header_prefix; + int header_prefix_len; +}; + +#endif diff --git a/src/Logger.h b/src/Logger.h index 0c3f0d1ed7..6512776550 100644 --- a/src/Logger.h +++ b/src/Logger.h @@ -1,5 +1,3 @@ -// $Id: Logger.h 6219 2008-10-01 05:39:07Z vern $ -// // See the file "COPYING" in the main distribution directory for copyright. #ifndef logger_h diff --git a/src/Login.h b/src/Login.h index 4bcecb0634..b186cc52d2 100644 --- a/src/Login.h +++ b/src/Login.h @@ -1,5 +1,3 @@ -// $Id: Login.h 6219 2008-10-01 05:39:07Z vern $ -// // See the file "COPYING" in the main distribution directory for copyright. #ifndef login_h diff --git a/src/NetVar.cc b/src/NetVar.cc index a8644fc059..6b88a75b1a 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -263,6 +263,7 @@ TableType* id_table; #include "const.bif.netvar_def" #include "types.bif.netvar_def" #include "event.bif.netvar_def" +#include "logging.bif.netvar_def" void init_event_handlers() { @@ -318,6 +319,7 @@ void init_net_var() { #include "const.bif.netvar_init" #include "types.bif.netvar_init" +#include "logging.bif.netvar_init" conn_id = internal_type("conn_id")->AsRecordType(); endpoint = internal_type("endpoint")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 2de1962f4d..f1164c0e48 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -273,5 +273,6 @@ extern void init_net_var(); #include "const.bif.netvar_h" #include "types.bif.netvar_h" #include "event.bif.netvar_h" +#include "logging.bif.netvar_h" #endif diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index b634ba7eba..82b7dc7dde 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -183,6 +183,7 @@ #include "Sessions.h" #include "File.h" #include "Conn.h" +#include "LogMgr.h" extern "C" { #include "setsignal.h" @@ -190,7 +191,7 @@ extern "C" { // Gets incremented each time there's an incompatible change // to the communication internals. -static const unsigned short PROTOCOL_VERSION = 0x06; +static const unsigned short PROTOCOL_VERSION = 0x07; static const char MSG_NONE = 0x00; static const char MSG_VERSION = 0x01; @@ -216,9 +217,12 @@ static const char MSG_SYNC_POINT = 0x14; static const char MSG_TERMINATE = 0x15; static const char MSG_DEBUG_DUMP = 0x16; static const char MSG_REMOTE_PRINT = 0x17; +static const char MSG_LOG_CREATE_WRITER = 0x18; +static const char MSG_LOG_WRITE = 0x19; +static const char MSG_REQUEST_LOGS = 0x20; // Update this one whenever adding a new ID: -static const char MSG_ID_MAX = MSG_REMOTE_PRINT; +static const char MSG_ID_MAX = MSG_REQUEST_LOGS; static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U; @@ -226,6 +230,9 @@ static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U; static const int PRINT_BUFFER_SIZE = 10 * 1024; static const int SOCKBUF_SIZE = 1024 * 1024; +// Buffer size for remote-log data. +static const int LOG_BUFFER_SIZE = 50 * 1024; + struct ping_args { uint32 seq; double time1; // Round-trip time parent1<->parent2 @@ -303,6 +310,9 @@ static const char* msgToStr(int msg) MSG_STR(MSG_TERMINATE) MSG_STR(MSG_DEBUG_DUMP) MSG_STR(MSG_REMOTE_PRINT) + MSG_STR(MSG_LOG_CREATE_WRITER) + MSG_STR(MSG_LOG_WRITE) + MSG_STR(MSG_REQUEST_LOGS) default: return "UNKNOWN_MSG"; } @@ -469,7 +479,10 @@ static inline bool is_peer_msg(int msg) msg == MSG_CAPS || msg == MSG_COMPRESS || msg == MSG_SYNC_POINT || - msg == MSG_REMOTE_PRINT; + msg == MSG_REMOTE_PRINT || + msg == MSG_LOG_CREATE_WRITER || + msg == MSG_LOG_WRITE || + msg == MSG_REQUEST_LOGS; } bool RemoteSerializer::IsConnectedPeer(PeerID id) @@ -704,6 +717,7 @@ bool RemoteSerializer::CloseConnection(Peer* peer) return true; FlushPrintBuffer(peer); + FlushLogBuffer(peer); Log(LogInfo, "closing connection", peer); @@ -738,6 +752,31 @@ bool RemoteSerializer::RequestSync(PeerID id, bool auth) return true; } +bool RemoteSerializer::RequestLogs(PeerID id) + { + if ( ! using_communication ) + return true; + + Peer* peer = LookupPeer(id, true); + if ( ! peer ) + { + run_time(fmt("unknown peer id %d for request logs", int(id))); + return false; + } + + if ( peer->phase != Peer::HANDSHAKE ) + { + run_time(fmt("can't request logs from peer; wrong phase %d", + peer->phase)); + return false; + } + + if ( ! SendToChild(MSG_REQUEST_LOGS, peer, 0) ) + return false; + + return true; + } + bool RemoteSerializer::RequestEvents(PeerID id, RE_Matcher* pattern) { if ( ! using_communication ) @@ -1443,6 +1482,7 @@ bool RemoteSerializer::Poll(bool may_block) case MSG_PHASE_DONE: case MSG_TERMINATE: case MSG_DEBUG_DUMP: + case MSG_REQUEST_LOGS: { // No further argument chunk. msgstate = TYPE; @@ -1465,6 +1505,8 @@ bool RemoteSerializer::Poll(bool may_block) case MSG_LOG: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: { // One further argument chunk. msgstate = ARGS; @@ -1601,6 +1643,15 @@ bool RemoteSerializer::DoMessage() case MSG_REMOTE_PRINT: return ProcessRemotePrint(); + case MSG_LOG_CREATE_WRITER: + return ProcessLogCreateWriter(); + + case MSG_LOG_WRITE: + return ProcessLogWrite(); + + case MSG_REQUEST_LOGS: + return ProcessRequestLogs(); + default: DEBUG_COMM(fmt("unexpected msg type: %d", int(current_msgtype))); @@ -1663,6 +1714,7 @@ void RemoteSerializer::PeerConnected(Peer* peer) peer->cache_out->Clear(); peer->our_runtime = int(current_time(true) - bro_start_time); peer->sync_point = 0; + peer->logs_requested = false; if ( ! SendCMsgToChild(MSG_VERSION, peer) ) return; @@ -1725,6 +1777,7 @@ RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port, peer->orig = false; peer->accept_state = false; peer->send_state = false; + peer->logs_requested = false; peer->caps = 0; peer->comp_level = 0; peer->suspended_processing = false; @@ -1735,6 +1788,8 @@ RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port, peer->sync_point = 0; peer->print_buffer = 0; peer->print_buffer_used = 0; + peer->log_buffer = 0; + peer->log_buffer_used = 0; peers.append(peer); Log(LogInfo, "added peer", peer); @@ -1767,6 +1822,7 @@ void RemoteSerializer::RemovePeer(Peer* peer) int id = peer->id; Unref(peer->val); delete [] peer->print_buffer; + delete [] peer->log_buffer; delete peer->cache_in; delete peer->cache_out; delete peer; @@ -1960,6 +2016,19 @@ bool RemoteSerializer::ProcessRequestSyncMsg() return true; } +bool RemoteSerializer::ProcessRequestLogs() + { + if ( ! current_peer ) + return false; + + Log(LogInfo, "peer requested logs", current_peer); + + current_peer->logs_requested = true; + current_peer->log_buffer = new char[LOG_BUFFER_SIZE]; + current_peer->log_buffer_used = 0; + return true; + } + bool RemoteSerializer::ProcessPhaseDone() { switch ( current_peer->phase ) { @@ -2024,6 +2093,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer) if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) ) Log(LogInfo, "peer supports keep-in-cache; using that", peer); + if ( peer->logs_requested ) + log_mgr->SendAllWritersTo(peer->id); + if ( peer->sync_requested != Peer::NONE ) { if ( in_sync ) @@ -2377,6 +2449,260 @@ bool RemoteSerializer::ProcessRemotePrint() return true; } +bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) + { + loop_over_list(peers, i) + { + SendLogCreateWriter(peers[i]->id, id, writer, path, num_fields, fields); + } + + return true; + } + +bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields) + { + SetErrorDescr("logging"); + + ChunkedIO::Chunk* c = 0; + + Peer* peer = LookupPeer(peer_id, true); + if ( ! peer ) + return false; + + if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) + return false; + + BinarySerializationFormat fmt; + + fmt.StartWrite(); + + bool success = fmt.Write(id->AsEnum(), "id") && + fmt.Write(writer->AsEnum(), "writer") && + fmt.Write(path, "path") && + fmt.Write(num_fields, "num_fields"); + + if ( ! success ) + goto error; + + for ( int i = 0; i < num_fields; i++ ) + { + if ( ! fields[i]->Write(&fmt) ) + goto error; + } + + c = new ChunkedIO::Chunk; + c->len = fmt.EndWrite(&c->data); + + if ( ! SendToChild(MSG_LOG_CREATE_WRITER, peer, 0) ) + goto error; + + if ( ! SendToChild(c) ) + goto error; + + return true; + +error: + FatalError(io->Error()); + return false; + } + +bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) + { + loop_over_list(peers, i) + { + SendLogWrite(peers[i], id, writer, path, num_fields, vals); + } + + return true; + } + +bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals) + { + if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING ) + return false; + + if ( ! peer->logs_requested ) + return false; + + assert(peer->log_buffer); + + // Serialize the log record entry. + + BinarySerializationFormat fmt; + + fmt.StartWrite(); + + bool success = fmt.Write(id->AsEnum(), "id") && + fmt.Write(writer->AsEnum(), "writer") && + fmt.Write(path, "path") && + fmt.Write(num_fields, "num_fields"); + + if ( ! success ) + goto error; + + for ( int i = 0; i < num_fields; i++ ) + { + if ( ! vals[i]->Write(&fmt) ) + goto error; + } + + // Ok, we have the binary data now. + char* data; + int len; + + len = fmt.EndWrite(&data); + + // Do we have enough space in the buffer? If not, flush first. + if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) ) + FlushLogBuffer(peer); + + // If the data is actually larger than our complete buffer, just send it out. + if ( len > LOG_BUFFER_SIZE ) + return SendToChild(MSG_LOG_WRITE, peer, data, len); + + // Now we have space in the buffer, copy it into there. + memcpy(peer->log_buffer + peer->log_buffer_used, data, len); + peer->log_buffer_used += len; + assert(peer->log_buffer_used <= LOG_BUFFER_SIZE); + + FlushLogBuffer(peer); + return false; + +error: + FatalError(io->Error()); + return false; + } + +bool RemoteSerializer::FlushLogBuffer(Peer* p) + { + if ( p->state == Peer::CLOSING ) + return false; + + if ( ! p->log_buffer ) + return true; + + SendToChild(MSG_LOG_WRITE, p, p->log_buffer, p->log_buffer_used); + + p->log_buffer = new char[LOG_BUFFER_SIZE]; + p->log_buffer_used = 0; + return true; + } + +bool RemoteSerializer::ProcessLogCreateWriter() + { + if ( current_peer->state == Peer::CLOSING ) + return false; + + assert(current_args); + + EnumVal* id_val = 0; + EnumVal* writer_val = 0; + LogField** fields = 0; + + BinarySerializationFormat fmt; + fmt.StartRead(current_args->data, current_args->len); + + int id, writer; + string path; + int num_fields; + + bool success = fmt.Read(&id, "id") && + fmt.Read(&writer, "writer") && + fmt.Read(&path, "path") && + fmt.Read(&num_fields, "num_fields"); + + if ( ! success ) + goto error; + + fields = new LogField* [num_fields]; + + for ( int i = 0; i < num_fields; i++ ) + { + fields[i] = new LogField; + if ( ! fields[i]->Read(&fmt) ) + goto error; + } + + fmt.EndRead(); + + id_val = new EnumVal(id, BifType::Enum::Log::ID); + writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); + + if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) ) + goto error; + + Unref(id_val); + Unref(writer_val); + + return true; + +error: + Unref(id_val); + Unref(writer_val); + + Error("write error for creating writer"); + return false; + } + +bool RemoteSerializer::ProcessLogWrite() + { + if ( current_peer->state == Peer::CLOSING ) + return false; + + assert(current_args); + + BinarySerializationFormat fmt; + fmt.StartRead(current_args->data, current_args->len); + + while ( fmt.BytesRead() != (int)current_args->len ) + { + // Unserialize one entry. + EnumVal* id_val = 0; + EnumVal* writer_val = 0; + LogVal** vals = 0; + + int id, writer; + string path; + int num_fields; + + bool success = fmt.Read(&id, "id") && + fmt.Read(&writer, "writer") && + fmt.Read(&path, "path") && + fmt.Read(&num_fields, "num_fields"); + + if ( ! success ) + goto error; + + vals = new LogVal* [num_fields]; + + for ( int i = 0; i < num_fields; i++ ) + { + vals[i] = new LogVal; + if ( ! vals[i]->Read(&fmt) ) + goto error; + } + + id_val = new EnumVal(id, BifType::Enum::Log::ID); + writer_val = new EnumVal(writer, BifType::Enum::Log::Writer); + + success = log_mgr->Write(id_val, writer_val, path, num_fields, vals); + + Unref(id_val); + Unref(writer_val); + + if ( ! success ) + goto error; + + } + + fmt.EndRead(); + + return true; + +error: + Error("write error for log entry"); + return false; + } void RemoteSerializer::GotEvent(const char* name, double time, EventHandlerPtr event, val_list* args) @@ -3048,6 +3374,7 @@ bool SocketComm::ProcessParentMessage() case MSG_TERMINATE: case MSG_PHASE_DONE: case MSG_DEBUG_DUMP: + case MSG_REQUEST_LOGS: { // No further argument chunk. parent_msgstate = TYPE; @@ -3067,6 +3394,8 @@ bool SocketComm::ProcessParentMessage() case MSG_CAPS: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: { // One further argument chunk. parent_msgstate = ARGS; @@ -3167,18 +3496,6 @@ bool SocketComm::DoParentMessage() return true; } - case MSG_PHASE_DONE: - { - // No argument block follows. - if ( parent_peer && parent_peer->connected ) - { - DEBUG_COMM("child: forwarding with MSG_PHASE_DONE to peer"); - if ( ! SendToPeer(parent_peer, MSG_PHASE_DONE, 0) ) - return false; - } - return true; - } - case MSG_LISTEN: return ProcessListen(); @@ -3206,6 +3523,20 @@ bool SocketComm::DoParentMessage() return ForwardChunkToPeer(); } + case MSG_PHASE_DONE: + case MSG_REQUEST_LOGS: + { + // No argument block follows. + if ( parent_peer && parent_peer->connected ) + { + DEBUG_COMM(fmt("child: forwarding %s to peer", msgToStr(parent_msgtype))); + if ( ! SendToPeer(parent_peer, parent_msgtype, 0) ) + return false; + } + + return true; + } + case MSG_REQUEST_EVENTS: case MSG_REQUEST_SYNC: case MSG_SERIAL: @@ -3214,6 +3545,8 @@ bool SocketComm::DoParentMessage() case MSG_CAPS: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: assert(parent_args); return ForwardChunkToPeer(); @@ -3331,6 +3664,7 @@ bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) switch ( msg->Type() ) { case MSG_PHASE_DONE: + case MSG_REQUEST_LOGS: // No further argument block. DEBUG_COMM("child: forwarding with 0 args to parent"); if ( ! SendToParent(msg->Type(), peer, 0) ) @@ -3387,6 +3721,8 @@ bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer) case MSG_CAPS: case MSG_SYNC_POINT: case MSG_REMOTE_PRINT: + case MSG_LOG_CREATE_WRITER: + case MSG_LOG_WRITE: { // Messages with one further argument block which we simply // forward to our parent. diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 6afec4ec6f..20886544c6 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -16,6 +16,8 @@ // FIXME: Change this to network byte order class IncrementalSendTimer; +class LogField; +class LogVal; // This class handles the communication done in Bro's main loop. class RemoteSerializer : public Serializer, public IOSource { @@ -42,6 +44,9 @@ public: // the peer right after the handshake. bool RequestSync(PeerID peer, bool auth); + // Requests logs from the remote side. + bool RequestLogs(PeerID id); + // Sets flag whether we're accepting state from this peer // (default: yes). bool SetAcceptState(PeerID peer, bool accept); @@ -92,6 +97,15 @@ public: // Broadcast remote print. bool SendPrintHookEvent(BroFile* f, const char* txt); + // Send a request to create a writer on a remote side. + bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + + // Broadcasts a request to create a writer. + bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields); + + // Broadcast a log entry to everybody interested. + bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); + // Synchronzizes time with all connected peers. Returns number of // current sync-point, or -1 on error. uint32 SendSyncPoint(); @@ -205,6 +219,7 @@ protected: bool accept_state; // True if we accept state from peer. bool send_state; // True if we're supposed to initially sent our state. int comp_level; // Compression level. + bool logs_requested; // True if the peer has requested logs. // True if this peer triggered a net_suspend_processing(). bool suspended_processing; @@ -217,6 +232,8 @@ protected: uint32 sync_point; // Highest sync-point received so far char* print_buffer; // Buffer for remote print or null. int print_buffer_used; // Number of bytes used in buffer. + char* log_buffer; // Buffer for remote log or null. + int log_buffer_used; // Number of bytes used in buffer. }; // Shuts down remote serializer. @@ -255,6 +272,9 @@ protected: bool ProcessCapsMsg(); bool ProcessSyncPointMsg(); bool ProcessRemotePrint(); + bool ProcessLogCreateWriter(); + bool ProcessLogWrite(); + bool ProcessRequestLogs(); Peer* AddPeer(uint32 ip, uint16 port, PeerID id = PEER_NONE); Peer* LookupPeer(PeerID id, bool only_if_connected); @@ -282,11 +302,13 @@ protected: bool SendID(SerialInfo* info, Peer* peer, const ID& id); bool SendCapabilities(Peer* peer); bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p); + bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals); void UnregisterHandlers(Peer* peer); void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0); bool EnterPhaseRunning(Peer* peer); bool FlushPrintBuffer(Peer* p); + bool FlushLogBuffer(Peer* p); void ChildDied(); void InternalCommError(const char* msg); diff --git a/src/Scope.cc b/src/Scope.cc index 64cf61080f..c9b12c356c 100644 --- a/src/Scope.cc +++ b/src/Scope.cc @@ -113,9 +113,11 @@ TraversalCode Scope::Traverse(TraversalCallback* cb) const } -ID* lookup_ID(const char* name, const char* curr_module, bool no_global) +ID* lookup_ID(const char* name, const char* curr_module, bool no_global, + bool same_module_only) { string fullname = make_full_var_name(curr_module, name); + string ID_module = extract_module_name(fullname.c_str()); bool need_export = ID_module != GLOBAL_MODULE_NAME && ID_module != curr_module; @@ -134,7 +136,8 @@ ID* lookup_ID(const char* name, const char* curr_module, bool no_global) } } - if ( ! no_global ) + if ( ! no_global && (strcmp(GLOBAL_MODULE_NAME, curr_module) == 0 || + ! same_module_only) ) { string globalname = make_full_var_name(GLOBAL_MODULE_NAME, name); ID* id = global_scope()->Lookup(globalname.c_str()); diff --git a/src/Scope.h b/src/Scope.h index ca86a5ec1f..1ed8d6da42 100644 --- a/src/Scope.h +++ b/src/Scope.h @@ -65,7 +65,7 @@ extern bool in_debug; // If no_global is true, don't search in the default "global" namespace. extern ID* lookup_ID(const char* name, const char* module, - bool no_global = false); + bool no_global = false, bool same_module_only=false); extern ID* install_ID(const char* name, const char* module_name, bool is_global, bool is_export); diff --git a/src/SerialTypes.h b/src/SerialTypes.h index a33b026fae..4f9ab4585b 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -146,6 +146,7 @@ SERIAL_EXPR(TABLE_CONSTRUCTOR_EXPR, 40) SERIAL_EXPR(SET_CONSTRUCTOR_EXPR, 41) SERIAL_EXPR(VECTOR_CONSTRUCTOR_EXPR, 42) SERIAL_EXPR(TABLE_COERCE_EXPR, 43) +SERIAL_EXPR(VECTOR_COERCE_EXPR, 44) #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT) SERIAL_STMT(STMT, 1) diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 55e35eb30e..c181107870 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -44,6 +44,7 @@ void SerializationFormat::StartWrite() output_pos = 0; bytes_written = 0; + bytes_read = 0; } uint32 SerializationFormat::EndWrite(char** data) @@ -64,6 +65,8 @@ bool SerializationFormat::ReadData(void* b, size_t count) memcpy(b, input + input_pos, count); input_pos += count; + bytes_read += count; + return true; } @@ -214,6 +217,20 @@ bool BinarySerializationFormat::Read(char** str, int* len, const char* tag) return true; } +bool BinarySerializationFormat::Read(string* v, const char* tag) + { + char* buffer; + int len; + + if ( ! Read(&buffer, &len, tag) ) + return false; + + *v = string(buffer, len); + + delete [] buffer; + return true; + } + bool BinarySerializationFormat::Write(char v, const char* tag) { DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag); @@ -278,6 +295,11 @@ bool BinarySerializationFormat::Write(const char* s, const char* tag) return Write(s, strlen(s), tag); } +bool BinarySerializationFormat::Write(const string& s, const char* tag) + { + return Write(s.data(), s.size(), tag); + } + bool BinarySerializationFormat::WriteOpenTag(const char* tag) { return true; @@ -362,6 +384,12 @@ bool XMLSerializationFormat::Read(char** str, int* len, const char* tag) return false; } +bool XMLSerializationFormat::Read(string* s, const char* tag) + { + internal_error("no reading of xml"); + return false; + } + bool XMLSerializationFormat::Write(char v, const char* tag) { return WriteElem(tag, "char", &v, 1); @@ -416,6 +444,11 @@ bool XMLSerializationFormat::Write(const char* s, const char* tag) return WriteElem(tag, "string", s, strlen(s)); } +bool XMLSerializationFormat::Write(const string& s, const char* tag) + { + return WriteElem(tag, "string", s.data(), s.size()); + } + bool XMLSerializationFormat::WriteOpenTag(const char* tag) { return WriteData("<", 1) && WriteData(tag, strlen(tag) && WriteData(">", 1)); diff --git a/src/SerializationFormat.h b/src/SerializationFormat.h index 8127343274..b9c7ec1549 100644 --- a/src/SerializationFormat.h +++ b/src/SerializationFormat.h @@ -5,6 +5,10 @@ #ifndef SERIALIZATION_FORMAT #define SERIALIZATION_FORMAT +#include + +using namespace std; + #include "util.h" // Abstract base class. @@ -25,6 +29,10 @@ public: virtual bool Read(char* v, const char* tag) = 0; virtual bool Read(bool* v, const char* tag) = 0; virtual bool Read(double* d, const char* tag) = 0; + virtual bool Read(string* s, const char* tag) = 0; + + // Returns number of raw bytes read since last call to StartRead(). + int BytesRead() const { return bytes_read; } // Passes ownership of string. virtual bool Read(char** str, int* len, const char* tag) = 0; @@ -43,6 +51,7 @@ public: virtual bool Write(double d, const char* tag) = 0; virtual bool Write(const char* s, const char* tag) = 0; virtual bool Write(const char* buf, int len, const char* tag) = 0; + virtual bool Write(const string& s, const char* tag) = 0; virtual bool WriteOpenTag(const char* tag) = 0; virtual bool WriteCloseTag(const char* tag) = 0; @@ -65,6 +74,7 @@ protected: uint32 input_pos; int bytes_written; + int bytes_read; }; class BinarySerializationFormat : public SerializationFormat { @@ -81,6 +91,7 @@ public: virtual bool Read(bool* v, const char* tag); virtual bool Read(double* d, const char* tag); virtual bool Read(char** str, int* len, const char* tag); + virtual bool Read(string* s, const char* tag); virtual bool Write(int v, const char* tag); virtual bool Write(uint16 v, const char* tag); virtual bool Write(uint32 v, const char* tag); @@ -91,6 +102,7 @@ public: virtual bool Write(double d, const char* tag); virtual bool Write(const char* s, const char* tag); virtual bool Write(const char* buf, int len, const char* tag); + virtual bool Write(const string& s, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -112,6 +124,7 @@ public: virtual bool Write(double d, const char* tag); virtual bool Write(const char* s, const char* tag); virtual bool Write(const char* buf, int len, const char* tag); + virtual bool Write(const string& s, const char* tag); virtual bool WriteOpenTag(const char* tag); virtual bool WriteCloseTag(const char* tag); virtual bool WriteSeparator(); @@ -126,6 +139,7 @@ public: virtual bool Read(bool* v, const char* tag); virtual bool Read(double* d, const char* tag); virtual bool Read(char** str, int* len, const char* tag); + virtual bool Read(string* s, const char* tag); private: // Encodes non-printable characters. diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 17f702f07e..74481a155a 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -935,7 +935,7 @@ void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) attr_list* a = new attr_list; Attr* attr = new Attr(ATTR_TRACKED); a->append(attr); - id->SetAttrs(new Attributes(a, id->Type())); + id->SetAttrs(new Attributes(a, id->Type(), false)); Unref(attr); } diff --git a/src/Type.cc b/src/Type.cc index bdb56b6aaa..458a672d41 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -33,6 +33,7 @@ const char* type_name(TypeTag t) "func", "file", "vector", + "type", "error", }; @@ -102,6 +103,7 @@ BroType::BroType(TypeTag t, bool arg_base_type) case TYPE_FUNC: case TYPE_FILE: case TYPE_VECTOR: + case TYPE_TYPE: internal_tag = TYPE_INTERNAL_OTHER; break; @@ -781,10 +783,10 @@ bool FuncType::DoUnserialize(UnserialInfo* info) return UNSERIALIZE(&is_event); } -TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs) +TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_record) { type = t; - attrs = arg_attrs ? new Attributes(arg_attrs, t) : 0; + attrs = arg_attrs ? new Attributes(arg_attrs, t, in_record) : 0; id = i; } @@ -902,6 +904,8 @@ RecordType::RecordType(TypeList* arg_base, type_decl_list* refinements) void RecordType::Init(TypeList* arg_base) { + assert(false); // Is this ever used? + base = arg_base; if ( ! base ) @@ -1066,6 +1070,46 @@ void RecordType::DescribeReST(ODesc* d) const DescribeFieldsReST(d, false); } +const char* RecordType::AddFields(type_decl_list* others, attr_list* attr) + { + assert(types); + + bool log = false; + + if ( attr ) + { + loop_over_list(*attr, j) + { + if ( (*attr)[j]->Tag() == ATTR_LOG ) + log = true; + } + } + + loop_over_list(*others, i) + { + TypeDecl* td = (*others)[i]; + + if ( ! td->FindAttr(ATTR_DEFAULT) && + ! td->FindAttr(ATTR_OPTIONAL) ) + return "extension field must be &optional or have &default"; + + if ( log ) + { + if ( ! td->attrs ) + td->attrs = new Attributes(new attr_list, td->type, true); + + td->attrs->AddAttr(new Attr(ATTR_LOG)); + } + + types->append(td); + } + + delete others; + + num_fields = types->length(); + return 0; + } + void RecordType::DescribeFields(ODesc* d) const { if ( d->IsReadable() ) @@ -1518,6 +1562,11 @@ int VectorType::MatchesIndex(ListExpr*& index) const MATCHES_INDEX_SCALAR : DOES_NOT_MATCH_INDEX; } +bool VectorType::IsUnspecifiedVector() const + { + return yield_type->Tag() == TYPE_ANY; + } + IMPLEMENT_SERIAL(VectorType, SER_VECTOR_TYPE); bool VectorType::DoSerialize(SerialInfo* info) const @@ -1591,7 +1640,9 @@ static int is_init_compat(const BroType* t1, const BroType* t2) int same_type(const BroType* t1, const BroType* t2, int is_init) { - if ( t1 == t2 ) + if ( t1 == t2 || + t1->Tag() == TYPE_ANY || + t2->Tag() == TYPE_ANY ) return 1; t1 = flatten_type(t1); @@ -1718,12 +1769,26 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) case TYPE_FILE: return same_type(t1->YieldType(), t2->YieldType(), is_init); + case TYPE_TYPE: + return same_type(t1, t2, is_init); + case TYPE_UNION: error("union type in same_type()"); } return 0; } +int same_attrs(const Attributes* a1, const Attributes* a2) + { + if ( ! a1 ) + return (a2 == 0); + + if ( ! a2 ) + return (a1 == 0); + + return (*a1 == *a2); + } + int record_promotion_compatible(const RecordType* /* super_rec */, const RecordType* /* sub_rec */) { @@ -1795,6 +1860,7 @@ int is_assignable(BroType* t) case TYPE_VECTOR: case TYPE_FILE: case TYPE_TABLE: + case TYPE_TYPE: return 1; case TYPE_VOID: @@ -1862,6 +1928,7 @@ BroType* merge_types(const BroType* t1, const BroType* t2) case TYPE_ADDR: case TYPE_NET: case TYPE_SUBNET: + case TYPE_BOOL: case TYPE_ANY: case TYPE_ERROR: return base_type(tg1); diff --git a/src/Type.h b/src/Type.h index 9827e7f5e6..403ccffa89 100644 --- a/src/Type.h +++ b/src/Type.h @@ -32,6 +32,7 @@ typedef enum { TYPE_FUNC, TYPE_FILE, TYPE_VECTOR, + TYPE_TYPE, TYPE_ERROR #define NUM_TYPES (int(TYPE_ERROR) + 1) } TypeTag; @@ -60,6 +61,7 @@ class ListExpr; class EnumType; class Serializer; class VectorType; +class TypeType; const int DOES_NOT_MATCH_INDEX = 0; const int MATCHES_INDEX_SCALAR = 1; @@ -153,6 +155,7 @@ public: CHECK_TYPE_TAG(TYPE_SUBNET, "BroType::AsSubNetType"); return (const SubNetType*) this; } + SubNetType* AsSubNetType() { CHECK_TYPE_TAG(TYPE_SUBNET, "BroType::AsSubNetType"); @@ -194,6 +197,18 @@ public: return (VectorType*) this; } + const TypeType* AsTypeType() const + { + CHECK_TYPE_TAG(TYPE_TYPE, "BroType::AsTypeType"); + return (TypeType*) this; + } + + TypeType* AsTypeType() + { + CHECK_TYPE_TAG(TYPE_TYPE, "BroType::AsTypeType"); + return (TypeType*) this; + } + int IsSet() const { return tag == TYPE_TABLE && (YieldType() == 0); @@ -371,9 +386,22 @@ protected: ID* return_value; }; +class TypeType : public BroType { +public: + TypeType(BroType* t) : BroType(TYPE_TYPE) { type = t->Ref(); } + ~TypeType() { Unref(type); } + + BroType* Type() { return type; } + +protected: + TypeType() {} + + BroType* type; +}; + class TypeDecl { public: - TypeDecl(BroType* t, const char* i, attr_list* attrs = 0); + TypeDecl(BroType* t, const char* i, attr_list* attrs = 0, bool in_record = false); virtual ~TypeDecl(); const Attr* FindAttr(attr_tag a) const @@ -434,6 +462,10 @@ public: int NumFields() const { return num_fields; } + // Returns 0 if all is ok, otherwise a pointer to an error message. + // Takes ownership of list. + const char* AddFields(type_decl_list* types, attr_list* attr); + void Describe(ODesc* d) const; void DescribeReST(ODesc* d) const; void DescribeFields(ODesc* d) const; @@ -542,6 +574,10 @@ public: int MatchesIndex(ListExpr*& index) const; + // Returns true if this table type is "unspecified", which is what one + // gets using an empty "vector()" constructor. + bool IsUnspecifiedVector() const; + protected: VectorType() { yield_type = 0; } @@ -564,6 +600,9 @@ inline BroType* error_type() { return base_type(TYPE_ERROR); } // test is done in the context of an initialization. extern int same_type(const BroType* t1, const BroType* t2, int is_init=0); +// True if the two attribute lists are equivalent. +extern int same_attrs(const Attributes* a1, const Attributes* a2); + // Returns true if the record sub_rec can be promoted to the record // super_rec. extern int record_promotion_compatible(const RecordType* super_rec, diff --git a/src/Val.cc b/src/Val.cc index 060aaf17ec..a313e810cc 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1753,7 +1753,7 @@ void TableVal::CheckExpireAttr(attr_tag at) a->AttrExpr()->Error("value of timeout not fixed"); return; } - + expire_time = timeout->AsInterval(); if ( timer ) @@ -2042,7 +2042,23 @@ Val* TableVal::Default(Val* index) return 0; if ( ! def_val ) - def_val = def_attr->AttrExpr()->Eval(0); + { + BroType* ytype = Type()->YieldType(); + BroType* dtype = def_attr->AttrExpr()->Type(); + + if ( dtype->Tag() == TYPE_RECORD && ytype->Tag() == TYPE_RECORD && + ! same_type(dtype, ytype) && + record_promotion_compatible(dtype->AsRecordType(), + ytype->AsRecordType()) ) + { + Expr* coerce = new RecordCoerceExpr(def_attr->AttrExpr(), ytype->AsRecordType()); + def_val = coerce->Eval(0); + Unref(coerce); + } + + else + def_val = def_attr->AttrExpr()->Eval(0); + } if ( ! def_val ) { @@ -2912,6 +2928,83 @@ Val* RecordVal::Lookup(int field) const return (*AsRecord())[field]; } +Val* RecordVal::LookupWithDefault(int field) const + { + Val* val = (*AsRecord())[field]; + + if ( val ) + return val->Ref(); + + // Check for &default. + const Attr* def_attr = + record_type->FieldDecl(field)->attrs->FindAttr(ATTR_DEFAULT); + + return def_attr ? def_attr->AttrExpr()->Eval(0) : 0; + } + +RecordVal* RecordVal::CoerceTo(const RecordType* t, Val* aggr) const + { + if ( ! record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) ) + return 0; + + if ( ! aggr ) + aggr = new RecordVal(const_cast(t->AsRecordType())); + + RecordVal* ar = aggr->AsRecordVal(); + RecordType* ar_t = aggr->Type()->AsRecordType(); + + const RecordType* rv_t = Type()->AsRecordType(); + + int i; + for ( i = 0; i < rv_t->NumFields(); ++i ) + { + int t_i = ar_t->FieldOffset(rv_t->FieldName(i)); + + if ( t_i < 0 ) + { + char buf[512]; + safe_snprintf(buf, sizeof(buf), + "orphan field \"%s\" in initialization", + rv_t->FieldName(i)); + Error(buf); + break; + } + + if ( ar_t->FieldType(t_i)->Tag() == TYPE_RECORD + && ! same_type(ar_t->FieldType(t_i), Lookup(i)->Type()) ) + { + Expr* rhs = new ConstExpr(Lookup(i)->Ref()); + Expr* e = new RecordCoerceExpr(rhs, ar_t->FieldType(t_i)->AsRecordType()); + ar->Assign(t_i, e->Eval(0)); + } + + ar->Assign(t_i, Lookup(i)->Ref()); + } + + for ( i = 0; i < ar_t->NumFields(); ++i ) + if ( ! ar->Lookup(i) && + ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) ) + { + char buf[512]; + safe_snprintf(buf, sizeof(buf), + "non-optional field \"%s\" missing in initialization", ar_t->FieldName(i)); + Error(buf); + } + + return ar; + } + +RecordVal* RecordVal::CoerceTo(RecordType* t) + { + if ( same_type(Type(), t) ) + { + this->Ref(); + return this; + } + + return CoerceTo(t, 0); + } + void RecordVal::Describe(ODesc* d) const { const val_list* vl = AsRecord(); @@ -3340,6 +3433,10 @@ Val* check_and_promote(Val* v, const BroType* t, int is_init) TypeTag t_tag = t->Tag(); TypeTag v_tag = vt->Tag(); + // More thought definitely needs to go into this. + if ( t_tag == TYPE_ANY || v_tag == TYPE_ANY ) + return v; + if ( ! EitherArithmetic(t_tag, v_tag) || /* allow sets as initializers */ (is_init && v_tag == TYPE_TABLE) ) diff --git a/src/Val.h b/src/Val.h index 0225fb6725..2dede2364e 100644 --- a/src/Val.h +++ b/src/Val.h @@ -39,6 +39,7 @@ class TableVal; class RecordVal; class ListVal; class StringVal; +class EnumVal; class MutableVal; class StateAccess; @@ -143,6 +144,15 @@ public: // class has ref'd it. Val(BroFile* f); + Val(BroType* t, bool type_type) // Extra arg to differentiate from protected version. + { + type = new TypeType(t->Ref()); + attribs = 0; +#ifdef DEBUG + bound_id = 0; +#endif + } + Val() { val.int_val = 0; @@ -225,6 +235,12 @@ public: return &val.subnet_val; } + BroType* AsType() const + { + CHECK_TAG(type->Tag(), TYPE_TYPE, "Val::Type", type_name) + return type; + } + // ... in network byte order const addr_type AsAddr() const { @@ -275,6 +291,7 @@ public: CONVERTER(TYPE_LIST, ListVal*, AsListVal) CONVERTER(TYPE_STRING, StringVal*, AsStringVal) CONVERTER(TYPE_VECTOR, VectorVal*, AsVectorVal) + CONVERTER(TYPE_ENUM, EnumVal*, AsEnumVal) #define CONST_CONVERTER(tag, ctype, name) \ const ctype name() const \ @@ -894,7 +911,8 @@ public: { return new Val(record_type->NumFields(), TYPE_COUNT); } void Assign(int field, Val* new_val, Opcode op = OP_ASSIGN); - Val* Lookup(int field) const; + Val* Lookup(int field) const; // Does not Ref() value. + Val* LookupWithDefault(int field) const; // Does Ref() value. void Describe(ODesc* d) const; @@ -903,6 +921,17 @@ public: void SetOrigin(BroObj* o) { origin = o; } BroObj* GetOrigin() const { return origin; } + // Returns a new value representing the value coerced to the given + // type. If coercion is not possible, returns 0. The non-const + // version may return the current value ref'ed if its type matches + // directly. + // + // *aggr* is optional; if non-zero, we add to it. See + // Expr::InitVal(). We leave it out in the non-const version to make + // the choice unambigious. + RecordVal* CoerceTo(const RecordType* other, Val* aggr) const; + RecordVal* CoerceTo(RecordType* other); + unsigned int MemoryAllocation() const; void DescribeReST(ODesc* d) const; diff --git a/src/Var.cc b/src/Var.cc index 2b94c45461..1211a5315b 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -109,7 +109,7 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, id->SetType(t); if ( attr ) - id->AddAttrs(new Attributes(attr, t)); + id->AddAttrs(new Attributes(attr, t, false)); if ( id->FindAttr(ATTR_PERSISTENT) || id->FindAttr(ATTR_SYNCHRONIZED) ) { @@ -172,6 +172,16 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init, } id->UpdateValAttrs(); + + if ( t && t->Tag() == TYPE_FUNC && t->AsFuncType()->IsEvent() ) + { + // For events, add a function value (without any body) here so that + // we can later access the ID even if no implementations have been + // defined. + Func* f = new BroFunc(id, 0, 0, 0); + id->SetVal(new Val(f)); + id->SetConst(); + } } @@ -256,7 +266,7 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */) id->MakeType(); if ( attr ) - id->SetAttrs(new Attributes(attr, tnew)); + id->SetAttrs(new Attributes(attr, tnew, false)); } void begin_func(ID* id, const char* module_name, function_flavor flavor, diff --git a/src/bro.bif b/src/bro.bif index ebd4df0aca..a05e1d1de7 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -360,6 +360,26 @@ function cat%(...%): string return new StringVal(s); %} +function record_type_to_vector%(rt: string%): string_vec + %{ + VectorVal* result = + new VectorVal(internal_type("string_vec")->AsVectorType()); + + RecordType *type = internal_type(rt->CheckString())->AsRecordType(); + + if ( type ) + { + for ( int i = 0; i < type->NumFields(); ++i ) + { + StringVal* val = new StringVal(type->FieldName(i)); + result->Assign(i+1, val, 0); + } + } + + return result; + %} + + function cat_sep%(sep: string, def: string, ...%): string %{ ODesc d; @@ -2086,6 +2106,12 @@ function request_remote_sync%(p: event_peer, auth: bool%) : bool return new Val(remote_serializer->RequestSync(id, auth), TYPE_BOOL); %} +function request_remote_logs%(p: event_peer%) : bool + %{ + RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); + return new Val(remote_serializer->RequestLogs(id), TYPE_BOOL); + %} + function set_accept_state%(p: event_peer, accept: bool%) : bool %{ RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount(); @@ -3019,7 +3045,7 @@ function lookup_location%(a: addr%) : geo_location #ifdef GEOIP_COUNTRY_EDITION_V6 if ( geoip_v6 && ! is_v4_addr(a) ) gir = GeoIP_record_by_ipnum_v6(geoip_v6, geoipv6_t(a)); - else + else #endif if ( geoip && is_v4_addr(a) ) { @@ -3107,7 +3133,7 @@ function lookup_asn%(a: addr%) : count gir = GeoIP_name_by_ipnum_v6(geoip_asn, geoipv6_t(a)); else #endif - if ( is_v4_addr(a) ) + if ( is_v4_addr(a) ) { uint32 addr = to_v4_addr(a); gir = GeoIP_name_by_ipnum(geoip_asn, ntohl(addr)); @@ -3119,7 +3145,7 @@ function lookup_asn%(a: addr%) : count if ( gir ) { - // Move the pointer +2 so we don't return + // Move the pointer +2 so we don't return // the first two characters: "AS". return new Val(atoi(gir+2), TYPE_COUNT); } diff --git a/src/logging.bif b/src/logging.bif new file mode 100644 index 0000000000..0e48633d5a --- /dev/null +++ b/src/logging.bif @@ -0,0 +1,76 @@ +# Internal functions and types used by the logging framework. + +module Log; + +%%{ +#include "LogMgr.h" +#include "NetVar.h" +%%} + +type Filter: record; +type Stream: record; +type RotationInfo: record; +type RotationControl: record; + +const Log::rotation_control: RotationControl; + +function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool + %{ + bool result = log_mgr->CreateStream(id->AsEnumVal(), stream->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + +function Log::__enable_stream%(id: Log::ID%) : bool + %{ + bool result = log_mgr->EnableStream(id->AsEnumVal()); + return new Val(result, TYPE_BOOL); + %} + +function Log::__disable_stream%(id: Log::ID%) : bool + %{ + bool result = log_mgr->DisableStream(id->AsEnumVal()); + return new Val(result, TYPE_BOOL); + %} + +function Log::__add_filter%(id: Log::ID, filter: Log::Filter%) : bool + %{ + bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + +function Log::__remove_filter%(id: Log::ID, name: string%) : bool + %{ + bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name); + return new Val(result, TYPE_BOOL); + %} + +function Log::__write%(id: Log::ID, columns: any%) : bool + %{ + bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal()); + return new Val(result, TYPE_BOOL); + %} + +function Log::__set_buf%(id: Log::ID, buffered: bool%): bool + %{ + bool result = log_mgr->SetBuf(id->AsEnumVal(), buffered); + return new Val(result, TYPE_BOOL); + %} + +function Log::__flush%(id: Log::ID%): bool + %{ + bool result = log_mgr->Flush(id->AsEnumVal()); + return new Val(result, TYPE_BOOL); + %} + +# Options for the ASCII writer. + +module LogAscii; + +const output_to_stdout: bool; +const include_header: bool; +const header_prefix: string; +const separator: string; +const set_separator: string; +const empty_field: string; +const unset_field: string; + diff --git a/src/main.cc b/src/main.cc index 7fc8e1b39d..a5c33040db 100644 --- a/src/main.cc +++ b/src/main.cc @@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "Event.h" #include "File.h" #include "Logger.h" +#include "LogMgr.h" #include "Net.h" #include "NetVar.h" #include "Var.h" @@ -71,6 +72,7 @@ name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; Logger* bro_logger; +LogMgr* log_mgr; Func* alarm_hook = 0; Stmt* stmts; EventHandlerPtr bro_signal = 0; @@ -288,6 +290,7 @@ void terminate_bro() delete conn_compressor; delete remote_serializer; delete dpm; + delete log_mgr; } void termination_signal() @@ -702,7 +705,8 @@ int main(int argc, char** argv) persistence_serializer = new PersistenceSerializer(); remote_serializer = new RemoteSerializer(); - event_registry = new EventRegistry; + event_registry = new EventRegistry(); + log_mgr = new LogMgr(); if ( events_file ) event_player = new EventPlayer(events_file); diff --git a/src/parse.y b/src/parse.y index c1f8f7ee6b..8e8a75cad7 100644 --- a/src/parse.y +++ b/src/parse.y @@ -3,7 +3,7 @@ // See the file "COPYING" in the main distribution directory for copyright. %} -%expect 81 +%expect 85 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF @@ -24,7 +24,7 @@ %token TOK_ATTR_EXPIRE_CREATE TOK_ATTR_EXPIRE_READ TOK_ATTR_EXPIRE_WRITE %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE -%token TOK_ATTR_PRIORITY TOK_ATTR_GROUP +%token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG %token TOK_DEBUG @@ -45,7 +45,7 @@ %type TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC %type opt_doc_list opt_post_doc_list -%type local_id global_id event_id global_or_event_id resolve_id begin_func +%type local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func %type local_id_list %type init_class %type opt_init @@ -112,8 +112,10 @@ extern Expr* g_curr_debug_expr; Expr* bro_this = 0; int in_init = 0; +int in_record = 0; bool in_debug = false; bool resolving_global_ID = false; +bool defining_global_ID = false; ID* func_id = 0; EnumType *cur_enum_type = 0; @@ -464,6 +466,12 @@ expr: $$ = $2; } + | '[' ']' + { + // We interpret this as an empty record constructor. + $$ = new RecordConstructorExpr(new ListExpr); + } + | TOK_RECORD '(' expr_list ')' { @@ -805,7 +813,11 @@ type: $$ = new SetType($3, 0); } - | TOK_RECORD '{' { do_doc_token_start(); } type_decl_list '}' + | TOK_RECORD '{' + { ++in_record; do_doc_token_start(); } + type_decl_list + { --in_record; } + '}' { do_doc_token_stop(); set_location(@1, @5); @@ -938,7 +950,7 @@ type_decl: $4, $2, a_copy, concat_opt_docs($1, $7)); } - $$ = new TypeDecl($4, $2, $5); + $$ = new TypeDecl($4, $2, $5, (in_record > 0)); } ; @@ -980,7 +992,7 @@ decl: | TOK_EXPORT '{' { is_export = true; } decl_list '}' { is_export = false; } - | TOK_GLOBAL global_id opt_type init_class opt_init opt_attr ';' + | TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_REGULAR); @@ -1005,7 +1017,7 @@ decl: } } - | TOK_CONST global_id opt_type init_class opt_init opt_attr ';' + | TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';' { add_global($2, $3, $4, $5, $6, VAR_CONST); @@ -1054,6 +1066,25 @@ decl: } } + | TOK_REDEF TOK_RECORD global_id TOK_ADD_TO + '{' type_decl_list '}' opt_attr ';' + { + if ( ! $3->Type() ) + $3->Error("unknown identifier"); + else + { + RecordType* add_to = $3->Type()->AsRecordType(); + if ( ! add_to ) + $3->Error("not a record type"); + else + { + const char* error = add_to->AddFields($6, $8); + if ( error ) + $3->Error(error); + } + } + } + | TOK_TYPE global_id ':' refined_type opt_attr ';' { add_type($2, $4, $5, 0); @@ -1113,7 +1144,7 @@ conditional: ; func_hdr: - TOK_FUNCTION global_id func_params + TOK_FUNCTION def_global_id func_params { begin_func($2, current_module.c_str(), FUNC_FLAVOR_FUNCTION, 0, $3); @@ -1263,6 +1294,8 @@ attr: { $$ = new Attr(ATTR_PRIORITY, $3); } | TOK_ATTR_GROUP '=' expr { $$ = new Attr(ATTR_GROUP, $3); } + | TOK_ATTR_LOG + { $$ = new Attr(ATTR_LOG); } ; stmt: @@ -1501,6 +1534,11 @@ global_id: { $$ = $2; } ; +def_global_id: + { defining_global_ID = 1; } global_id { defining_global_ID = 0; } + { $$ = $2; } + ; + event_id: { resolving_global_ID = 0; } global_or_event_id { $$ = $2; } @@ -1511,7 +1549,7 @@ global_or_event_id: { set_location(@1); - $$ = lookup_ID($1, current_module.c_str(), false); + $$ = lookup_ID($1, current_module.c_str(), false, defining_global_ID); if ( $$ ) { if ( ! $$->IsGlobal() ) diff --git a/src/scan.l b/src/scan.l index 5549e9543f..b56583ab44 100644 --- a/src/scan.l +++ b/src/scan.l @@ -296,6 +296,7 @@ when return TOK_WHEN; &encrypt return TOK_ATTR_ENCRYPT; &expire_func return TOK_ATTR_EXPIRE_FUNC; &group return TOK_ATTR_GROUP; +&log return TOK_ATTR_LOG; &mergeable return TOK_ATTR_MERGEABLE; &optional return TOK_ATTR_OPTIONAL; &persistent return TOK_ATTR_PERSISTENT; diff --git a/src/strings.bif b/src/strings.bif index e41ae196b2..764260676b 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -135,6 +135,26 @@ function sort_string_array%(a: string_array%): string_array vs_to_string_array(vs, b, 1, n); return b; %} + +function join_string_vec%(vec: string_vec, sep: string%): string + %{ + ODesc d; + VectorVal *v = vec->AsVectorVal(); + + for ( unsigned i = 0; i < v->Size(); ++i ) + { + if ( i > 0 ) + d.Add(sep->CheckString(), 0); + + v->Lookup(i+1)->Describe(&d); + } + + BroString* s = new BroString(1, d.TakeBytes(), d.Len()); + s->SetUseFreeToDelete(true); + + return new StringVal(s); + %} + function edit%(arg_s: string, arg_edit_char: string%): string %{ diff --git a/src/types.bif b/src/types.bif index 7b60192155..4496e444a1 100644 --- a/src/types.bif +++ b/src/types.bif @@ -50,3 +50,14 @@ enum rpc_status %{ RPC_AUTH_ERROR, RPC_UNKNOWN_ERROR, %} + +module Log; + +enum Writer %{ + WRITER_DEFAULT, + WRITER_ASCII, +%} + +enum ID %{ + Unknown, +%} diff --git a/testing/btest/Baseline/language.rec-nested-opt/output b/testing/btest/Baseline/language.rec-nested-opt/output new file mode 100644 index 0000000000..e41358b71d --- /dev/null +++ b/testing/btest/Baseline/language.rec-nested-opt/output @@ -0,0 +1,3 @@ +{ +[Wget/1.9+cvs-stable (Red Hat modified)] = [name=Wget, version=[major=1, minor=9, addl=+cvs], host=0.0.0.0, ts=0.0] +} diff --git a/testing/btest/Baseline/language.rec-table-default/output b/testing/btest/Baseline/language.rec-table-default/output new file mode 100644 index 0000000000..e810588b34 --- /dev/null +++ b/testing/btest/Baseline/language.rec-table-default/output @@ -0,0 +1,14 @@ +{ +[foo] = T +} +{ + +} +{ +B, +A, +C +} +{ + +} diff --git a/testing/btest/Baseline/language.record-extension/output b/testing/btest/Baseline/language.record-extension/output new file mode 100644 index 0000000000..1e38084e0c --- /dev/null +++ b/testing/btest/Baseline/language.record-extension/output @@ -0,0 +1,2 @@ +[a=21, b=, c=42, d=] +[a=21, b=, c=42, d=XXX] diff --git a/testing/btest/Baseline/language.record-ref-assign/output b/testing/btest/Baseline/language.record-ref-assign/output new file mode 100644 index 0000000000..4caf2f8fa8 --- /dev/null +++ b/testing/btest/Baseline/language.record-ref-assign/output @@ -0,0 +1 @@ +XXX, XXX diff --git a/testing/btest/Baseline/language.vector-coerce-expr/output b/testing/btest/Baseline/language.vector-coerce-expr/output new file mode 100644 index 0000000000..baabf9d89d --- /dev/null +++ b/testing/btest/Baseline/language.vector-coerce-expr/output @@ -0,0 +1,4 @@ +[] +[1, 2, 3] +[T, F, T] +[] diff --git a/testing/btest/Baseline/language.wrong-record-extension/output b/testing/btest/Baseline/language.wrong-record-extension/output new file mode 100644 index 0000000000..144a065f59 --- /dev/null +++ b/testing/btest/Baseline/language.wrong-record-extension/output @@ -0,0 +1 @@ + error, extension field must be &optional or have &default diff --git a/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log b/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log new file mode 100644 index 0000000000..469f2d1991 --- /dev/null +++ b/testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log @@ -0,0 +1,3 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.40319 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718503.40319 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/logging.ascii-empty/output b/testing/btest/Baseline/logging.ascii-empty/output new file mode 100644 index 0000000000..d377ca15d7 --- /dev/null +++ b/testing/btest/Baseline/logging.ascii-empty/output @@ -0,0 +1,6 @@ +PREFIX<>t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET +1299718506.56593|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T diff --git a/testing/btest/Baseline/logging.ascii-escape/ssh.log b/testing/btest/Baseline/logging.ascii-escape/ssh.log new file mode 100644 index 0000000000..6797e1e302 Binary files /dev/null and b/testing/btest/Baseline/logging.ascii-escape/ssh.log differ diff --git a/testing/btest/Baseline/logging.ascii-options/output b/testing/btest/Baseline/logging.ascii-options/output new file mode 100644 index 0000000000..33a922cc2b --- /dev/null +++ b/testing/btest/Baseline/logging.ascii-options/output @@ -0,0 +1,5 @@ +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|unknown +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|US +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|UK +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|BR +1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|MX diff --git a/testing/btest/Baseline/logging.attr-extend/ssh.log b/testing/btest/Baseline/logging.attr-extend/ssh.log new file mode 100644 index 0000000000..d543af3a43 --- /dev/null +++ b/testing/btest/Baseline/logging.attr-extend/ssh.log @@ -0,0 +1,2 @@ +# status country a1 b1 b2 +success unknown 1 3 4 diff --git a/testing/btest/Baseline/logging.attr/ssh.log b/testing/btest/Baseline/logging.attr/ssh.log new file mode 100644 index 0000000000..c4355d2fd5 --- /dev/null +++ b/testing/btest/Baseline/logging.attr/ssh.log @@ -0,0 +1,6 @@ +# status country +success unknown +failure US +failure UK +success BR +failure MX diff --git a/testing/btest/Baseline/logging.empty-event/ssh.log b/testing/btest/Baseline/logging.empty-event/ssh.log new file mode 100644 index 0000000000..7f21430ea7 --- /dev/null +++ b/testing/btest/Baseline/logging.empty-event/ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 success unknown +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 failure US +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 failure UK +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 success BR +1299809561.67372 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.events/output b/testing/btest/Baseline/logging.events/output new file mode 100644 index 0000000000..c3dbf607a6 --- /dev/null +++ b/testing/btest/Baseline/logging.events/output @@ -0,0 +1,2 @@ +[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=] +[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US] diff --git a/testing/btest/Baseline/logging.exclude/ssh.log b/testing/btest/Baseline/logging.exclude/ssh.log new file mode 100644 index 0000000000..4defa5ced1 --- /dev/null +++ b/testing/btest/Baseline/logging.exclude/ssh.log @@ -0,0 +1,6 @@ +# id.orig_p id.resp_h id.resp_p status country +1234 2.3.4.5 80 success unknown +1234 2.3.4.5 80 failure US +1234 2.3.4.5 80 failure UK +1234 2.3.4.5 80 success BR +1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.file/ssh.log b/testing/btest/Baseline/logging.file/ssh.log new file mode 100644 index 0000000000..49115ab1df --- /dev/null +++ b/testing/btest/Baseline/logging.file/ssh.log @@ -0,0 +1,2 @@ +# t f +1303098703.62603 Foo.log diff --git a/testing/btest/Baseline/logging.include/ssh.log b/testing/btest/Baseline/logging.include/ssh.log new file mode 100644 index 0000000000..881704257e --- /dev/null +++ b/testing/btest/Baseline/logging.include/ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 +1303064007.48299 1.2.3.4 diff --git a/testing/btest/Baseline/logging.path-func/output b/testing/btest/Baseline/logging.path-func/output new file mode 100644 index 0000000000..25e4ca6696 --- /dev/null +++ b/testing/btest/Baseline/logging.path-func/output @@ -0,0 +1,13 @@ +static-prefix-0.log +static-prefix-1.log +static-prefix-2.log +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 success BR +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX3 +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.05867 1.2.3.4 1234 2.3.4.5 80 failure MX2 diff --git a/testing/btest/Baseline/logging.pred/ssh.failure.log b/testing/btest/Baseline/logging.pred/ssh.failure.log new file mode 100644 index 0000000000..c46990dc65 --- /dev/null +++ b/testing/btest/Baseline/logging.pred/ssh.failure.log @@ -0,0 +1,2 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.16177 1.2.3.4 1234 2.3.4.5 80 failure US diff --git a/testing/btest/Baseline/logging.pred/ssh.success.log b/testing/btest/Baseline/logging.pred/ssh.success.log new file mode 100644 index 0000000000..c6adcd86aa --- /dev/null +++ b/testing/btest/Baseline/logging.pred/ssh.success.log @@ -0,0 +1,2 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.16177 1.2.3.4 1234 2.3.4.5 80 success - diff --git a/testing/btest/Baseline/logging.remote-types/receiver.ssh.log b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log new file mode 100644 index 0000000000..a7818428a7 --- /dev/null +++ b/testing/btest/Baseline/logging.remote-types/receiver.ssh.log @@ -0,0 +1,2 @@ +# b i e c p sn n a d t iv s sc ss se vc ve +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1301360085.98852 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.failure.log b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log new file mode 100644 index 0000000000..815b0366b3 --- /dev/null +++ b/testing/btest/Baseline/logging.remote/sender.ssh.failure.log @@ -0,0 +1,4 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.log b/testing/btest/Baseline/logging.remote/sender.ssh.log new file mode 100644 index 0000000000..2f159ad6a7 --- /dev/null +++ b/testing/btest/Baseline/logging.remote/sender.ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success - +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success BR +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.remote/sender.ssh.success.log b/testing/btest/Baseline/logging.remote/sender.ssh.success.log new file mode 100644 index 0000000000..2838cc68b8 --- /dev/null +++ b/testing/btest/Baseline/logging.remote/sender.ssh.success.log @@ -0,0 +1,3 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success - +1299718503.72819 1.2.3.4 1234 2.3.4.5 80 success BR diff --git a/testing/btest/Baseline/logging.remove/ssh.failure.log b/testing/btest/Baseline/logging.remove/ssh.failure.log new file mode 100644 index 0000000000..ddbacda28e --- /dev/null +++ b/testing/btest/Baseline/logging.remove/ssh.failure.log @@ -0,0 +1,3 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure UK diff --git a/testing/btest/Baseline/logging.remove/ssh.log b/testing/btest/Baseline/logging.remove/ssh.log new file mode 100644 index 0000000000..123e8e3a87 --- /dev/null +++ b/testing/btest/Baseline/logging.remove/ssh.log @@ -0,0 +1,4 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure US +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718503.28253 1.2.3.4 1234 2.3.4.5 80 failure BR diff --git a/testing/btest/Baseline/logging.rotate-custom/out b/testing/btest/Baseline/logging.rotate-custom/out new file mode 100644 index 0000000000..e270b6a773 --- /dev/null +++ b/testing/btest/Baseline/logging.rotate-custom/out @@ -0,0 +1,134 @@ +2nd test2-11-03-06_19.00.05.log test2.log 11-03-06_19.00.05 11-03-06_19.59.55 0 +1st test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0 +2nd test2-11-03-06_19.59.55.log test2.log 11-03-06_19.59.55 11-03-06_20.00.05 0 +2nd test2-11-03-06_20.00.05.log test2.log 11-03-06_20.00.05 11-03-06_20.59.55 0 +1st test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0 +2nd test2-11-03-06_20.59.55.log test2.log 11-03-06_20.59.55 11-03-06_21.00.05 0 +2nd test2-11-03-06_21.00.05.log test2.log 11-03-06_21.00.05 11-03-06_21.59.55 0 +1st test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0 +2nd test2-11-03-06_21.59.55.log test2.log 11-03-06_21.59.55 11-03-06_22.00.05 0 +2nd test2-11-03-06_22.00.05.log test2.log 11-03-06_22.00.05 11-03-06_22.59.55 0 +1st test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0 +2nd test2-11-03-06_22.59.55.log test2.log 11-03-06_22.59.55 11-03-06_23.00.05 0 +2nd test2-11-03-06_23.00.05.log test2.log 11-03-06_23.00.05 11-03-06_23.59.55 0 +1st test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0 +2nd test2-11-03-06_23.59.55.log test2.log 11-03-06_23.59.55 11-03-07_00.00.05 0 +2nd test2-11-03-07_00.00.05.log test2.log 11-03-07_00.00.05 11-03-07_00.59.55 0 +1st test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0 +2nd test2-11-03-07_00.59.55.log test2.log 11-03-07_00.59.55 11-03-07_01.00.05 0 +2nd test2-11-03-07_01.00.05.log test2.log 11-03-07_01.00.05 11-03-07_01.59.55 0 +1st test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0 +2nd test2-11-03-07_01.59.55.log test2.log 11-03-07_01.59.55 11-03-07_02.00.05 0 +2nd test2-11-03-07_02.00.05.log test2.log 11-03-07_02.00.05 11-03-07_02.59.55 0 +1st test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0 +2nd test2-11-03-07_02.59.55.log test2.log 11-03-07_02.59.55 11-03-07_03.00.05 0 +2nd test2-11-03-07_03.00.05.log test2.log 11-03-07_03.00.05 11-03-07_03.59.55 0 +1st test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0 +2nd test2-11-03-07_03.59.55.log test2.log 11-03-07_03.59.55 11-03-07_04.00.05 0 +2nd test2-11-03-07_04.00.05.log test2.log 11-03-07_04.00.05 11-03-07_04.59.55 0 +1st test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 +2nd test2-11-03-07_04.59.55.log test2.log 11-03-07_04.59.55 11-03-07_04.59.55 1 +> test-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +1299502795.0 10.0.0.2 20 10.0.0.3 9 +> test2-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +> test2-11-03-06_19.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test2-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +> test2-11-03-06_20.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test2-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +> test2-11-03-06_21.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test2-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +> test2-11-03-06_22.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test2-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +> test2-11-03-06_23.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test2-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +> test2-11-03-07_00.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test2-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +> test2-11-03-07_01.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test2-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +> test2-11-03-07_02.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test2-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +> test2-11-03-07_03.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test2-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +> test2-11-03-07_04.59.55.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299502795.0 10.0.0.2 20 10.0.0.3 9 +> test2.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +> test.log +# t id.orig_h id.orig_p id.resp_h id.resp_p diff --git a/testing/btest/Baseline/logging.rotate/out b/testing/btest/Baseline/logging.rotate/out new file mode 100644 index 0000000000..006cd2c851 --- /dev/null +++ b/testing/btest/Baseline/logging.rotate/out @@ -0,0 +1,50 @@ +test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0 +test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0 +test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0 +test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0 +test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0 +test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0 +test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0 +test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0 +test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0 +test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1 +> test-11-03-06_19.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299466805.0 10.0.0.1 20 10.0.0.2 1024 +1299470395.0 10.0.0.2 20 10.0.0.3 0 +> test-11-03-06_20.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299470405.0 10.0.0.1 20 10.0.0.2 1025 +1299473995.0 10.0.0.2 20 10.0.0.3 1 +> test-11-03-06_21.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299474005.0 10.0.0.1 20 10.0.0.2 1026 +1299477595.0 10.0.0.2 20 10.0.0.3 2 +> test-11-03-06_22.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299477605.0 10.0.0.1 20 10.0.0.2 1027 +1299481195.0 10.0.0.2 20 10.0.0.3 3 +> test-11-03-06_23.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299481205.0 10.0.0.1 20 10.0.0.2 1028 +1299484795.0 10.0.0.2 20 10.0.0.3 4 +> test-11-03-07_00.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299484805.0 10.0.0.1 20 10.0.0.2 1029 +1299488395.0 10.0.0.2 20 10.0.0.3 5 +> test-11-03-07_01.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299488405.0 10.0.0.1 20 10.0.0.2 1030 +1299491995.0 10.0.0.2 20 10.0.0.3 6 +> test-11-03-07_02.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299492005.0 10.0.0.1 20 10.0.0.2 1031 +1299495595.0 10.0.0.2 20 10.0.0.3 7 +> test-11-03-07_03.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299495605.0 10.0.0.1 20 10.0.0.2 1032 +1299499195.0 10.0.0.2 20 10.0.0.3 8 +> test-11-03-07_04.00.05.log +# t id.orig_h id.orig_p id.resp_h id.resp_p +1299499205.0 10.0.0.1 20 10.0.0.2 1033 +1299502795.0 10.0.0.2 20 10.0.0.3 9 diff --git a/testing/btest/Baseline/logging.stdout/output b/testing/btest/Baseline/logging.stdout/output new file mode 100644 index 0000000000..4c73aed8e4 --- /dev/null +++ b/testing/btest/Baseline/logging.stdout/output @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 failure US +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 success BR +1299718506.28824 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.test-logging/ssh.log b/testing/btest/Baseline/logging.test-logging/ssh.log new file mode 100644 index 0000000000..82523b7c13 --- /dev/null +++ b/testing/btest/Baseline/logging.test-logging/ssh.log @@ -0,0 +1,6 @@ +# t id.orig_h id.orig_p id.resp_h id.resp_p status country +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 success unknown +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 failure US +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 failure UK +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 success BR +1299718506.1313 1.2.3.4 1234 2.3.4.5 80 failure MX diff --git a/testing/btest/Baseline/logging.types/ssh.log b/testing/btest/Baseline/logging.types/ssh.log new file mode 100644 index 0000000000..ee657ab462 --- /dev/null +++ b/testing/btest/Baseline/logging.types/ssh.log @@ -0,0 +1,2 @@ +# b i e c p sn n a d t iv s sc ss se vc ve +T -42 SSH::SSH 21 123 10.0.0.0/24 10.0.0.0 1.2.3.4 3.14 1301359781.8203 100.0 hurz 4,1,3,2 CC,BB,AA EMPTY 10,20,30 EMPTY diff --git a/testing/btest/Baseline/logging.vec/ssh.log b/testing/btest/Baseline/logging.vec/ssh.log new file mode 100644 index 0000000000..1602f7d1c0 --- /dev/null +++ b/testing/btest/Baseline/logging.vec/ssh.log @@ -0,0 +1,2 @@ +# vec +-,2,-,-,5 diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 52f7e6280a..cdccac8184 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,10 +1,9 @@ - [btest] -TestDirs = doc bifs +TestDirs = doc bifs logging language TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp -IgnoreFiles = *.tmp *.swp #* +IgnoreFiles = *.tmp *.swp #* *.trace [environment] BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev` diff --git a/testing/btest/language/rec-nested-opt.bro b/testing/btest/language/rec-nested-opt.bro new file mode 100644 index 0000000000..eb7375541b --- /dev/null +++ b/testing/btest/language/rec-nested-opt.bro @@ -0,0 +1,25 @@ + +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type Version: record { + major: count &optional; ##< Major version number + minor: count &optional; ##< Minor version number + addl: string &optional; ##< Additional version string (e.g. "beta42") +} &log; + +type Info: record { + name: string; + version: Version; + host: addr; + ts: time; +}; + + +# Important thing to note here is that $minor2 is not include in the $version field. +global matched_software: table[string] of Info = { + ["Wget/1.9+cvs-stable (Red Hat modified)"] = + [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=0.0.0.0, $ts=network_time()], +}; + +print matched_software; diff --git a/testing/btest/language/rec-table-default.bro b/testing/btest/language/rec-table-default.bro new file mode 100644 index 0000000000..1473933e6a --- /dev/null +++ b/testing/btest/language/rec-table-default.bro @@ -0,0 +1,19 @@ + +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type X: record { + a: table[string] of bool &default=table( ["foo"] = T ); + b: table[string] of bool &default=table(); + c: set[string] &default=set("A", "B", "C"); + d: set[string] &default=set(); +}; + +global x: X; +global y: table[string] of bool &default=T; + +print x$a; +print x$b; +print x$c; +print x$d; + diff --git a/testing/btest/language/record-extension.bro b/testing/btest/language/record-extension.bro new file mode 100644 index 0000000000..c05b9da5e8 --- /dev/null +++ b/testing/btest/language/record-extension.bro @@ -0,0 +1,19 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +type Foo: record { + a: count; + b: count &optional; +}; + +redef record Foo += { + c: count &default=42; + d: count &optional; +}; + +global f1: Foo = [$a=21]; +global f2: Foo = [$a=21, $d="XXX"]; + +print f1; +print f2; + diff --git a/testing/btest/language/record-ref-assign.bro b/testing/btest/language/record-ref-assign.bro new file mode 100644 index 0000000000..f71bc3890c --- /dev/null +++ b/testing/btest/language/record-ref-assign.bro @@ -0,0 +1,12 @@ +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +type State: record { + host: string &default="NOT SET"; +}; + +global session: State; +global s: State; +s = session; +s$host = "XXX"; +print s$host, session$host; diff --git a/testing/btest/language/vector-coerce-expr.bro b/testing/btest/language/vector-coerce-expr.bro new file mode 100644 index 0000000000..d58417f226 --- /dev/null +++ b/testing/btest/language/vector-coerce-expr.bro @@ -0,0 +1,20 @@ +# @TEST-EXEC: bro %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +type X: record { + a: vector of bool &default=vector(T, F, T); + b: vector of bool &default=vector(); +}; + +global x: X; + +global a: vector of count; + +a = vector(); +print a; + +a = vector(1,2,3); +print a; + +print x$a; +print x$b; diff --git a/testing/btest/language/wrong-record-extension.bro b/testing/btest/language/wrong-record-extension.bro new file mode 100644 index 0000000000..4e0210546a --- /dev/null +++ b/testing/btest/language/wrong-record-extension.bro @@ -0,0 +1,14 @@ +# @TEST-EXEC-FAIL: bro %INPUT >output.tmp 2>&1 +# @TEST-EXEC: sed 's#^.*:##g' output +# @TEST-EXEC: btest-diff output + +type Foo: record { + a: count; + b: count &optional; +}; + +redef record Foo += { + c: count; + d: string &optional; +}; + diff --git a/testing/btest/logging/adapt-filter.bro b/testing/btest/logging/adapt-filter.bro new file mode 100644 index 0000000000..6312e4c106 --- /dev/null +++ b/testing/btest/logging/adapt-filter.bro @@ -0,0 +1,35 @@ + +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh-new-default.log +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local filter = Log::get_filter(SSH, "default"); + filter$path= "ssh-new-default"; + Log::add_filter(SSH, filter); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); +} diff --git a/testing/btest/logging/ascii-empty.bro b/testing/btest/logging/ascii-empty.bro new file mode 100644 index 0000000000..9e91ebc089 --- /dev/null +++ b/testing/btest/logging/ascii-empty.bro @@ -0,0 +1,38 @@ +# +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +redef LogAscii::output_to_stdout = T; +redef LogAscii::separator = "|"; +redef LogAscii::empty_field = "EMPTY"; +redef LogAscii::unset_field = "NOT-SET"; +redef LogAscii::header_prefix = "PREFIX<>"; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + b: bool &optional; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $b=T, $status="failure", $country=""]); + +} + diff --git a/testing/btest/logging/ascii-escape.bro b/testing/btest/logging/ascii-escape.bro new file mode 100644 index 0000000000..7acea46250 --- /dev/null +++ b/testing/btest/logging/ascii-escape.bro @@ -0,0 +1,32 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +redef LogAscii::separator = "||"; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="fa||ure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="su||ess", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); +} + diff --git a/testing/btest/logging/ascii-options.bro b/testing/btest/logging/ascii-options.bro new file mode 100644 index 0000000000..3f33c17b96 --- /dev/null +++ b/testing/btest/logging/ascii-options.bro @@ -0,0 +1,35 @@ +# +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +redef LogAscii::output_to_stdout = T; +redef LogAscii::separator = "|"; +redef LogAscii::include_header = F; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/attr-extend.bro b/testing/btest/logging/attr-extend.bro new file mode 100644 index 0000000000..4d7e96b98e --- /dev/null +++ b/testing/btest/logging/attr-extend.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; + status: string &optional &log; + country: string &default="unknown" &log; + }; +} + +redef record Log += { + a1: count &log &optional; + a2: count &optional; +}; + +redef record Log += { + b1: count &optional; + b2: count &optional; +} &log; + + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $a1=1, $a2=2, $b1=3, $b2=4]); +} + diff --git a/testing/btest/logging/attr.bro b/testing/btest/logging/attr.bro new file mode 100644 index 0000000000..658dcae04b --- /dev/null +++ b/testing/btest/logging/attr.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; + status: string &optional &log; + country: string &default="unknown" &log; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/disable-stream.bro b/testing/btest/logging/disable-stream.bro new file mode 100644 index 0000000000..472766d8e0 --- /dev/null +++ b/testing/btest/logging/disable-stream.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + Log::disable_stream(SSH); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/empty-event.bro b/testing/btest/logging/empty-event.bro new file mode 100644 index 0000000000..3e343b75a2 --- /dev/null +++ b/testing/btest/logging/empty-event.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +global log_ssh: event(rec: Log); + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log, $ev=log_ssh]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/events.bro b/testing/btest/logging/events.bro new file mode 100644 index 0000000000..101b13b095 --- /dev/null +++ b/testing/btest/logging/events.bro @@ -0,0 +1,39 @@ + +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +global ssh_log: event(rec: Log); + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log, $ev=ssh_log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + +} + +event ssh_log(rec: Log) + { + print rec; + } diff --git a/testing/btest/logging/exclude.bro b/testing/btest/logging/exclude.bro new file mode 100644 index 0000000000..46603d3202 --- /dev/null +++ b/testing/btest/logging/exclude.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + Log::remove_default_filter(SSH); + Log::add_filter(SSH, [$name="f1", $exclude=set("t", "id.orig_h")]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/file.bro b/testing/btest/logging/file.bro new file mode 100644 index 0000000000..6d73ec52dd --- /dev/null +++ b/testing/btest/logging/file.bro @@ -0,0 +1,23 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + f: file; + } &log; +} + +const foo_log = open_log_file("Foo") &redef; + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::write(SSH, [$t=network_time(), $f=foo_log]); +} + diff --git a/testing/btest/logging/include.bro b/testing/btest/logging/include.bro new file mode 100644 index 0000000000..f1fac64bbd --- /dev/null +++ b/testing/btest/logging/include.bro @@ -0,0 +1,34 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + Log::remove_default_filter(SSH); + Log::add_filter(SSH, [$name="default", $include=set("t", "id.orig_h")]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/no-local.bro b/testing/btest/logging/no-local.bro new file mode 100644 index 0000000000..eb6cb60151 --- /dev/null +++ b/testing/btest/logging/no-local.bro @@ -0,0 +1,33 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +redef Log::enable_local_logging = F; + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/path-func.bro b/testing/btest/logging/path-func.bro new file mode 100644 index 0000000000..b744057a84 --- /dev/null +++ b/testing/btest/logging/path-func.bro @@ -0,0 +1,50 @@ + +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: ( ls static-*; cat static-* ) >output +# @TEST-EXEC: btest-diff output + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +global c = -1; + +function path_func(id: Log::ID, path: string) : string + { + c = (c + 1) % 3; + + return fmt("%s-%d", path, c); + } + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::remove_default_filter(SSH); + + Log::add_filter(SSH, [$name="dyn", $path="static-prefix", $path_func=path_func]); + + Log::set_buf(SSH, F); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX2"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX3"]); +} diff --git a/testing/btest/logging/pred.bro b/testing/btest/logging/pred.bro new file mode 100644 index 0000000000..eb1a02cf08 --- /dev/null +++ b/testing/btest/logging/pred.bro @@ -0,0 +1,41 @@ + +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.success.log +# @TEST-EXEC: btest-diff ssh.failure.log + +module SSH; + +@load logging + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::remove_default_filter(SSH); + Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); + Log::add_filter(SSH, [$name="f2", $path="ssh.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + +} diff --git a/testing/btest/logging/remote-types.bro b/testing/btest/logging/remote-types.bro new file mode 100644 index 0000000000..ecf3e96bc5 --- /dev/null +++ b/testing/btest/logging/remote-types.bro @@ -0,0 +1,93 @@ +# +# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-diff receiver/ssh.log +# @TEST-EXEC: cmp receiver/ssh.log sender/ssh.log + +# Remote version testing all types. + +# This is the common part loaded by both sender and receiver. + +redef LogAscii::empty_field = "EMPTY"; + +module SSH; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + n: net; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of count; + ve: vector of string; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); +} + +##### + +@TEST-START-FILE sender.bro + +module SSH; + +@load listen-clear + +event remote_connection_handshake_done(p: event_peer) + { + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(SSH, [ + $b=T, + $i=-42, + $e=SSH, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $n=10.0., + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector + ]); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +@load remote + +redef Remote::destinations += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/btest/logging/remote.bro b/testing/btest/logging/remote.bro new file mode 100644 index 0000000000..cd0a361b7c --- /dev/null +++ b/testing/btest/logging/remote.bro @@ -0,0 +1,77 @@ +# +# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-wait -k 1 +# @TEST-EXEC: btest-diff sender/ssh.log +# @TEST-EXEC: btest-diff sender/ssh.failure.log +# @TEST-EXEC: btest-diff sender/ssh.success.log +# @TEST-EXEC: cmp receiver/ssh.log sender/ssh.log +# @TEST-EXEC: cmp receiver/ssh.failure.log sender/ssh.failure.log +# @TEST-EXEC: cmp receiver/ssh.success.log sender/ssh.success.log + +# This is the common part loaded by both sender and receiver. +module SSH; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::add_filter(SSH, [$name="f1", $path="ssh.success", $pred=function(rec: Log): bool { return rec$status == "success"; }]); +} + +##### + +@TEST-START-FILE sender.bro + +module SSH; + +@load listen-clear + +function fail(rec: Log): bool + { + return rec$status != "success"; + } + +event remote_connection_handshake_done(p: event_peer) + { + Log::add_filter(SSH, [$name="f2", $path="ssh.failure", $pred=fail]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + local r: Log = [$t=network_time(), $id=cid, $status="success"]; + + # Log something. + Log::write(SSH, r); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + } +@TEST-END-FILE + +@TEST-START-FILE receiver.bro + +##### + +@load remote + +redef Remote::destinations += { + ["foo"] = [$host = 127.0.0.1, $connect=T, $request_logs=T] +}; + +@TEST-END-FILE diff --git a/testing/btest/logging/remove.bro b/testing/btest/logging/remove.bro new file mode 100644 index 0000000000..488b21408f --- /dev/null +++ b/testing/btest/logging/remove.bro @@ -0,0 +1,41 @@ +# +# @TEST-EXEC: bro -B logging %INPUT +# @TEST-EXEC: btest-diff ssh.log +# @TEST-EXEC: btest-diff ssh.failure.log + +module SSH; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { SSH }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + Log::add_filter(SSH, [$name="f1", $path="ssh.failure", $pred=function(rec: Log): bool { return rec$status == "failure"; }]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + # Log something. + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + + Log::remove_filter(SSH, "f1"); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="BR"]); + + Log::remove_filter(SSH, "default"); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + + Log::remove_filter(SSH, "doesn-not-exist"); +} + diff --git a/testing/btest/logging/rotate-custom.bro b/testing/btest/logging/rotate-custom.bro new file mode 100644 index 0000000000..66e90de8c3 --- /dev/null +++ b/testing/btest/logging/rotate-custom.bro @@ -0,0 +1,37 @@ +# +# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out +# @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: btest-diff out + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { Test }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + } &log; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor = "echo 1st"; + +redef Log::rotation_control += { + [Log::WRITER_ASCII, "test2"] = [$interv=30mins, $postprocessor="echo 2nd"] +}; + +event bro_init() +{ + Log::create_stream(Test, [$columns=Log]); + Log::add_filter(Test, [$name="2nd", $path="test2"]); + +} + +event new_connection(c: connection) + { + Log::write(Test, [$t=network_time(), $id=c$id]); + } diff --git a/testing/btest/logging/rotate.bro b/testing/btest/logging/rotate.bro new file mode 100644 index 0000000000..dc7cd79d56 --- /dev/null +++ b/testing/btest/logging/rotate.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT >out +# @TEST-EXEC: for i in test-*.log; do printf '> %s\n' $i; cat $i; done >>out +# @TEST-EXEC: btest-diff out + +module Test; + +export { + # Create a new ID for our log stream + redef enum Log::ID += { Test }; + + # Define a record with all the columns the log file can have. + # (I'm using a subset of fields from ssh-ext for demonstration.) + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + } &log; +} + +redef Log::default_rotation_interval = 1hr; +redef Log::default_rotation_postprocessor = "echo"; + +event bro_init() +{ + Log::create_stream(Test, [$columns=Log]); +} + +event new_connection(c: connection) + { + Log::write(Test, [$t=network_time(), $id=c$id]); + } diff --git a/testing/btest/logging/rotation.trace b/testing/btest/logging/rotation.trace new file mode 100644 index 0000000000..9954b22e15 Binary files /dev/null and b/testing/btest/logging/rotation.trace differ diff --git a/testing/btest/logging/stdout.bro b/testing/btest/logging/stdout.bro new file mode 100644 index 0000000000..a482f742a0 --- /dev/null +++ b/testing/btest/logging/stdout.bro @@ -0,0 +1,36 @@ +# +# @TEST-EXEC: bro %INPUT >output +# @TEST-EXEC: btest-diff output +# @TEST-EXEC: test '!' -e ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local filter = Log::get_filter(SSH, "default"); + filter$path= "/dev/stdout"; + Log::add_filter(SSH, filter); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/test-logging.bro b/testing/btest/logging/test-logging.bro new file mode 100644 index 0000000000..8443bc2236 --- /dev/null +++ b/testing/btest/logging/test-logging.bro @@ -0,0 +1,31 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + t: time; + id: conn_id; # Will be rolled out into individual columns. + status: string &optional; + country: string &default="unknown"; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp]; + + Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]); + Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]); + +} + diff --git a/testing/btest/logging/types.bro b/testing/btest/logging/types.bro new file mode 100644 index 0000000000..8cd59192bd --- /dev/null +++ b/testing/btest/logging/types.bro @@ -0,0 +1,62 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log +# +# Testing all possible types. + +redef LogAscii::empty_field = "EMPTY"; + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + n: net; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of count; + ve: vector of string; + } &log; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(SSH, [ + $b=T, + $i=-42, + $e=SSH, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $n=10.0., + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1,2,3,4), + $ss=set("AA", "BB", "CC"), + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector + ]); +} + diff --git a/testing/btest/logging/vec.bro b/testing/btest/logging/vec.bro new file mode 100644 index 0000000000..8d882d53bb --- /dev/null +++ b/testing/btest/logging/vec.bro @@ -0,0 +1,27 @@ +# +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff ssh.log + +module SSH; + +export { + redef enum Log::ID += { SSH }; + + type Log: record { + vec: vector of string &log; + }; +} + +event bro_init() +{ + Log::create_stream(SSH, [$columns=Log]); + + local v: vector of string; + + v[2] = "2"; + v[5] = "5"; + + Log::write(SSH, [$vec=v]); +} + +