diff --git a/scripts/base/frameworks/logging/main.bro b/scripts/base/frameworks/logging/main.bro index 979aff0f63..69fcff2936 100644 --- a/scripts/base/frameworks/logging/main.bro +++ b/scripts/base/frameworks/logging/main.bro @@ -117,19 +117,11 @@ export { ## data storage and analysis systems. const default_field_name_map: table[string] of string = table() &redef; - ## Default separator for unrolled and flattened fields names for - ## nested records. - const default_unrolling_sep = "." &redef; - - ## A prefix for metadata fields which can be optionally prefixed - ## on all log lines by setting the `metadata_func` field in the - ## log filter. - const Log::default_metadata_prefix: string = "_" &redef; - - ## Default metadata function in the case that you would like to - ## apply the same metadata to all logs. The function *must* return - ## a record with all of the fields to be included in the metadata. - const Log::default_metadata_func: function(path: string): any &redef; + ## Default separator for log field scopes when logs are unrolled and + ## flattened. This will be the string between field name components. + ## For example, setting this to "_" will cause the typical field + ## "id.orig_h" to turn into "id_orig_h". + const default_scope_sep = "." &redef; ## A filter type describes how to customize logging streams. type Filter: record { @@ -211,21 +203,11 @@ export { ## A string that is used for unrolling and flattening field names ## for nested record types. - unrolling_sep: string &default=default_unrolling_sep; + scope_sep: string &default=default_scope_sep; ## Rotation interval. Zero disables rotation. interv: interval &default=default_rotation_interval; - ## Default prefix for all metadata fields. It's typically - ## prudent to set this to something that Bro's logging - ## framework can't normally write out in a field name. - metadata_prefix: string &default="_"; - - ## Function to collect a metadata value. If not specified, no - ## metadata will be provided for the log. - ## The return value from the function *must* be a record. - metadata_func: function(path: string): any &optional; - ## Callback function to trigger for rotated files. If not set, the ## default comes out of :bro:id:`Log::default_rotation_postprocessors`. postprocessor: function(info: RotationInfo) : bool &optional; @@ -236,6 +218,33 @@ export { config: table[string] of string &default=table(); }; + ## A prefix for extension fields which can be optionally prefixed + ## on all log lines by setting the `ext_func` field in the + ## log filter. + const Log::default_ext_prefix: string = "_" &redef; + + ## Default log extension function in the case that you would like to + ## apply the same extensions to all logs. The function *must* return + ## a record with all of the fields to be included in the log. The + ## default function included here returns F as a marker to indicate + ## that it has no implementation. + const Log::default_ext_func: function(filter: Log::Filter): any = + function(filter: Log::Filter): bool { return F; } &redef; + + # This is a hack for now since fields can't self-reference the + # record type they are contained within. + redef record Log::Filter += { + ## Default prefix for all extension fields. It's typically + ## prudent to set this to something that Bro's logging + ## framework can't normally write out in a field name. + ext_prefix: string &default=Log::default_ext_prefix; + + ## Function to collect a log extension value. If not specified, + ## no log extension will be provided for the log. + ## The return value from the function *must* be a record. + ext_func: function(filter: Log::Filter): any &default=Log::default_ext_func; + }; + ## Sentinel value for indicating that a filter was not found when looked up. const no_filter: Filter = [$name=""]; diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 3666a1cc84..fc967d8390 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -23,6 +23,7 @@ using namespace logging; struct Manager::Filter { + Val* fval; string name; EnumVal* id; Func* pred; @@ -32,10 +33,10 @@ struct Manager::Filter { EnumVal* writer; TableVal* config; TableVal* field_name_map; - string unrolling_sep; - string metadata_prefix; - Func* metadata_func; - int num_metadata; + string scope_sep; + string ext_prefix; + Func* ext_func; + int num_ext_fields; bool local; bool remote; double interval; @@ -88,6 +89,8 @@ struct Manager::Stream { Manager::Filter::~Filter() { + Unref(fval); + for ( int i = 0; i < num_fields; ++i ) delete fields[i]; @@ -381,22 +384,22 @@ bool Manager::DisableStream(EnumVal* id) bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list indices) { - // Only include metadata for the outer record. - int num_metadata = (indices.size() == 0) ? filter->num_metadata : 0; + // Only include extensions for the outer record. + int num_ext_fields = (indices.size() == 0) ? filter->num_ext_fields : 0; int i = 0; - for ( int j = 0; j < num_metadata + rt->NumFields(); ++j ) + for ( int j = 0; j < num_ext_fields + rt->NumFields(); ++j ) { RecordType* rtype; - // If this is a metadata field, set the rtype appropriately - if ( j < num_metadata ) + // If this is an ext field, set the rtype appropriately + if ( j < num_ext_fields ) { i = j; - rtype = filter->metadata_func->FType()->YieldType()->AsRecordType(); + rtype = filter->ext_func->FType()->YieldType()->AsRecordType(); } else { - i = j - num_metadata; + i = j - num_ext_fields; rtype = rt; } @@ -415,11 +418,11 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt, if ( ! path.size() ) new_path = rtype->FieldName(i); else - new_path = path + filter->unrolling_sep + rtype->FieldName(i); + new_path = path + filter->scope_sep + rtype->FieldName(i); - // Add the metadata prefix if this is a metadata field. - if ( j < num_metadata ) - new_path = filter->metadata_prefix + new_path; + // Add the ext prefix if this is an ext field. + if ( j < num_ext_fields ) + new_path = filter->ext_prefix + new_path; if ( t->InternalType() == TYPE_INTERNAL_OTHER ) { @@ -549,11 +552,12 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval) Val* postprocessor = fval->Lookup("postprocessor", true); Val* config = fval->Lookup("config", true); Val* field_name_map = fval->Lookup("field_name_map", true); - Val* unrolling_sep = fval->Lookup("unrolling_sep", true); - Val* metadata_prefix = fval->Lookup("metadata_prefix", true); - Val* metadata_func = fval->Lookup("metadata_func", true); + Val* scope_sep = fval->Lookup("scope_sep", true); + Val* ext_prefix = fval->Lookup("ext_prefix", true); + Val* ext_func = fval->Lookup("ext_func", true); Filter* filter = new Filter; + filter->fval = fval->Ref(); filter->name = name->AsString()->CheckString(); filter->id = id->Ref()->AsEnumVal(); filter->pred = pred ? pred->AsFunc() : 0; @@ -565,9 +569,9 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval) filter->postprocessor = postprocessor ? postprocessor->AsFunc() : 0; filter->config = config->Ref()->AsTableVal(); filter->field_name_map = field_name_map->Ref()->AsTableVal(); - filter->unrolling_sep = unrolling_sep->AsString()->CheckString(); - filter->metadata_prefix = metadata_prefix->AsString()->CheckString(); - filter->metadata_func = metadata_func ? metadata_func->AsFunc() : 0; + filter->scope_sep = scope_sep->AsString()->CheckString(); + filter->ext_prefix = ext_prefix->AsString()->CheckString(); + filter->ext_func = ext_func ? ext_func->AsFunc() : 0; Unref(name); Unref(pred); @@ -578,18 +582,33 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval) Unref(postprocessor); Unref(config); Unref(field_name_map); - Unref(unrolling_sep); - Unref(metadata_prefix); - Unref(metadata_func); + Unref(scope_sep); + Unref(ext_prefix); + Unref(ext_func); // Build the list of fields that the filter wants included, including // potentially rolling out fields. Val* include = fval->Lookup("include"); Val* exclude = fval->Lookup("exclude"); - filter->num_metadata = 0; - if ( filter->metadata_func ) - filter->num_metadata = filter->metadata_func->FType()->YieldType()->AsRecordType()->NumFields(); + filter->num_ext_fields = 0; + if ( filter->ext_func ) + { + if ( filter->ext_func->FType()->YieldType()->Tag() == TYPE_RECORD ) + { + filter->num_ext_fields = filter->ext_func->FType()->YieldType()->AsRecordType()->NumFields(); + } + else if ( filter->ext_func->FType()->YieldType()->Tag() == TYPE_BOOL ) + { + // This is a special marker for the default no-implementation + // of the ext_func and we'll allow it to slide. + } + else + { + reporter->Error("return value of log_ext is not a record"); + return false; + } + } filter->num_fields = 0; filter->fields = 0; @@ -1040,21 +1059,21 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty) threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, RecordVal* columns) { - RecordVal* metadata_rec = 0; - if ( filter->metadata_func ) + RecordVal* ext_rec = 0; + if ( filter->num_ext_fields > 0 ) { val_list vl(1); - vl.append(new StringVal(filter->path)); - metadata_rec = filter->metadata_func->Call(&vl)->AsRecordVal(); + vl.append(filter->fval->AsRecordVal()->Ref()); + ext_rec = filter->ext_func->Call(&vl)->AsRecordVal(); } threading::Value** vals = new threading::Value*[filter->num_fields]; for ( int i=0; i < filter->num_fields; ++i ) { - if ( i < filter->num_metadata ) + if ( i < filter->num_ext_fields ) { - vals[i] = ValToLogVal(metadata_rec->Lookup(i)); + vals[i] = ValToLogVal(ext_rec->Lookup(i)); } else { @@ -1066,15 +1085,15 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, // potentially be nested inside other records. list& indices = filter->indices[i]; - bool metadata_done = false; + bool ext_done = false; for ( list::iterator j = indices.begin(); j != indices.end(); ++j ) { - // Only check for metadata on the outermost record. + // Only check for extension fields on the outermost record. int nmd = 0; - if ( !metadata_done ) + if ( ! ext_done ) { - nmd = filter->num_metadata; - metadata_done = true; + nmd = filter->num_ext_fields; + ext_done = true; } val = val->AsRecordVal()->Lookup((*j) - nmd); diff --git a/testing/btest/scripts/base/frameworks/logging/unrolling_sep.bro b/testing/btest/scripts/base/frameworks/logging/unrolling_sep.bro index 94736acaf5..9d58ef11c2 100644 --- a/testing/btest/scripts/base/frameworks/logging/unrolling_sep.bro +++ b/testing/btest/scripts/base/frameworks/logging/unrolling_sep.bro @@ -3,4 +3,4 @@ @load base/protocols/conn -redef Log::default_unrolling_sep = "_"; \ No newline at end of file +redef Log::default_scope_sep = "_"; \ No newline at end of file diff --git a/testing/btest/scripts/base/frameworks/logging/unrolling_sep_and_field_name_map.bro b/testing/btest/scripts/base/frameworks/logging/unrolling_sep_and_field_name_map.bro index 3363992c0e..3c72b7a833 100644 --- a/testing/btest/scripts/base/frameworks/logging/unrolling_sep_and_field_name_map.bro +++ b/testing/btest/scripts/base/frameworks/logging/unrolling_sep_and_field_name_map.bro @@ -6,7 +6,7 @@ @load base/protocols/conn -redef Log::default_unrolling_sep = "*"; +redef Log::default_scope_sep = "*"; redef Log::default_field_name_map = { ["id*orig_h"] = "src", ["id*orig_p"] = "src_port",