Final changes to log framework ext code.

The "metadata" functionality has been renamed to "ext" to
represent that the logs are being extended.  The function that
returns the record which is used to extend the log now receives
a log filter as it's single argument.

The field name "unrolling" is now renamed to "scope" so the variables
names now look like this: "Log::default_scope_sep"

scripts/base/frameworks/logging/main.bro

@@ -117,19 +117,11 @@ export {
 	## data storage and analysis systems.
 	const default_field_name_map: table[string] of string = table() &redef;
 
-	## Default separator for unrolled and flattened fields names for 
-	## nested records.
-	const default_unrolling_sep = "." &redef;
-
-	## A prefix for metadata fields which can be optionally prefixed
-	## on all log lines by setting the `metadata_func` field in the
-	## log filter.
-	const Log::default_metadata_prefix: string = "_" &redef;
-
-	## Default metadata function in the case that you would like to 
-	## apply the same metadata to all logs.  The function *must* return
-	## a record with all of the fields to be included in the metadata.
-	const Log::default_metadata_func: function(path: string): any &redef;
+	## Default separator for log field scopes when logs are unrolled and 
+	## flattened.  This will be the string between field name components.
+	## For example, setting this to "_" will cause the typical field 
+	## "id.orig_h" to turn into "id_orig_h".
+	const default_scope_sep = "." &redef;
 
 	## A filter type describes how to customize logging streams.
 	type Filter: record {
@@ -211,21 +203,11 @@ export {
 
 		## A string that is used for unrolling and flattening field names
 		## for nested record types.
-		unrolling_sep: string &default=default_unrolling_sep;
+		scope_sep: string &default=default_scope_sep;
 
 		## Rotation interval. Zero disables rotation.
 		interv: interval &default=default_rotation_interval;
 
-		## Default prefix for all metadata fields. It's typically
-		## prudent to set this to something that Bro's logging
-		## framework can't normally write out in a field name.
-		metadata_prefix: string &default="_";
-
-		## Function to collect a metadata value.  If not specified, no
-		## metadata will be provided for the log.
-		## The return value from the function *must* be a record.
-		metadata_func: function(path: string): any &optional;
-
 		## Callback function to trigger for rotated files. If not set, the
 		## default comes out of :bro:id:`Log::default_rotation_postprocessors`.
 		postprocessor: function(info: RotationInfo) : bool &optional;
@@ -236,6 +218,33 @@ export {
 		config: table[string] of string &default=table();
 	};
 
+	## A prefix for extension fields which can be optionally prefixed
+	## on all log lines by setting the `ext_func` field in the
+	## log filter.
+	const Log::default_ext_prefix: string = "_" &redef;
+
+	## Default log extension function in the case that you would like to 
+	## apply the same extensions to all logs.  The function *must* return
+	## a record with all of the fields to be included in the log.  The 
+	## default function included here returns F as a marker to indicate
+	## that it has no implementation.
+	const Log::default_ext_func: function(filter: Log::Filter): any = 
+		function(filter: Log::Filter): bool { return F; } &redef;
+
+	# This is a hack for now since fields can't self-reference the
+	# record type they are contained within.
+	redef record Log::Filter += {
+		## Default prefix for all extension fields. It's typically
+		## prudent to set this to something that Bro's logging
+		## framework can't normally write out in a field name.
+		ext_prefix: string &default=Log::default_ext_prefix;
+
+		## Function to collect a log extension value.  If not specified, 
+		## no log extension will be provided for the log.
+		## The return value from the function *must* be a record.
+		ext_func: function(filter: Log::Filter): any &default=Log::default_ext_func;
+	};
+
 	## Sentinel value for indicating that a filter was not found when looked up.
 	const no_filter: Filter = [$name="<not found>"];
 

src/logging/Manager.cc

@@ -23,6 +23,7 @@
 using namespace logging;
 
 struct Manager::Filter {
+	Val* fval;
 	string name;
 	EnumVal* id;
 	Func* pred;
@@ -32,10 +33,10 @@ struct Manager::Filter {
 	EnumVal* writer;
 	TableVal* config;
 	TableVal* field_name_map;
-	string unrolling_sep;
-	string metadata_prefix;
-	Func* metadata_func;
-	int num_metadata;
+	string scope_sep;
+	string ext_prefix;
+	Func* ext_func;
+	int num_ext_fields;
 	bool local;
 	bool remote;
 	double interval;
@@ -88,6 +89,8 @@ struct Manager::Stream {
 
 Manager::Filter::~Filter()
 	{
+	Unref(fval);
+
 	for ( int i = 0; i < num_fields; ++i )
 		delete fields[i];
 
@@ -381,22 +384,22 @@ bool Manager::DisableStream(EnumVal* id)
 bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
 			    TableVal* include, TableVal* exclude, string path, list<int> indices)
 	{
-	// Only include metadata for the outer record.
-	int num_metadata = (indices.size() == 0) ? filter->num_metadata : 0;
+	// Only include extensions for the outer record.
+	int num_ext_fields = (indices.size() == 0) ? filter->num_ext_fields : 0;
 	
 	int i = 0;
-	for ( int j = 0; j < num_metadata + rt->NumFields(); ++j )
+	for ( int j = 0; j < num_ext_fields + rt->NumFields(); ++j )
 		{
 		RecordType* rtype;
-		// If this is a metadata field, set the rtype appropriately
-		if ( j < num_metadata )
+		// If this is an ext field, set the rtype appropriately
+		if ( j < num_ext_fields )
 			{
 			i = j;
-			rtype = filter->metadata_func->FType()->YieldType()->AsRecordType();
+			rtype = filter->ext_func->FType()->YieldType()->AsRecordType();
 			}
 		else
 			{
-			i = j - num_metadata;
+			i = j - num_ext_fields;
 			rtype = rt;
 			}
 
@@ -415,11 +418,11 @@ bool Manager::TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
 		if ( ! path.size() )
 			new_path = rtype->FieldName(i);
 		else
-			new_path = path + filter->unrolling_sep + rtype->FieldName(i);
+			new_path = path + filter->scope_sep + rtype->FieldName(i);
 
-		// Add the metadata prefix if this is a metadata field.
-		if ( j < num_metadata )
-			new_path = filter->metadata_prefix + new_path;
+		// Add the ext prefix if this is an ext field.
+		if ( j < num_ext_fields )
+			new_path = filter->ext_prefix + new_path;
 
 		if ( t->InternalType() == TYPE_INTERNAL_OTHER )
 			{
@@ -549,11 +552,12 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
 	Val* postprocessor = fval->Lookup("postprocessor", true);
 	Val* config = fval->Lookup("config", true);
 	Val* field_name_map = fval->Lookup("field_name_map", true);
-	Val* unrolling_sep = fval->Lookup("unrolling_sep", true);
-	Val* metadata_prefix = fval->Lookup("metadata_prefix", true);
-	Val* metadata_func = fval->Lookup("metadata_func", true);
+	Val* scope_sep = fval->Lookup("scope_sep", true);
+	Val* ext_prefix = fval->Lookup("ext_prefix", true);
+	Val* ext_func = fval->Lookup("ext_func", true);
 
 	Filter* filter = new Filter;
+	filter->fval = fval->Ref();
 	filter->name = name->AsString()->CheckString();
 	filter->id = id->Ref()->AsEnumVal();
 	filter->pred = pred ? pred->AsFunc() : 0;
@@ -565,9 +569,9 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
 	filter->postprocessor = postprocessor ? postprocessor->AsFunc() : 0;
 	filter->config = config->Ref()->AsTableVal();
 	filter->field_name_map = field_name_map->Ref()->AsTableVal();
-	filter->unrolling_sep = unrolling_sep->AsString()->CheckString();
-	filter->metadata_prefix = metadata_prefix->AsString()->CheckString();
-	filter->metadata_func = metadata_func ? metadata_func->AsFunc() : 0;
+	filter->scope_sep = scope_sep->AsString()->CheckString();
+	filter->ext_prefix = ext_prefix->AsString()->CheckString();
+	filter->ext_func = ext_func ? ext_func->AsFunc() : 0;
 
 	Unref(name);
 	Unref(pred);
@@ -578,18 +582,33 @@ bool Manager::AddFilter(EnumVal* id, RecordVal* fval)
 	Unref(postprocessor);
 	Unref(config);
 	Unref(field_name_map);
-	Unref(unrolling_sep);
-	Unref(metadata_prefix);
-	Unref(metadata_func);
+	Unref(scope_sep);
+	Unref(ext_prefix);
+	Unref(ext_func);
 
 	// Build the list of fields that the filter wants included, including
 	// potentially rolling out fields.
 	Val* include = fval->Lookup("include");
 	Val* exclude = fval->Lookup("exclude");
 
-	filter->num_metadata = 0;
-	if ( filter->metadata_func )
-		filter->num_metadata = filter->metadata_func->FType()->YieldType()->AsRecordType()->NumFields();
+	filter->num_ext_fields = 0;
+	if ( filter->ext_func )
+		{
+		if ( filter->ext_func->FType()->YieldType()->Tag() == TYPE_RECORD ) 
+			{
+			filter->num_ext_fields = filter->ext_func->FType()->YieldType()->AsRecordType()->NumFields();
+			}
+		else if ( filter->ext_func->FType()->YieldType()->Tag() == TYPE_BOOL )
+			{
+			// This is a special marker for the default no-implementation
+			// of the ext_func and we'll allow it to slide.
+			}
+		else
+			{
+			reporter->Error("return value of log_ext is not a record");
+			return false;
+			}
+		}
 
 	filter->num_fields = 0;
 	filter->fields = 0;
@@ -1040,21 +1059,21 @@ threading::Value* Manager::ValToLogVal(Val* val, BroType* ty)
 threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 				    RecordVal* columns)
 	{
-	RecordVal* metadata_rec = 0;
-	if ( filter->metadata_func )
+	RecordVal* ext_rec = 0;
+	if ( filter->num_ext_fields > 0 )
 		{
 		val_list vl(1);
-		vl.append(new StringVal(filter->path));
-		metadata_rec = filter->metadata_func->Call(&vl)->AsRecordVal();
+		vl.append(filter->fval->AsRecordVal()->Ref());
+		ext_rec = filter->ext_func->Call(&vl)->AsRecordVal();
 		}
 
 	threading::Value** vals = new threading::Value*[filter->num_fields];
 
 	for ( int i=0; i < filter->num_fields; ++i )
 		{
-		if ( i < filter->num_metadata )
+		if ( i < filter->num_ext_fields )
 			{
-			vals[i] = ValToLogVal(metadata_rec->Lookup(i));
+			vals[i] = ValToLogVal(ext_rec->Lookup(i));
 			}
 		else
 			{
@@ -1066,15 +1085,15 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter,
 			// potentially be nested inside other records.
 			list<int>& indices = filter->indices[i];
 
-			bool metadata_done = false;
+			bool ext_done = false;
 			for ( list<int>::iterator j = indices.begin(); j != indices.end(); ++j )
 				{
-				// Only check for metadata on the outermost record.
+				// Only check for extension fields on the outermost record.
 				int nmd = 0;
-				if ( !metadata_done )
+				if ( ! ext_done )
 					{
-					nmd = filter->num_metadata;
-					metadata_done = true;
+					nmd = filter->num_ext_fields;
+					ext_done = true;
 					}
 
 				val = val->AsRecordVal()->Lookup((*j) - nmd);

testing/btest/scripts/base/frameworks/logging/unrolling_sep.bro

@@ -3,4 +3,4 @@
 
 @load base/protocols/conn
 
-redef Log::default_unrolling_sep = "_";
+redef Log::default_scope_sep = "_";

testing/btest/scripts/base/frameworks/logging/unrolling_sep_and_field_name_map.bro

@@ -6,7 +6,7 @@
 
 @load base/protocols/conn
 
-redef Log::default_unrolling_sep = "*";
+redef Log::default_scope_sep = "*";
 redef Log::default_field_name_map = {
 	["id*orig_h"] = "src",
 	["id*orig_p"] = "src_port",