Lots of infracstructure for the new logging framework.

This pretty much follows the proposal on the projects page.

It includes:

    - A new LogMgr, maintaining the set of writers.

    - The abstract LogWriter API.

    - An initial implementation in the form of LogWriterAscii
      producing tab-separated columns.

Note that things are only partially working right now, things are
subject to change, and it's all not much tested at all. That's why I'm
creating separate branch for now.

Example:

     bro -B logging test-logging && cat debug.log
    1298063168.409852/1298063168.410368 [logging] Created new logging stream 'SSH::LOG_SSH'
    1298063168.409852/1298063168.410547 [logging] Created new filter 'default' for stream 'SSH::LOG_SSH'
    1298063168.409852/1298063168.410564 [logging]    writer    : Ascii
    1298063168.409852/1298063168.410574 [logging]    path      : ssh_log_ssh
    1298063168.409852/1298063168.410584 [logging]    path_func : not set
    1298063168.409852/1298063168.410594 [logging]    event     : not set
    1298063168.409852/1298063168.410604 [logging]    pred      : not set
    1298063168.409852/1298063168.410614 [logging]    field          t: time
    1298063168.409852/1298063168.410625 [logging]    field  id.orig_h: addr
    1298063168.409852/1298063168.410635 [logging]    field  id.orig_p: port
    1298063168.409852/1298063168.410645 [logging]    field  id.resp_h: addr
    1298063168.409852/1298063168.410655 [logging]    field  id.resp_p: port
    1298063168.409852/1298063168.410665 [logging]    field     status: string
    1298063168.409852/1298063168.410675 [logging]    field    country: string
    1298063168.409852/1298063168.410817 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH'
    1298063168.409852/1298063168.410865 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH'
    1298063168.409852/1298063168.410906 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH'
    1298063168.409852/1298063168.410945 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH'
    1298063168.409852/1298063168.411044 [logging] Wrote record to filter 'default' on stream 'SSH::LOG_SSH

> cat ssh_log_ssh.log
1298063168.40985        1.2.3.4 66770   2.3.4.5 65616   success unknown
1298063168.40985        1.2.3.4 66770   2.3.4.5 65616   failure US
1298063168.40985        1.2.3.4 66770   2.3.4.5 65616   failure UK
1298063168.40985        1.2.3.4 66770   2.3.4.5 65616   success BR
1298063168.40985        1.2.3.4 66770   2.3.4.5 65616   failure MX

policy/bro.init

@@ -272,6 +272,60 @@ type entropy_test_result: record {
 	serial_correlation: double;
 };
 
+type Log_Writer: enum { # TODO: Move these into bif and use from C++ as well.
+	WRITER_DEFAULT, # See default_writer below.
+	WRITER_ASCII,
+};
+	
+# Each stream gets a unique ID. This type will be extended by
+# other scripts.
+type Log_ID: enum {
+	Unknown
+};
+	
+# The default writer to use if a filter does not specify
+# anything else.
+const Log_default_writer = WRITER_ASCII &redef;
+
+# A filter defining what to log.
+type log_filter: record {
+	# A name to reference this filter.
+	name: string;
+
+	# A predicate returning True if the filter wants a log entry
+	# to be recorded. If not given, an implicit True is assumed
+	# for all entries. The predicate receives one parameter:
+	# an instance of the log's record type with the fields to be
+	# logged.
+	pred: function(rec: any): bool &optional;
+
+	# A path for outputting everything matching this
+	# filter. The path is either a string, or a function
+	# called with a single ``ID`` argument and returning a string.
+	#
+	# The specific interpretation of the string is left to the
+	# Writer, but if it's refering to a file, it's assumed that no
+	# extension is given; the writer will add whatever is
+	# appropiate.
+	path: string &optional;
+	path_func: function(id: string): string &optional;
+
+	# A subset of column names to record. If not given, all
+	# columns are recorded.
+	include: set[string] &optional;
+	exclude: set[string] &optional;
+
+	# An event that is raised whenever the filter is applied
+	# to an entry. The event receives the same parameter
+	# as the predicate. It will always be generated,
+	# independent of what the predicate returns.
+	ev: event(rec: any) &optional;
+
+	# The writer to use.
+	writer: Log_Writer &default=Log_default_writer;
+};
+
+
 # Prototypes of Bro built-in functions.
 @load strings.bif.bro
 @load bro.bif.bro
@@ -1389,3 +1443,4 @@ const trace_output_file = "";
 # packets out before we actually process them, which can be helpful
 # for debugging in case the analysis triggers a crash.
 const record_all_packets = F &redef;
+

policy/logging.bro

@@ -1,176 +1,10 @@
-module Logging;
 
-export {
+function Log_add_default_filter(id: Log_ID)
-	# The set of writers Bro provides.
-	type Writer: enum {
-		WRITER_DEFAULT, # See default_writer below.
-		WRITER_CSV,
-		WRITER_DATA_SERIES,
-		WRITER_SYSLOG
-	};
-	
-	# Each stream gets a unique ID. This type will be extended by
-	# other scripts.
-	type ID: enum {
-		Unknown
-		};
-	
-	# The default writer to use if a filter does not specify
-	# anything else.
-	const default_writer = WRITER_CSV &redef;
-	
-	# Type defining a stream.
-	type Stream: record {
-		name:     string;
-		columns:  string_vec;
-		};
-	
-	# A filter defining what to record.
-	type Filter: record {
-		# A name to reference this filter.
-		name: string;
-		
-		# A predicate returning True if the filter wants a log entry
-		# to be recorded. If not given, an implicit True is assumed
-		# for all entries. The predicate receives one parameter:
-		# an instance of the log's record type with the fields to be
-		# logged.
-		pred: function(rec: any): bool &optional;
-		
-		# A path for outputting everything matching this
-		# filter. The path is either a string, or a function
-		# called with a single ``ID`` argument and returning a string.
-		#
-		# The specific interpretation of the string is left to the
-		# Writer, but if it's refering to a file, it's assumed that no
-		# extension is given; the writer will add whatever is
-		# appropiate.
-		path: string &optional;
-		dynamic_path: function(id: string): string &optional;
-		
-		# A subset of column names to record. If not given, all
-		# columns are recorded.
-		#select: set[string] &optional;
-		
-		# An event that is raised whenever the filter is applied
-		# to an entry. The event receives the same parameter
-		# as the predicate. It will always be generated,
-		# independent of what the predicate returns.
-		#ev: event(rec: any) &optional;
-		
-		# The writer to use.
-		writer: Writer &default=default_writer;
-		
-		# Internal tracking of header names and order for this filter.
-		#columns: string_vec &optional;
-		};
-	
-	# Logs the record "rec" to the stream "id". The type of
-	# "rec" must match the stream's "columns" field.
-	global log: function(id: string, rec: any);
-	#global log_ev: event(id: string, rec: any);
-	
-	# Returns an existing filter previously installed for stream
-	# "id" under the given "name". If no such filter exists,
-	# the record "NoSuchFilter" is returned.
-	global get_filter: function(id: string, name: string) : Filter;
-	
-	global create_stream: function(id: string, log_record_type: string);
-	global add_filter: function(id: string, filter: Filter);
-	global remove_filter: function(id: string, filter: string): bool;
-	
-	global add_default_filter: function(id: string);
-	global remove_default_filter: function(id: string): bool;
-	
-	global open_log_files: function(id: string);
-	
-	# This is the internal filter store.  The outer table is indexed with a string
-	# representing the stream name that the set of Logging::Filters is applied to.
-	global filters: table[string] of set[Filter];
-	
-	# This is the internal stream store.  The table is indexed by the stream name.
-	global streams: table[string] of Stream;
-	
-	global files: table[string] of file;
-}
-
-
-# Sentinel representing an unknown filter.d
-const NoSuchFilter: Filter = [$name="<unknown filter>", $path="unknown"];
-
-function create_stream(id: string, log_record_type: string)
 	{
-	if ( id in streams )
+	log_add_filter(id, [$name="default"]);
-		print fmt("Stream %s already exists!", id);
-	
-	streams[id] = [$name=log_record_type, $columns=record_type_to_vector(log_record_type)];
-	# Insert this as a separate step because the file_opened event needs
-	# the stream id to already exist.
-	#streams[id]$_file = open_log_file(id);
 	}
 	
-function add_filter(id: string, filter: Filter)
+function Log_remove_default_filter(id: Log_ID): bool
 	{
-	if ( id !in filters )
+	log_remove_filter(id, "default");
-		filters[id] = set();
-	
-	add filters[id][filter];
-	}
-	
-function remove_filter(id: string, filter: string): bool
-	{
-	for ( filt in filters[id] )
-		{
-		if ( filt$name == "default" )
-			{
-			delete filters[id][filt];
-			return T;
-			}
-		}
-	return F;
-	}
-	
-function add_default_filter(id: string)
-	{
-	add_filter(id, [$name="default", $path=id]);
-	}
-	
-function remove_default_filter(id: string): bool
-	{
-	return remove_filter("ssh", "default");
-	}
-
-event file_opened(f: file) &priority=10
-	{
-	# Only do any of this for files opened locally.
-	if ( is_remote_event() ) return;
-
-	# TODO: this shouldn't rely on .log being the extension
-	local filename = gsub(get_file_name(f), /\.log$/, "");
-	if ( filename in streams )
-		{
-		enable_raw_output(f);
-		
-		if (peer_description == "" ||
-		    peer_description == "manager" ||
-		    peer_description == "standalone")
-			{
-			print f, join_string_vec(streams[filename]$columns, "\t");
-			}
-		}
-	else 
-		{
-		print "no raw output", filename;
-		}
-	}
-	
-function log(id: string, rec: any)
-	{
-	logging_log(id, rec);
-	}
-
-
-event bro_init() &priority=-10
-	{
-	# TODO: Check for logging streams without filters.
 	}

policy/test-logging.bro

@@ -4,7 +4,7 @@ module SSH;
 
 export {
 	# Create a new ID for our log stream
-	#redef enum Logging::ID += { LOG_SSH };
+	redef enum Log_ID += { LOG_SSH };
 
 	# Define a record with all the columns the log file can have.
 	# (I'm using a subset of fields from ssh-ext for demonstration.)
@@ -14,10 +14,6 @@ export {
 		status: string &optional;
 		country: string &default="unknown";
 	};
-	
-	# This is the prototype for the event that the logging framework tries
-	# to generate if there is a handler for it.
-	#global log: event(rec: Log);
 }
 
 event bro_init()
@@ -25,35 +21,27 @@ event bro_init()
 	# Create the stream.
 	# First argument is the ID for the stream.
 	# Second argument is the log record type.
-	Logging::create_stream("ssh", "SSH::Log");
+	log_create_stream(LOG_SSH, SSH::Log);
 
 	# Add a default filter that simply logs everything to "ssh.log" using the default writer.
-	# Log line event generation is autogenerated for now by checking for 
+	Log_add_default_filter(LOG_SSH);
-	# handlers for MODULE_NAME::log (which isn't the right thing to do, but it will be dealt with later)
-	Logging::add_default_filter("ssh");
-	
-	# There is currently some problem with &optional values in the records 
-	# passed into the predicate.  Maybe it's because I'm not really coercing
-	# the record to the correct record type before passing it as an argument
-	# to the Call method?
-	
-	# There is also a problem with using &optional sets in the filter records.
-	#   It was found when trying to include the "select" variable.
 
 	# Printing headers for the filters doesn't work yet either and needs to 
 	# be considered in the final design. (based on the "select" set).
-	#Logging::add_filter("ssh", [$name="successful logins",
+	#Log::add_filter("ssh", [$name="successful logins",
 	#                            #$pred(rec: Log) = { print rec$status; return T; },
 	#                            $path="ssh-logins",
 	#                            #$select=set("t"),
-	#                            $writer=Logging::WRITER_CSV]);
+	#                            $writer=Log::WRITER_CSV]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
 
 	# Log something.
-	Logging::log("ssh", [$t=network_time(),$status="success"]);
+	log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success"]);
-	Logging::log("ssh", [$t=network_time(),$status="failure", $country="US"]);
+	log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
-	Logging::log("ssh", [$t=network_time(),$status="failure", $country="UK"]);
+	log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
-	Logging::log("ssh", [$t=network_time(),$status="success", $country="BR"]);
+	log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
-	Logging::log("ssh", [$t=network_time(),$status="failure", $country="MX"]);
+	log_write(LOG_SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
 	
 }
 

src/CMakeLists.txt

@@ -297,6 +297,9 @@ set(bro_SRCS
     IRC.cc
     List.cc
     Logger.cc
+    LogMgr.cc
+    LogWriter.cc
+    LogWriterAscii.cc
     Login.cc
     MIME.cc
     NCP.cc

src/DebugLogger.cc

@@ -17,6 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
+	{ "logging", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -25,6 +25,7 @@ enum DebugStream {
 	DBG_MAINLOOP,	// Main IOSource loop
 	DBG_DPD,	// Dynamic application detection framework
 	DBG_TM,	// Time-machine packet input via Brocolli
+	DBG_LOGGING,	// Logging streams
 
 	NUM_DBGS // Has to be last
 };

src/LogMgr.cc

@@ -0,0 +1,499 @@
+
+#include "LogMgr.h"
+#include "EventHandler.h"
+#include "NetVar.h"
+
+#include "LogWriterAscii.h"
+
+struct LogWriterDefinition {
+    LogWriterType::Type type;	// The type.
+	const char *name;	// Descriptive name for error messages.
+	bool (*init)();	// An optional one-time initialization function.
+	LogWriter* (*factory)();	// A factory function creating instances.
+};
+
+LogWriterDefinition log_writers[] = {
+	{ LogWriterType::Ascii, "Ascii", 0, LogWriterAscii::Instantiate },
+
+	// End marker.
+	{ LogWriterType::None, "None", 0, (LogWriter* (*)())0 }
+};
+
+struct LogMgr::Filter {
+	string name;
+	Func* pred;
+	Func* path_func;
+	EventHandlerPtr* event;
+	string path;
+	LogWriterDefinition* writer;
+
+	int num_fields;
+	LogField** fields;
+	vector<list<int> > indices; // List of record indices per field.
+
+	typedef map<string, LogWriter *> WriterMap;
+	WriterMap writers; // Writers indexed by path.
+
+	~Filter();
+};
+
+struct LogMgr::Stream {
+	string name;
+	RecordType* columns;
+	list<Filter*> filters;
+
+	~Stream();
+	};
+
+
+LogMgr::Filter::~Filter()
+	{
+	for ( int i = 0; i < num_fields; ++i )
+		delete fields[i];
+
+	for ( WriterMap::iterator i = writers.begin(); i != writers.end(); i++ )
+		delete i->second;
+	}
+
+LogMgr::Stream::~Stream()
+	{
+	Unref(columns);
+	for ( list<Filter*>::iterator f = filters.begin(); f != filters.end(); ++f )
+		delete *f;
+	}
+
+LogMgr::LogMgr()
+	{
+	}
+
+LogMgr::~LogMgr()
+	{
+	for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
+		delete *s;
+	}
+
+bool LogMgr::CreateStream(EnumVal* stream_id, RecordType* columns)
+	{
+	// TODO: Should check that the record has only supported types.
+
+	unsigned int idx = stream_id->AsEnum();
+
+	// Make sure the vector has an entries for all streams up to the one
+	// given.
+	while ( idx >= streams.size() )
+		streams.push_back(0);
+
+	if ( streams[idx] )
+		// We already know this one, delete the previous definition.
+		delete streams[idx];
+
+	// Create new stream and record the type for the columns.
+	streams[idx] = new Stream;
+	streams[idx]->name = stream_id->Type()->AsEnumType()->Lookup(idx);
+	streams[idx]->columns = columns;
+	columns->Ref();
+
+	DBG_LOG(DBG_LOGGING, "Created new logging stream '%s'", streams[idx]->name.c_str());
+
+	return true;
+	}
+
+// Helper for recursive record field unrolling.
+bool LogMgr::TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list<int> indices)
+	{
+	for ( int i = 0; i < rt->NumFields(); ++i )
+		{
+		BroType* t = rt->FieldType(i);
+
+		list<int> new_indices = indices;
+		new_indices.push_back(i);
+
+		// Build path name.
+		string new_path;
+	   	if ( ! path.size() )
+			new_path = rt->FieldName(i);
+		else
+			new_path = path + "." + rt->FieldName(i);
+
+		StringVal* new_path_val = new StringVal(path.c_str());
+
+		if ( t->InternalType() == TYPE_INTERNAL_OTHER )
+			{
+			if ( t->Tag() == TYPE_RECORD )
+				{
+				// Recurse.
+				if ( ! TraverseRecord(filter, t->AsRecordType(), include, exclude, new_path, new_indices) )
+					return false;
+				}
+			else
+				{
+				run_time("unsupported field type for log column");
+				return false;
+				}
+
+			continue;
+			}
+
+		// If include fields are specified, only include if explicitly listed.
+		if ( include )
+			{
+			if ( ! include->Lookup(new_path_val) )
+				return true;
+			}
+
+		// If exclude fields are specified, do not only include if listed.
+		if ( exclude )
+			{
+			if ( exclude->Lookup(new_path_val) )
+				return true;
+			}
+
+		// Alright, we want this field.
+
+		filter->indices.push_back(new_indices);
+		filter->fields = (LogField**) realloc(filter->fields, sizeof(LogField) * ++filter->num_fields);
+		if ( ! filter->fields )
+			{
+			run_time("out of memory in add_filter");
+			return false;
+			}
+
+		LogField* field = new LogField();
+		field->name = new_path;
+		field->type = t->Tag();
+		filter->fields[filter->num_fields - 1] = field;
+		}
+
+	return true;
+	}
+
+bool LogMgr::AddFilter(EnumVal* stream_id, RecordVal* fval)
+	{
+	RecordType* rtype = fval->Type()->AsRecordType();
+
+	if ( ! same_type(rtype, log_filter, 0) )
+		{
+		run_time("filter argument not of right type");
+		return false;
+		}
+
+	Stream* stream = streams[stream_id->AsEnum()];
+	if ( ! stream )
+		{
+		run_time("undefined log stream");
+		return false;
+		}
+
+	// Find the right writer type.
+	int writer = 0;
+	int idx = rtype->FieldOffset("writer");
+	Val* writer_val = fval->Lookup(idx);
+
+	if ( ! writer_val )
+		{
+		// Use default.
+		// FIXME: Shouldn't Lookup() already take care if this?
+		const Attr* def_attr = log_filter->FieldDecl(idx)->FindAttr(ATTR_DEFAULT);
+		if ( ! def_attr )
+			internal_error("log_filter missing &default for writer attribute");
+
+		writer_val = def_attr->AttrExpr()->Eval(0);
+		writer = writer_val->AsEnum();
+		Unref(writer_val);
+		}
+	else
+		writer = writer_val->AsEnum();
+
+	LogWriterDefinition* ld;
+	for ( ld = log_writers; ld->type != LogWriterType::None; ++ld )
+		{
+		if ( ld->type == writer )
+			break;
+		}
+
+	if ( ld->type == LogWriterType::None )
+		internal_error("unknow writer in add_filter");
+
+	if ( ! ld->factory )
+		// Oops, we can't instantuate this guy.
+		return true; // Count as success, as we will have reported it earlier already.
+
+	// If the writer has an init function, call it.
+	if ( ld->init )
+		{
+		if ( (*ld->init)() )
+			// Clear the init function so that we won't call it again later.
+			ld->init = 0;
+		else
+			// Init failed, disable by deleting factory function.
+			ld->factory = 0;
+			return false;
+		}
+
+	// Create a new Filter instance.
+
+	Val* event = fval->Lookup(rtype->FieldOffset("ev"));
+	Val* pred = fval->Lookup(rtype->FieldOffset("pred"));
+	Val* path_func = fval->Lookup(rtype->FieldOffset("path_func"));
+
+	Filter* filter = new Filter;
+	filter->name = fval->Lookup(rtype->FieldOffset("name"))->AsString()->CheckString();
+	filter->pred = pred ? pred->AsFunc() : 0;
+	filter->pred = path_func ? path_func->AsFunc() : 0;
+	filter->writer = ld;
+
+	if ( event )
+		{
+		// TODO: Implement
+		filter->event = 0;
+		}
+
+	// Build the list of fields that the filter wants included, including
+	// potentially rolling out fields.
+	Val* include = fval->Lookup(rtype->FieldOffset("include"));
+	Val* exclude = fval->Lookup(rtype->FieldOffset("exclude"));
+
+	filter->num_fields = 0;
+	filter->fields = 0;
+	if ( ! TraverseRecord(filter, stream->columns, include ? include->AsTableVal() : 0, exclude ? exclude->AsTableVal() : 0, "", list<int>()) )
+		return false;
+
+	// Get the path for the filter.
+	Val* path_val = fval->Lookup(rtype->FieldOffset("path"));
+
+	if ( path_val )
+		filter->path = path_val->AsString()->CheckString();
+
+	else
+		{
+		// If no path is given, use the Stream ID as the default.
+		const char* n = stream->name.c_str();
+		char* lower = new char[strlen(n) + 1];
+		for ( char* s = lower; *n; ++n, ++s )
+			{
+			if ( strncmp(n, "::", 2) == 0 )
+				{
+				// Remove the scope operator. TODO: We need ab better way to
+				// generate the default here, but let's wait until we have
+				// everything in the right namespace.
+				*s = '_';
+				++n;
+				}
+
+			else
+				*s = tolower(*n);
+			}
+
+		filter->path = string(lower);
+		free(lower);
+		}
+
+	stream->filters.push_back(filter);
+
+#ifdef DEBUG
+	DBG_LOG(DBG_LOGGING, "Created new filter '%s' for stream '%s'", filter->name.c_str(), stream->name.c_str());
+	DBG_LOG(DBG_LOGGING, "   writer    : %s", ld->name);
+	DBG_LOG(DBG_LOGGING, "   path      : %s", filter->path.c_str());
+	DBG_LOG(DBG_LOGGING, "   path_func : %s", (filter->path_func ? "set" : "not set"));
+	DBG_LOG(DBG_LOGGING, "   event     : %s", (filter->event ? "set" : "not set"));
+	DBG_LOG(DBG_LOGGING, "   pred      : %s", (filter->pred ? "set" : "not set"));
+
+	for ( int i = 0; i < filter->num_fields; i++ )
+		{
+		LogField* field = filter->fields[i];
+		DBG_LOG(DBG_LOGGING, "   field %10s: %s", field->name.c_str(), type_name(field->type));
+		}
+#endif
+
+	return true;
+	}
+
+bool LogMgr::RemoveFilter(EnumVal* stream_id, StringVal* filter)
+	{
+#if 0
+	int idx = stream_id->AsEnum();
+
+	if ( idx >= streams.size() || ! streams[idx] )
+		{
+		run_time("unknown log stream");
+		return false;
+		}
+
+
+#endif
+	return true;
+	}
+
+bool LogMgr::Write(EnumVal* stream_id, RecordVal* columns)
+	{
+	unsigned int idx = stream_id->AsEnum();
+
+	if ( idx >= streams.size() || ! streams[idx] )
+		{
+		run_time("unknown log stream");
+		return false;
+		}
+
+	Stream* stream = streams[idx];
+
+	columns = columns->CoerceTo(stream->columns);
+
+	if ( ! columns )
+		{
+		run_time("imcompatible log record type");
+		return false;
+		}
+
+	// Send to each of our filters.
+	for ( list<Filter*>::iterator i = stream->filters.begin(); i != stream->filters.end(); ++i )
+		{
+		Filter* filter = *i;
+
+		string path = filter->path;
+
+		if ( filter->event )
+			{
+			// XXX Raise event here.
+			// TODO: Actually, the filter should be an attribute of the stream, right?
+			}
+
+		if ( filter->pred )
+			{
+			// XXX Check predicate here.
+			}
+
+		if ( filter->path_func )
+			{
+			// XXX Do dynamic path here.
+			}
+
+		// See if we already have a writer for this path.
+		Filter::WriterMap::iterator w = filter->writers.find(path);
+
+		LogWriter* writer = 0;
+		if ( w == filter->writers.end() )
+			{
+			// No, need to create one.
+			assert(filter->writer->factory);
+			writer = (*filter->writer->factory)();
+
+			// Copy the fields for LogWriter::Init() as it will take
+			// ownership.
+			LogField** arg_fields = new LogField*[filter->num_fields];
+			for ( int j = 0; j < filter->num_fields; ++j )
+				arg_fields[j] = new LogField(*filter->fields[j]);
+
+			if ( ! writer->Init(path, filter->num_fields, arg_fields) )
+				{
+				Unref(columns);
+				return false;
+				}
+
+			filter->writers.insert(Filter::WriterMap::value_type(path, writer));
+			}
+
+		else
+			// We have a writer already.
+			writer = w->second;
+
+		// Alright, can do the write now.
+		LogVal** vals = RecordToFilterVals(filter, columns);
+		writer->Write(vals);
+
+#ifdef DEBUG
+		DBG_LOG(DBG_LOGGING, "Wrote record to filter '%s' on stream '%s'", filter->name.c_str(), stream->name.c_str());
+#endif
+		}
+
+	Unref(columns);
+	return true;
+	}
+
+LogVal** LogMgr::RecordToFilterVals(Filter* filter, RecordVal* columns)
+	{
+	LogVal** vals = new LogVal*[filter->num_fields];
+
+	for ( int i = 0; i < filter->num_fields; ++i )
+		{
+		Val* val = columns;
+
+		// For each field, first find the right value, which can potentially
+		// be nested inside other records.
+		list<int>& indices = filter->indices[i];
+
+		for ( list<int>::iterator j = indices.begin(); j != indices.end(); ++j )
+			{
+			val = val->AsRecordVal()->Lookup(*j);
+
+			if ( ! val )
+				{
+				// Value, or any of its parents, is not set.
+				vals[i] = new LogVal(false);
+				break;
+				}
+			}
+
+		if ( ! val )
+			continue;
+
+		switch ( val->Type()->Tag() ) {
+		case TYPE_BOOL:
+		case TYPE_INT:
+		case TYPE_ENUM:
+			vals[i] = new LogVal();
+			vals[i]->val.int_val = val->InternalInt();
+			break;
+
+		case TYPE_COUNT:
+		case TYPE_COUNTER:
+		case TYPE_PORT:
+			vals[i] = new LogVal();
+			vals[i]->val.uint_val = val->InternalUnsigned();
+			break;
+
+		case TYPE_SUBNET:
+			vals[i] = new LogVal();
+			vals[i]->val.subnet_val = *val->AsSubNet();
+			break;
+
+		case TYPE_NET:
+		case TYPE_ADDR:
+			{
+			vals[i] = new LogVal();
+			addr_type t = val->AsAddr();
+			copy_addr(&t, &vals[i]->val.addr_val);
+			break;
+			}
+
+		case TYPE_DOUBLE:
+		case TYPE_TIME:
+		case TYPE_INTERVAL:
+			vals[i] = new LogVal();
+			vals[i]->val.double_val = val->InternalDouble();
+			break;
+
+		case TYPE_STRING:
+			{
+			const BroString* s = val->AsString();
+			LogVal* lval = (LogVal*) new char[sizeof(LogVal) + sizeof(log_string_type) + s->Len()];
+			new (lval) LogVal(); // Run ctor.
+			lval->val.string_val.len = s->Len();
+			memcpy(&lval->val.string_val.string, s->Bytes(), s->Len());
+			vals[i] = lval;
+			break;
+			}
+
+		default:
+			internal_error("unsupported type for log_write");
+		}
+		}
+
+	return vals;
+	}
+
+
+void LogMgr::Error(LogWriter* writer, const char* msg)
+	{
+#if 0
+#endif
+	}

src/LogMgr.h

@@ -0,0 +1,82 @@
+//
+// A class managing log writers and filters.
+
+#ifndef LOGMGR_H
+#define LOGMGR_H
+
+#include "Val.h"
+
+// One value per writer type we have.
+namespace LogWriterType {
+	enum Type {
+		None,
+		Ascii
+	};
+};
+
+struct LogField {
+	LogField()	{ }
+	LogField(const LogField& other) : name(other.name), type(other.type)	{ }
+	string name;
+	TypeTag type;
+};
+
+// A string that we can directly include as part of the value union below.
+struct log_string_type {
+	int len;
+	char string[]; // The string starts right here.
+};
+
+// All values that can be directly logged by a Writer.
+struct LogVal {
+	LogVal(bool arg_present = true) : present(arg_present)	{}
+
+	bool present; // If false, the field is unset (i.e., &optional and not initialzed).
+
+	// The following union is a subset of BroValUnion, including only the
+	// atomic types.
+ 	union {
+		bro_int_t int_val;
+		bro_uint_t uint_val;
+		addr_type addr_val;
+		subnet_type subnet_val;
+    	double double_val;
+		log_string_type string_val;
+	} val;
+};
+
+class LogWriter;
+
+class LogMgr {
+public:
+    LogMgr();
+    ~LogMgr();
+
+    // These correspond to the BiFs visible on the scripting layer. The
+    // actual BiFs just forward here.
+    bool CreateStream(EnumVal* stream_id, RecordType* columns);
+    bool AddFilter(EnumVal* stream_id, RecordVal* filter);
+    bool RemoveFilter(EnumVal* stream_id, StringVal* filter);
+    bool Write(EnumVal* stream_id, RecordVal* columns);
+
+protected:
+    friend class LogWriter;
+
+	/// Functions also used by the writers.
+
+    // Reports an error for the given writer.
+    void Error(LogWriter* writer, const char* msg);
+
+private:
+	struct Filter;
+	struct Stream;
+
+	bool TraverseRecord(Filter* filter, RecordType* rt, TableVal* include, TableVal* exclude, string path, list<int> indices);
+	LogVal** RecordToFilterVals(Filter* filter, RecordVal* columns);
+
+	vector<Stream *> streams; // Indexed by stream enum.
+};
+
+extern LogMgr* log_mgr;
+
+#endif

src/LogWriter.cc

@@ -0,0 +1,76 @@
+
+#include "util.h"
+#include "LogWriter.h"
+
+LogWriter::LogWriter()
+	{
+	buf = 0;
+	buf_len = 1024;
+	}
+
+LogWriter::~LogWriter()
+	{
+	if ( buf )
+		free(buf);
+
+	delete [] fields;
+	}
+
+bool LogWriter::Init(string arg_path, int arg_num_fields, LogField** arg_fields)
+	{
+	path = arg_path;
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+	DoInit(arg_path, arg_num_fields, arg_fields);
+	return true;
+	}
+
+bool LogWriter::Write(LogVal** vals)
+	{
+	bool result = DoWrite(num_fields, fields, vals);
+	DeleteVals(vals);
+	return result;
+	}
+
+void LogWriter::Finish()
+	{
+	DoFinish();
+	}
+
+const char* LogWriter::Fmt(const char* format, ...)
+	{
+	if ( ! buf )
+		buf = (char*) malloc(buf_len);
+
+	va_list al;
+	va_start(al, format);
+	int n = safe_vsnprintf(buf, buf_len, format, al);
+	va_end(al);
+
+	if ( (unsigned int) n >= buf_len )
+		{ // Not enough room, grow the buffer.
+		buf_len = n + 32;
+		buf = (char*) realloc(buf, buf_len);
+
+		// Is it portable to restart?
+		va_start(al, format);
+		n = safe_vsnprintf(buf, buf_len, format, al);
+		va_end(al);
+		}
+
+	return buf;
+	}
+
+
+void LogWriter::Error(const char *msg)
+	{
+	run_time(msg);
+	}
+
+void LogWriter::DeleteVals(LogVal** vals)
+	{
+	for ( int i = 0; i < num_fields; i++ )
+		delete vals[i];
+	}
+
+

src/LogWriter.h

@@ -0,0 +1,82 @@
+//
+// Interface API for a log writer backend.
+//
+// Note than classes derived from LogWriter must be fully thread-safe and not
+// use any non-safe Bro functionality (which is almost all ...). In
+// particular, do not use fmt() but LogWriter::Fmt()!.
+
+#ifndef LOGWRITER_H
+#define LOGWRITER_H
+
+#include "LogMgr.h"
+#include "BroString.h"
+
+class LogWriter {
+public:
+	LogWriter();
+    virtual ~LogWriter();
+
+	// One-time initialization of the writer, defining the logged fields.
+	// Interpretation of "path" is left to the writer, and will be the value
+	// configured on the script-level.  Returns false if an error occured, in
+	// which case the writer must not be used futher. 
+	//
+    // The new instance takes ownership of "fields", and will delete them
+    // when done.
+    bool Init(string path, int num_fields, LogField** fields);
+
+    // Writes one log entry. The method takes ownership of "vals" and will
+    // return immediately after queueing the write request, potentially
+    // before the output has actually taken place. Returns false if an error
+    // occured, in which case the writer must not be used further. 
+    bool Write(LogVal** vals);
+
+	// Finished writing to this logger. Will not be called if an error has
+	// been indicated earlier. After calling this, no more writing must be
+	// performed.
+	void Finish();
+
+protected:
+
+    //// Methods for Writers to override.
+
+    // Called once for initialization of the Writer. Must return false if an
+    // error occured, in which case the writer will be disabled. The error
+    // reason should be reported via Error().
+    virtual bool DoInit(string path, int num_fields, LogField** fields) = 0;
+
+    // Called once per entry to record. Must return false if an error
+    // occured, in which case the writer will be disabled. The error reason
+    // should be reported via Error().
+    virtual bool DoWrite(int num_fields, LogField** fields, LogVal** vals) = 0;
+
+	// Called once on termination. Not called when any of the other methods
+	// has previously signaled an error, i.e., executing this method signals
+	// a regular shutdown.
+    virtual void DoFinish() = 0;
+
+    //// Methods for Writers to use. These are thread-safe.
+
+	// A thread-safe version of fmt().
+	const char* Fmt(const char* format, ...);
+
+    // Reports an error.
+    void Error(const char *msg);
+
+	// Returns the path as passed to Init().
+	const string Path() const	{ return path; }
+
+private:
+    // Delete values as passed into Write().
+    void DeleteVals(LogVal** vals);
+
+    string path;
+    int num_fields;
+    LogField** fields;
+
+	// For Fmt().
+	char* buf;
+	unsigned int buf_len;
+};
+
+#endif

src/LogWriterAscii.cc

@@ -0,0 +1,129 @@
+
+#include <string>
+#include <errno.h>
+
+#include "LogWriterAscii.h"
+
+LogWriterAscii::LogWriterAscii()
+	{
+	fname = 0;
+	file = 0;
+	}
+
+LogWriterAscii::~LogWriterAscii()
+	{
+	if ( fname )
+		free(fname);
+
+	if ( file )
+		fclose(file);
+	}
+
+bool LogWriterAscii::DoInit(string path, int num_fields, LogField** fields)
+	{
+	fname = strdup(Fmt("%s.log", path.c_str()));
+
+	if ( ! (file = fopen(fname, "w")) )
+		{
+		Error(Fmt("cannot open %s: %s", fname, strerror(errno)));
+		return false;
+		}
+
+	if ( fputs("# ", file) == EOF )
+		goto write_error;
+
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		LogField* field = fields[i];
+		if ( fputs(field->name.c_str(), file) == EOF )
+			goto write_error;
+
+		if ( fputc('\t', file) == EOF )
+			goto write_error;
+		}
+
+	if ( fputc('\n', file) == EOF )
+		goto write_error;
+
+	return true;
+
+write_error:
+	Error(Fmt("error writing to %s: %s", fname, strerror(errno)));
+	return false;
+	}
+
+void LogWriterAscii::DoFinish()
+	{
+	}
+
+bool LogWriterAscii::DoWrite(int num_fields, LogField** fields, LogVal** vals)
+	{
+	ODesc desc(DESC_READABLE);
+
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( i > 0 )
+			desc.Add("\t");
+
+		LogVal* val = vals[i];
+		LogField* field = fields[i];
+
+		if ( ! val->present )
+			{
+			desc.Add("-"); // TODO: Probably want to get rid of the "-".
+			continue;
+			}
+
+		switch ( field->type ) {
+		case TYPE_BOOL:
+			desc.Add(val->val.int_val ? "T" : "F");
+			break;
+
+		case TYPE_INT:
+		case TYPE_ENUM:
+			desc.Add(val->val.int_val);
+			break;
+
+		case TYPE_COUNT:
+		case TYPE_COUNTER:
+		case TYPE_PORT:
+			desc.Add(val->val.uint_val);
+			break;
+
+		case TYPE_SUBNET:
+			desc.Add(dotted_addr(val->val.subnet_val.net));
+			desc.Add("/");
+			desc.Add(val->val.subnet_val.width);
+			break;
+
+		case TYPE_NET:
+		case TYPE_ADDR:
+			desc.Add(dotted_addr(val->val.addr_val));
+			break;
+
+		case TYPE_DOUBLE:
+		case TYPE_TIME:
+		case TYPE_INTERVAL:
+			desc.Add(val->val.double_val);
+			break;
+
+		case TYPE_STRING:
+			desc.AddN((const char*)&val->val.string_val.string, val->val.string_val.len);
+			break;
+
+		default:
+			Error(Fmt("unsupported field format %d for %s", field->type, field->name.c_str()));
+			return false;
+		}
+		}
+
+	desc.Add("\n");
+
+	if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 )
+		{
+		Error(Fmt("error writing to %s: %s", fname, strerror(errno)));
+		return false;
+		}
+
+	return true;
+	}

src/LogWriterAscii.h

@@ -0,0 +1,27 @@
+//
+// Log writer for tab-separated ASCII logs. 
+//
+
+#ifndef LOGWRITERASCII_H
+#define LOGWRITERASCII_H
+
+#include "LogWriter.h"
+
+class LogWriterAscii : public LogWriter {
+public:
+	LogWriterAscii();
+	~LogWriterAscii();
+
+	static LogWriter* Instantiate()	{ return new LogWriterAscii; }
+
+protected:
+    bool DoInit(string path, int num_fields, LogField** fields);
+    bool DoWrite(int num_fields, LogField** fields, LogVal** vals);
+    void DoFinish();
+
+private:
+	FILE* file;
+	char* fname;
+};
+
+#endif

src/NetVar.cc

@@ -260,6 +260,8 @@ int record_all_packets;
 RecordType* script_id;
 TableType* id_table;
 
+RecordType* log_filter;
+
 #include "const.bif.netvar_def"
 #include "event.bif.netvar_def"
 
@@ -564,4 +566,6 @@ void init_net_var()
 
 	script_id = internal_type("script_id")->AsRecordType();
 	id_table = internal_type("id_table")->AsTableType();
+
+	log_filter = internal_type("log_filter")->AsRecordType();
 	}

src/NetVar.h

@@ -264,6 +264,8 @@ extern int record_all_packets;
 extern RecordType* script_id;
 extern TableType* id_table;
 
+extern RecordType* log_filter;
+
 // Initializes globals that don't pertain to network/event analysis.
 extern void init_general_global_var();
 

src/bro.bif

@@ -483,6 +483,47 @@ function logging_log%(index: string, rec: any%): any
 	return 0;
 	%}
 
+
+%%{
+#include "LogMgr.h"
+%%}
+
+function log_create_stream%(id: Log_ID, columns: any%) : bool
+    %{
+    if ( columns->Type()->Tag() != TYPE_TYPE )
+	    {
+		run_time("log columns must be a type");
+    	return new Val(0, TYPE_BOOL);
+		}
+
+    if ( columns->Type()->AsTypeType()->Type()->Tag() != TYPE_RECORD )
+	    {
+		run_time("log columns must be a record type");
+    	return new Val(0, TYPE_BOOL);
+		}
+
+	bool result = log_mgr->CreateStream(id->AsEnumVal(), columns->Type()->AsTypeType()->Type()->AsRecordType());
+	return new Val(result, TYPE_BOOL);
+    %}
+
+function log_add_filter%(id: Log_ID, filter: log_filter%) : bool
+    %{
+	bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal());
+	return new Val(result, TYPE_BOOL);
+    %}
+
+function log_remove_filter%(id: Log_ID, name: string%) : bool
+    %{
+	bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name);
+	return new Val(result, TYPE_BOOL);
+    %}
+
+function log_write%(id: Log_ID, columns: any%) : bool
+    %{
+	bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal());
+	return new Val(result, TYPE_BOOL);
+    %}
+
 function record_type_to_vector%(rt: string%): string_vec
 	%{
 	VectorVal* result =

src/logging.bif

@@ -0,0 +1,23 @@
+
+enum log_writer %{
+    WRITER_DEFAULT,
+    WRITER_TSV,
+    WRITER_SYSLOG,
+}%
+
+void log_create%(id: Logging::Stream, columns: any%)
+    %{
+    %}
+
+void log_add_filter%(id: Logging::Stream, filter: any%)
+    %{
+    %}
+
+void log_remove_filter%(id: Logging::Stream, name: string%)
+    %{
+    %}
+
+void log_write%(id: Logging::Stream, columns: any%)
+    %{
+    %}
+

src/logging.bro

@@ -0,0 +1,52 @@
+
+module Log;
+
+export {
+	# Each stream gets a unique ID. This type will be extended by
+	# other scripts.
+	type Stream: enum {
+	    Unknown,
+		Info,
+		Debug,
+		};
+
+	# The default writer to use if a filter does not specify
+	# anything else.
+	const default_writer = WRITER_CSV &redef;
+
+	# A filter defining what to record.
+	type Filter: record {
+		# A name to reference this filter.
+		name: string;
+
+		# A predicate returning True if the filter wants a log entry
+		# to be recorded. If not given, an implicit True is assumed
+		# for all entries. The predicate receives one parameter:
+		# an instance of the log's record type with the fields to be
+		# logged.
+		pred: function(log: any) &optional;
+    	
+		# A path for outputting everything matching this
+		# filter. The path is either a string, or a function
+		# called with a single ``ID`` argument and returning a string.
+		#
+		# The specific interpretation of the string is left to the
+		# Writer, but if it's refering to a file, it's assumed that no
+		# extension is given; the writer will add whatever is
+		# appropiate.
+		path: any &optional;
+    	
+		# A subset of column names to record. If not given, all
+		# columns are recorded.
+		select: set[string] &optional;
+    	
+		# An event that is raised whenever the filter is applied
+		# to an entry. The event receives the same parameter
+		# as the predicate. It will always be generated,
+		# independent of what the predicate returns.
+		ev: event(l: any) &optional;
+    	
+		# The writer to use.
+		writer: Writer &default=default_writer;
+		};
+}

src/main.cc

@@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "Event.h"
 #include "File.h"
 #include "Logger.h"
+#include "LogMgr.h"
 #include "Net.h"
 #include "NetVar.h"
 #include "Var.h"
@@ -71,6 +72,7 @@ name_list prefixes;
 DNS_Mgr* dns_mgr;
 TimerMgr* timer_mgr;
 Logger* bro_logger;
+LogMgr* log_mgr;
 Func* alarm_hook = 0;
 Stmt* stmts;
 EventHandlerPtr bro_signal = 0;
@@ -289,6 +291,7 @@ void terminate_bro()
 	delete conn_compressor;
 	delete remote_serializer;
 	delete dpm;
+	delete log_mgr;
 	}
 
 void termination_signal()
@@ -724,7 +727,8 @@ int main(int argc, char** argv)
 
 	persistence_serializer = new PersistenceSerializer();
 	remote_serializer = new RemoteSerializer();
-	event_registry = new EventRegistry;
+	event_registry = new EventRegistry();
+	log_mgr = new LogMgr();
 
 	if ( events_file )
 		event_player = new EventPlayer(events_file);