Merge branch 'master' into topic/jsiwek/doc-framework

CHANGES

@@ -1,3 +1,55 @@
+1.6-dev.88 Wed Apr 20 20:43:48 PDT 2011
+
+- Implementation of Bro's new logging framework. We will document this
+  separately. (Robin Sommer)
+
+- Already defined record types can now be further extended via the
+  '+=' operator. The added fields must be either &optional or have a
+  &default value. (Robin Sommer)
+    
+  Example:
+    
+        type Foo: record {
+            a: count;
+            b: count &optional;
+        };
+    
+        redef record Foo += {
+            c: count &default=42;
+            d: count &optional;
+        };
+    
+        global f: Foo = [$a=21];
+    
+        print f;
+    
+  Output:
+   
+        [a=21, b=<uninitialized>, c=42, d=<uninitialized>]
+
+- Enabling assignment of empty vectors ("vector()"). (Robin Sommer)
+
+- Fixing attributes to allow &default attributes to be associated with
+  records fields of type tables/sets/vector. (Robin Sommer)
+    
+- '[]' is now a valid record constructor. (Robin Sommer)
+  
+- A instance of a record type A is now coercable into one of type B if
+  the fields of type A are a subset of those of type B. (Robin Sommer)
+
+- A number of bug fixes and enhancements for record/set/table/vector
+  coercion. (Robin Sommer)
+
+- Fixing a problem with records that have optional fields when used as
+  table/set indices. Addresses #367. (Robin Sommer)
+
+- Fixing an off-by-one error in join_string_vec(). (Seth Hall)
+  
+- Updating to_count() to cope with 64bit ints. (Seth Hall)
+  
+- A new BiF count_to_v4_addr() to turn a count into an IPv4 address.
+  (Seth Hall)
+
 1.6-dev.80 Mon Apr 18 14:50:54 PDT 2011
 
 - New framework for generating documentation from Bro scripts. (Jon

VERSION

@@ -1 +1 @@
-1.6-dev.80
+1.6-dev.88

aux/btest

@@ -1 +1 @@
-Subproject commit b29e2214cc959b6b6c841497b7881df369331561
+Subproject commit aee56f4632c9a4224727da1b535b673a4bcb5d3b

policy/bro.init

@@ -21,7 +21,7 @@ type conn_id: record {
 	orig_p: port;
 	resp_h: addr;
 	resp_p: port;
-};
+} &log;
 
 type icmp_conn: record {
 	orig_h: addr;
@@ -277,6 +277,9 @@ type entropy_test_result: record {
 @load strings.bif.bro
 @load bro.bif.bro
 
+@load logging  # sic! Not logging.bif.
+@load logging-ascii
+
 global bro_alarm_file: file &redef;
 global alarm_hook: function(msg: string): bool &redef;
 global log_file_name: function(tag: string): string &redef;

policy/conn.bro

@@ -14,7 +14,9 @@ const conn_closed = { TCP_CLOSED, TCP_RESET };
 
 global have_FTP = F;	# if true, we've loaded ftp.bro
 global have_SMTP = F;	# if true, we've loaded smtp.bro
-global is_ftp_data_conn: function(c: connection): bool;
+
+# TODO: Do we have a nicer way of defining this prototype?
+export { global FTP::is_ftp_data_conn: function(c: connection): bool; }
 
 # Whether to include connection state history in the logs generated
 # by record_connection.
@@ -186,7 +188,7 @@ function determine_service_non_DPD(c: connection) : string
 			return i;	# return first;
 		}
 
-	else if ( have_FTP && is_ftp_data_conn(c) )
+	else if ( have_FTP && FTP::is_ftp_data_conn(c) )
 		return port_names[20/tcp];
 
 	else if ( [c$id$resp_h, c$id$resp_p] in RPC_server_map )

policy/firewall.bro

@@ -151,7 +151,7 @@ function do_match(c: connection, r: rule): bool
 				return F;
 		}
 
-	if ( r$is_ftp && ! is_ftp_data_conn(c) )
+	if ( r$is_ftp && ! FTP::is_ftp_data_conn(c) )
 		return F;
 
 	return T;

policy/logging-ascii.bro

@@ -0,0 +1,29 @@
+##! Interface for the ascii log writer.
+
+module LogAscii;
+
+export {
+	## If true, output everything to stdout rather than
+	## into files. This is primarily for debugging purposes.
+	const output_to_stdout = F &redef;
+
+	## If true, include a header line with column names.
+	const include_header = T &redef;
+
+	# Prefix for the header line if included.
+	const header_prefix = "# " &redef;
+
+	## Separator between fields.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	const empty_field = "" &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = "-" &redef;
+}
+
+

policy/logging.bro

@@ -0,0 +1,209 @@
+##! The Bro logging interface.
+##!
+##! See XXX for a introduction to Bro's logging framework.
+
+module Log;
+
+# Log::ID and Log::Writer are defined in bro.init due to circular dependencies.
+
+export {
+	## If true, is local logging is by default enabled for all filters.
+	const enable_local_logging = T &redef;
+
+	## If true, is remote logging is by default enabled for all filters.
+	const enable_remote_logging = T &redef;
+
+	## Default writer to use if a filter does not specify
+	## anything else.
+	const default_writer = WRITER_ASCII &redef;
+
+	## Type defining the content of a logging stream.
+	type Stream: record {
+		## A record type defining the log's columns.
+		columns: any;
+
+		## Event that will be raised once for each log entry.
+		## The event receives a single same parameter, an instance of type ``columns``.
+		ev: any &optional;
+	};
+
+	## Filter customizing logging.
+	type Filter: record {
+		## Descriptive name to reference this filter.
+		name: string;
+
+		## The writer to use.
+		writer: Writer &default=default_writer;
+
+		## Predicate indicating whether a log entry should be recorded.
+		## If not given, all entries are recorded.
+		##
+		## rec: An instance of the streams's ``columns`` type with its
+		## fields set to the values to logged.
+		##
+		## Returns: True if the entry is to be recorded.
+		pred: function(rec: any): bool &optional;
+
+		## Output path for recording entries matching this
+		## filter.
+                ##
+		## The specific interpretation of the string is up to
+		## the used writer, and may for example be the destination
+		## file name. Generally, filenames are expected to given
+		## without any extensions; writers will add appropiate 
+		## extensions automatically.
+		path: string &optional;
+
+		## A function returning the output path for recording entries
+		## matching this filter. This is similar to ``path`` yet allows
+		## to compute the string dynamically. It is ok to return
+		## different strings for separate calls, but be careful: it's
+		## easy to flood the disk by returning a new string for each
+		## connection ...
+		path_func: function(id: ID, path: string): string &optional;
+
+		## Subset of column names to record. If not given, all
+		## columns are recorded.
+		include: set[string] &optional;
+
+		## Subset of column names to exclude from recording. If not given,
+		## all columns are recorded.
+		exclude: set[string] &optional;
+
+		## If true, entries are recorded locally.
+		log_local: bool &default=enable_local_logging;
+
+		## If true, entries are passed on to remote peers.
+		log_remote: bool &default=enable_remote_logging;
+	};
+
+	# Log rotation support.
+
+	## Information passed into rotation callback functions.
+	type RotationInfo: record {
+		writer: Writer;	##> Writer.
+		path: string;	##> Original path value.
+		open: time;	##> Time when opened.
+		close: time;	##> Time when closed.
+	};
+
+	## Default rotation interval. Zero disables rotation.
+	const default_rotation_interval = 0secs &redef;
+
+	## Default naming suffix format. Uses a strftime() style.
+	const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef;
+
+	## Default postprocessor for writers outputting into files.
+	const default_rotation_postprocessor = "" &redef;
+
+	## Default function to construct the name of a rotated output file.
+	## The default implementation appends info$date_fmt to the original
+	## file name.
+	##
+	## info: Meta-data about the file to be rotated.
+	global default_rotation_path_func: function(info: RotationInfo) : string &redef;
+
+	## Type for controlling file rotation.
+	type RotationControl: record  {
+		## Rotation interval.
+		interv: interval &default=default_rotation_interval;
+		## Format for timestamps embedded into rotated file names.
+		date_fmt: string &default=default_rotation_date_format;
+		## Postprocessor process to run on rotate file.
+		postprocessor: string &default=default_rotation_postprocessor;
+	};
+
+	## Specifies rotation parameters per ``(id, path)`` tuple.
+	## If a pair is not found in this table, default values defined in
+	## ``RotationControl`` are used.
+	const rotation_control: table[Writer, string] of RotationControl &default=[] &redef;
+
+	## Sentinel value for indicating that a filter was not found when looked up.
+	const no_filter: Filter = [$name="<not found>"]; # Sentinel.
+
+	# TODO: Document.
+	global create_stream: function(id: ID, stream: Stream) : bool;
+	global enable_stream: function(id: ID) : bool;
+	global disable_stream: function(id: ID) : bool;
+	global add_filter: function(id: ID, filter: Filter) : bool;
+	global remove_filter: function(id: ID, name: string) : bool;
+	global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found.
+	global write: function(id: ID, columns: any) : bool;
+	global set_buf: function(id: ID, buffered: bool): bool;
+	global flush: function(id: ID): bool;
+	global add_default_filter: function(id: ID) : bool;
+	global remove_default_filter: function(id: ID) : bool;
+}
+
+# We keep a script-level copy of all filters so that we can manipulate them.
+global filters: table[ID, string] of Filter;
+
+@load logging.bif # Needs Filter and Stream defined.
+
+module Log;
+
+function default_rotation_path_func(info: RotationInfo) : string
+	{
+	local date_fmt = rotation_control[info$writer, info$path]$date_fmt;
+	return fmt("%s-%s", info$path, strftime(date_fmt, info$open));
+	}
+
+function create_stream(id: ID, stream: Stream) : bool
+	{
+	if ( ! __create_stream(id, stream) )
+		return F;
+
+	return add_default_filter(id);
+	}
+
+function disable_stream(id: ID) : bool
+	{
+	if ( ! __disable_stream(id) )
+		return F;
+	}
+						   
+function add_filter(id: ID, filter: Filter) : bool
+	{
+	filters[id, filter$name] = filter;
+	return __add_filter(id, filter);
+	}
+
+function remove_filter(id: ID, name: string) : bool
+	{
+	delete filters[id, name];
+	return __remove_filter(id, name);
+	}
+
+function get_filter(id: ID, name: string) : Filter
+	{
+	if ( [id, name] in filters )
+		return filters[id, name];
+
+	return no_filter;
+	}
+
+function write(id: ID, columns: any) : bool
+	{
+	return __write(id, columns);
+	}
+
+function set_buf(id: ID, buffered: bool): bool
+	{
+	return __set_buf(id, buffered);
+	}
+
+function flush(id: ID): bool
+	{
+	return __flush(id);
+	}
+
+function add_default_filter(id: ID) : bool
+	{
+	return add_filter(id, [$name="default"]);
+	}
+
+function remove_default_filter(id: ID) : bool
+	{
+	return remove_filter(id, "default");
+	}
+

policy/mime.bro

@@ -74,13 +74,14 @@ function mime_header_subject(session: mime_session_info,
 ### This is a bit clunky.  These are functions we call out to, defined
 # elsewhere.  The way we really ought to do this is to have them passed
 # in during initialization.  But for now, we presume knowledge of their
-# names in global scope.
+# names.
-module GLOBAL;
+export
-global check_relay_3:
+	{
-	function(session: MIME::mime_session_info, msg_id: string);
+	global SMTP::check_relay_3:
-global check_relay_4:
+		function(session: MIME::mime_session_info, msg_id: string);
-	function(session: MIME::mime_session_info, content_hash: string);
+	global SMTP::check_relay_4:
-module MIME;
+		function(session: MIME::mime_session_info, content_hash: string);
+	}
 
 function mime_header_message_id(session: mime_session_info, name: string, arg: string)
 	{
@@ -107,7 +108,7 @@ function mime_header_message_id(session: mime_session_info, name: string, arg: s
 	s = t[1];
 
 	if ( session$level == 1 && SMTP::process_smtp_relay )
-		check_relay_3(session, s);
+		SMTP::check_relay_3(session, s);
 	}
 
 redef mime_header_handler = {

policy/remote.bro

@@ -43,6 +43,9 @@ export {
 		# Whether to perform state synchronization with peer.
 		sync: bool &default = T;
 
+		# Whether to request logs from the peer.
+		request_logs: bool &default = F;
+
 		# When performing state synchronization, whether we consider
 		# our state to be authoritative.  If so, we will send the peer
 		# our current set when the connection is set up.
@@ -176,6 +179,12 @@ function setup_peer(p: event_peer, dst: Destination)
 		request_remote_sync(p, dst$auth);
 		}
 
+	if ( dst$request_logs )
+		{
+		do_script_log(p, "requesting logs");
+		request_remote_logs(p);
+		}
+
 	dst$peer = p;
 	dst$connected = T;
 	connected_peers[p$id] = dst;

src/Attr.cc

@@ -7,6 +7,7 @@
 #include "Attr.h"
 #include "Expr.h"
 #include "Serializer.h"
+#include "LogMgr.h"
 
 const char* attr_name(attr_tag t)
 	{
@@ -88,10 +89,11 @@ void Attr::AddTag(ODesc* d) const
 		d->Add(attr_name(Tag()));
 	}
 
-Attributes::Attributes(attr_list* a, BroType* t)
+Attributes::Attributes(attr_list* a, BroType* t, bool arg_in_record)
 	{
 	attrs = new attr_list(a->length());
 	type = t->Ref();
+	in_record = arg_in_record;
 
 	SetLocationInfo(&start_location, &end_location);
 
@@ -241,27 +243,72 @@ void Attributes::CheckAttr(Attr* a)
 		{
 		BroType* atype = a->AttrExpr()->Type();
 
-		if ( type->Tag() != TYPE_TABLE || type->IsSet() )
+		if ( type->Tag() != TYPE_TABLE || (type->IsSet() && ! in_record) )
 			{
-			if ( ! same_type(atype, type) )
+			if ( same_type(atype, type) )
-				a->AttrExpr()->Error("&default value has inconsistent type", type);
+				// Ok.
-			break;
+				break;
+
+			// Record defaults may be promotable.
+			if ( (type->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+			      record_promotion_compatible(atype->AsRecordType(),
+							  type->AsRecordType())) )
+				// Ok.
+				break;
+
+			a->AttrExpr()->Error("&default value has inconsistent type", type);
 			}
 
 		TableType* tt = type->AsTableType();
+		BroType* ytype = tt->YieldType();
 
-		if ( ! same_type(atype, tt->YieldType()) )
+		if ( ! in_record )
 			{
-			// It can still be a default function.
+			// &default applies to the type itself.
-			if ( atype->Tag() == TYPE_FUNC )
+			if ( ! same_type(atype, ytype) )
 				{
-				FuncType* f = atype->AsFuncType();
+				// It can still be a default function.
-				if ( ! f->CheckArgs(tt->IndexTypes()) ||
+				if ( atype->Tag() == TYPE_FUNC )
-				     ! same_type(f->YieldType(), tt->YieldType()) )
+					{
-					Error("&default function type clash");
+					FuncType* f = atype->AsFuncType();
+					if ( ! f->CheckArgs(tt->IndexTypes()) ||
+					     ! same_type(f->YieldType(), ytype) )
+						Error("&default function type clash");
+
+					// Ok.
+					break;
+					}
+
+				// Table defaults may be promotable.
+				if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+				      record_promotion_compatible(atype->AsRecordType(),
+								  ytype->AsRecordType())) )
+					// Ok.
+					break;
+
+				Error("&default value has inconsistent type 2");
 				}
-			else
+
-				Error("&default value has inconsistent type");
+			// Ok.
+			break;
+			}
+
+		else
+			{
+			// &default applies to record field.
+
+			if ( same_type(atype, type) ||
+			     (atype->Tag() == TYPE_TABLE && atype->AsTableType()->IsUnspecifiedTable()) )
+				// Ok.
+				break;
+
+			// Table defaults may be promotable.
+			if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+			      record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) )
+				// Ok.
+				break;
+
+			Error("&default value has inconsistent type");
 			}
 		}
 		break;
@@ -358,10 +405,12 @@ void Attributes::CheckAttr(Attr* a)
 	case ATTR_GROUP:
 		if ( type->Tag() != TYPE_FUNC ||
 		     ! type->AsFuncType()->IsEvent() )
-			{
 			Error("&group only applicable to events");
-			break;
+		break;
-			}
+
+	case ATTR_LOG:
+		if ( ! LogVal::IsCompatibleType(type) )
+			Error("&log applied to a type that cannot be logged");
 		break;
 
 	default:
@@ -369,6 +418,41 @@ void Attributes::CheckAttr(Attr* a)
 	}
 	}
 
+bool Attributes::operator==(const Attributes& other) const
+	{
+	if ( ! attrs )
+		return other.attrs;
+
+	if ( ! other.attrs )
+		return false;
+
+	loop_over_list(*attrs, i)
+		{
+		Attr* a = (*attrs)[i];
+		Attr* o = other.FindAttr(a->Tag());
+
+		if ( ! o )
+			return false;
+
+		if ( ! (*a == *o) )
+			return false;
+		}
+
+	loop_over_list(*other.attrs, j)
+		{
+		Attr* o = (*other.attrs)[j];
+		Attr* a = FindAttr(o->Tag());
+
+		if ( ! a )
+			return false;
+
+		if ( ! (*a == *o) )
+			return false;
+		}
+
+	return true;
+	}
+
 bool Attributes::Serialize(SerialInfo* info) const
 	{
 	return SerialObj::Serialize(info);

src/Attr.h

@@ -35,6 +35,7 @@ typedef enum {
 	ATTR_MERGEABLE,
 	ATTR_PRIORITY,
 	ATTR_GROUP,
+	ATTR_LOG,
 	ATTR_TRACKED,	// hidden attribute, tracked by NotifierRegistry
 #define NUM_ATTRS (int(ATTR_TRACKED) + 1)
 } attr_tag;
@@ -53,6 +54,20 @@ public:
 	void Describe(ODesc* d) const;
 	void DescribeReST(ODesc* d) const;
 
+	bool operator==(const Attr& other) const
+		{
+		if ( tag != other.tag )
+			return false;
+
+		if ( expr || other.expr )
+			// If any has an expression and they aren't the same object, we
+			// declare them unequal, as we can't really find out if the two
+			// expressions are equivalent.
+			return (expr == other.expr);
+
+		return true;
+		}
+
 protected:
 	void AddTag(ODesc* d) const;
 
@@ -63,7 +78,7 @@ protected:
 // Manages a collection of attributes.
 class Attributes : public BroObj {
 public:
-	Attributes(attr_list* a, BroType* t);
+	Attributes(attr_list* a, BroType* t, bool in_record);
 	~Attributes();
 
 	void AddAttr(Attr* a);
@@ -81,6 +96,8 @@ public:
 	bool Serialize(SerialInfo* info) const;
 	static Attributes* Unserialize(UnserialInfo* info);
 
+	bool operator==(const Attributes& other) const;
+
 protected:
 	Attributes()	{ type = 0; attrs = 0; }
 	void CheckAttr(Attr* attr);
@@ -89,6 +106,7 @@ protected:
 
 	BroType* type;
 	attr_list* attrs;
+	bool in_record;
 };
 
 #endif

src/CMakeLists.txt

@@ -132,6 +132,7 @@ endmacro(GET_BIF_OUTPUT_FILES)
 
 set(BIF_SRCS
     bro.bif
+    logging.bif
     event.bif
     const.bif
     types.bif
@@ -338,6 +339,9 @@ set(bro_SRCS
     IRC.cc
     List.cc
     Logger.cc
+    LogMgr.cc
+    LogWriter.cc
+    LogWriterAscii.cc
     Login.cc
     MIME.cc
     NCP.cc

src/CompHash.cc

@@ -65,11 +65,22 @@ CompositeHash::~CompositeHash()
 
 // Computes the piece of the hash for Val*, returning the new kp.
 char* CompositeHash::SingleValHash(int type_check, char* kp0,
-					BroType* bt, Val* v) const
+				   BroType* bt, Val* v, bool optional) const
 	{
 	char* kp1 = 0;
 	InternalTypeTag t = bt->InternalType();
 
+	if ( optional )
+		{
+		// Add a marker saying whether the optional field is set.
+		char* kp = AlignAndPadType<char>(kp0);
+		*kp = ( v ? 1 : 0);
+		kp0 = reinterpret_cast<char*>(kp+1);
+
+		if ( ! v ) 
+			return kp0;
+		}
+
 	if ( type_check )
 		{
 		InternalTypeTag vt = v->Type()->InternalType();
@@ -163,12 +174,16 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0,
 			for ( int i = 0; i < num_fields; ++i )
 				{
 				Val* rv_i = rv->Lookup(i);
-				if ( ! rv_i )
+
+				Attributes* a = rt->FieldDecl(i)->attrs;
+				bool optional = (a && a->FindAttr(ATTR_OPTIONAL));
+
+				if ( ! (rv_i || optional) )
 					return 0;
 
 				if ( ! (kp = SingleValHash(type_check, kp,
 							   rt->FieldType(i),
-							   rv_i)) )
+							   rv_i, optional)) )
 					return 0;
 				}
 
@@ -248,7 +263,7 @@ HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const
 	char* kp = k;
 	loop_over_list(*tl, i)
 		{
-		kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i]);
+		kp = SingleValHash(type_check, kp, (*tl)[i], (*vl)[i], false);
 		if ( ! kp )
 			return 0;
 		}
@@ -315,10 +330,13 @@ HashKey* CompositeHash::ComputeSingletonHash(const Val* v, int type_check) const
 	}
 
 int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
-					int type_check, int sz) const
+				     int type_check, int sz, bool optional) const
 	{
 	InternalTypeTag t = bt->InternalType();
 
+	if ( optional )
+		sz = SizeAlign(sz, sizeof(char));
+
 	if ( type_check && v )
 		{
 		InternalTypeTag vt = v->Type()->InternalType();
@@ -369,9 +387,12 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
 
 			for ( int i = 0; i < num_fields; ++i )
 				{
+				Attributes* a = rt->FieldDecl(i)->attrs;
+				bool optional = (a && a->FindAttr(ATTR_OPTIONAL));
+
 				sz = SingleTypeKeySize(rt->FieldType(i),
-							rv ? rv->Lookup(i) : 0,
+						       rv ? rv->Lookup(i) : 0,
-							type_check, sz);
+						       type_check, sz, optional);
 				if ( ! sz )
 					return 0;
 				}
@@ -386,7 +407,7 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
 
 	case TYPE_INTERNAL_STRING:
 		if ( ! v )
-			return 0;
+			return optional ? sz : 0;
 
 		// Factor in length field.
 		sz = SizeAlign(sz, sizeof(int));
@@ -418,7 +439,7 @@ int CompositeHash::ComputeKeySize(const Val* v, int type_check) const
 	loop_over_list(*tl, i)
 		{
 		sz = SingleTypeKeySize((*tl)[i], v ? v->AsListVal()->Index(i) : 0,
-				       type_check, sz);
+				       type_check, sz, false);
 		if ( ! sz )
 			return 0;
 		}
@@ -495,20 +516,20 @@ ListVal* CompositeHash::RecoverVals(const HashKey* k) const
 	loop_over_list(*tl, i)
 		{
 		Val* v;
-		kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v);
+		kp = RecoverOneVal(k, kp, k_end, (*tl)[i], v, false);
 		ASSERT(v);
 		l->Append(v);
 		}
 
 	if ( kp != k_end )
-		internal_error("under-ran key in CompositeHash::DescribeKey");
+		internal_error("under-ran key in CompositeHash::DescribeKey %ld", k_end - kp);
 
 	return l;
 	}
 
 const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 					 const char* const k_end, BroType* t,
-					 Val*& pval) const
+					 Val*& pval, bool optional) const
 	{
 	// k->Size() == 0 for a single empty string.
 	if ( kp0 >= k_end && k->Size() > 0 )
@@ -516,9 +537,20 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 
 	TypeTag tag = t->Tag();
 	InternalTypeTag it = t->InternalType();
-
 	const char* kp1 = 0;
 
+	if ( optional )
+		{
+		const char* kp = AlignType<char>(kp0);
+		kp0 = kp1 = reinterpret_cast<const char*>(kp+1);
+
+		if ( ! *kp )
+			{
+			pval = 0;
+			return kp0;
+			}
+		}
+
 	switch ( it ) {
 	case TYPE_INTERNAL_INT:
 		{
@@ -647,9 +679,13 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 			for ( i = 0; i < num_fields; ++i )
 				{
 				Val* v;
+
+				Attributes* a = rt->FieldDecl(i)->attrs;
+				bool optional = (a && a->FindAttr(ATTR_OPTIONAL));
+
 				kp = RecoverOneVal(k, kp, k_end,
-				                   rt->FieldType(i), v);
+				                   rt->FieldType(i), v, optional);
-				if ( ! v )
+				if ( ! (v || optional) )
 					{
 					internal_error("didn't recover expected number of fields from HashKey");
 					pval = 0;

src/CompHash.h

@@ -29,8 +29,8 @@ protected:
 
 	// Computes the piece of the hash for Val*, returning the new kp.
 	// Used as a helper for ComputeHash in the non-singleton case.
-	char* SingleValHash(int type_check, char* kp,
+	char* SingleValHash(int type_check, char* kp, BroType* bt, Val* v,
-				BroType* bt, Val* v) const;
+			    bool optional) const;
 
 	// Recovers just one Val of possibly many; called from RecoverVals.
 	// Upon return, pval will point to the recovered Val of type t.
@@ -38,7 +38,7 @@ protected:
 	// upon errors, so there is no return value for invalid input.
 	const char* RecoverOneVal(const HashKey* k,
 				  const char* kp, const char* const k_end,
-				  BroType* t, Val*& pval) const;
+				  BroType* t, Val*& pval, bool optional) const;
 
 	// Rounds the given pointer up to the nearest multiple of the
 	// given size, if not already a multiple.
@@ -77,7 +77,7 @@ protected:
 	int ComputeKeySize(const Val* v = 0, int type_check = 1) const;
 
 	int SingleTypeKeySize(BroType*, const Val*,
-				int type_check, int sz) const;
+			      int type_check, int sz, bool optional) const;
 
 	TypeList* type;
 	char* key;	// space for composite key

src/DebugLogger.cc

@@ -17,6 +17,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "compressor", 0, false }, {"string", 0, false },
 	{ "notifiers", 0, false },  { "main-loop", 0, false },
 	{ "dpd", 0, false }, { "tm", 0, false },
+	{ "logging", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -20,11 +20,12 @@ enum DebugStream {
 	DBG_STATE,	// StateAccess logging
 	DBG_CHUNKEDIO,	// ChunkedIO logging
 	DBG_COMPRESSOR,	// Connection compressor
-	DBG_STRING,     // String code
+	DBG_STRING,	// String code
-	DBG_NOTIFIERS,   // Notifiers (see StateAccess.h)
+	DBG_NOTIFIERS,	// Notifiers (see StateAccess.h)
 	DBG_MAINLOOP,	// Main IOSource loop
 	DBG_DPD,	// Dynamic application detection framework
-	DBG_TM,	// Time-machine packet input via Brocolli
+	DBG_TM,		// Time-machine packet input via Brocolli
+	DBG_LOGGING,	// Logging streams
 
 	NUM_DBGS // Has to be last
 };

src/Desc.cc

@@ -42,6 +42,8 @@ ODesc::ODesc(desc_type t, BroFile* arg_f)
 	do_flush = 1;
 	include_stats = 0;
 	indent_with_spaces = 0;
+	escape = 0;
+	escape_len = 0;
 	}
 
 ODesc::~ODesc()
@@ -55,6 +57,12 @@ ODesc::~ODesc()
 		free(base);
 	}
 
+void ODesc::SetEscape(const char* arg_escape, int len)
+	{
+	escape = arg_escape;
+	escape_len = len;
+	}
+
 void ODesc::PushIndent()
 	{
 	++indent_level;
@@ -199,8 +207,44 @@ void ODesc::Indent()
 		}
 	}
 
+static const char hex_chars[] = "0123456789ABCDEF";
 
 void ODesc::AddBytes(const void* bytes, unsigned int n)
+	{
+	if ( ! escape )
+		return AddBytesRaw(bytes, n);
+
+	const char* s = (const char*) bytes;
+	const char* e = (const char*) bytes + n;
+
+	while ( s < e )
+		{
+		const char* t = (const char*) memchr(s, escape[0], e - s);
+
+		if ( ! t )
+			break;
+
+		if ( memcmp(t, escape, escape_len) != 0 )
+			break;
+
+		AddBytesRaw(s, t - s);
+
+		for ( int i = 0; i < escape_len; ++i )
+			{
+			char hex[5] = "\\x00";
+			hex[2] = hex_chars[(*t) >> 4];
+			hex[3] = hex_chars[(*t) & 0x0f];
+			AddBytesRaw(hex, sizeof(hex));
+			++t;
+			}
+
+		s = t;
+		}
+
+	AddBytesRaw(s, e - s);
+	}
+
+void ODesc::AddBytesRaw(const void* bytes, unsigned int n)
 	{
 	if ( n == 0 )
 		return;

src/Desc.h

@@ -49,6 +49,9 @@ public:
 
 	void SetFlush(int arg_do_flush)	{ do_flush = arg_do_flush; }
 
+	// The string passed in must remain valid as long as this object lives.
+	void SetEscape(const char* escape, int len);
+
 	void PushIndent();
 	void PopIndent();
 	void PopIndentNoNL();
@@ -97,6 +100,9 @@ public:
 				Add("\n", 0);
 			}
 
+	// Bypasses the escaping enabled via SetEscape().
+	void AddRaw(const char* s, int len)	{ AddBytesRaw(s, len); }
+
 	// Returns the description as a string.
 	const char* Description() const		{ return (const char*) base; }
 
@@ -119,6 +125,7 @@ protected:
 	void Indent();
 
 	void AddBytes(const void* bytes, unsigned int n);
+	void AddBytesRaw(const void* bytes, unsigned int n);
 
 	// Make buffer big enough for n bytes beyond bufp.
 	void Grow(unsigned int n);
@@ -132,6 +139,9 @@ protected:
 	unsigned int offset;	// where we are in the buffer
 	unsigned int size;	// size of buffer in bytes
 
+	int escape_len;	// number of bytes in to escape sequence
+	const char* escape;	// bytes to escape on output
+
 	BroFile* f;	// or the file we're using.
 
 	int indent_level;

src/EventHandler.cc

@@ -23,6 +23,11 @@ EventHandler::~EventHandler()
 	delete [] group;
 	}
 
+EventHandler::operator bool() const
+	{
+	return enabled && ((local && local->HasBodies()) || receivers.length());
+	}
+
 FuncType* EventHandler::FType()
 	{
 	if ( type )

src/EventHandler.h

@@ -34,8 +34,7 @@ public:
 	void Call(val_list* vl, bool no_remote = false);
 
 	// Returns true if there is at least one local or remote handler.
-	operator  bool() const
+	operator  bool() const;
-		{ return enabled && (local || receivers.length()); }
 
 	void SetUsed()          { used = true; }
 	bool Used()             { return used; }

src/Expr.cc

@@ -284,7 +284,7 @@ Val* NameExpr::Eval(Frame* f) const
 	Val* v;
 
 	if ( id->AsType() )
-		RunTime("cannot evaluate type name");
+		return new Val(id->AsType(), true);
 
 	if ( id->IsGlobal() )
 		v = id->ID_Val();
@@ -2497,12 +2497,7 @@ AssignExpr::AssignExpr(Expr* arg_op1, Expr* arg_op2, int arg_is_init,
 bool AssignExpr::TypeCheck()
 	{
 	TypeTag bt1 = op1->Type()->Tag();
-	if ( IsVector(bt1) )
-		bt1 = op1->Type()->AsVectorType()->YieldType()->Tag();
-
 	TypeTag bt2 = op2->Type()->Tag();
-	if ( IsVector(bt2) )
-		bt2 = op2->Type()->AsVectorType()->YieldType()->Tag();
 
 	if ( bt1 == TYPE_LIST && bt2 == TYPE_ANY )
 		// This is ok because we cannot explicitly declare lists on
@@ -2531,16 +2526,42 @@ bool AssignExpr::TypeCheck()
 		return true;
 		}
 
+	if ( bt1 == TYPE_VECTOR && bt2 == bt1 &&
+	     op2->Type()->AsVectorType()->IsUnspecifiedVector() )
+		{
+		op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType());
+		return true;
+		}
+
+	if ( op1->Type()->Tag() == TYPE_RECORD &&
+	     op2->Type()->Tag() == TYPE_RECORD )
+		{
+		if ( same_type(op1->Type(), op2->Type()) )
+			{
+			RecordType* rt1 = op1->Type()->AsRecordType();
+			RecordType* rt2 = op2->Type()->AsRecordType();
+
+			// Make sure the attributes match as well.
+			for ( int i = 0; i < rt1->NumFields(); ++i )
+				{
+				const TypeDecl* td1 = rt1->FieldDecl(i);
+				const TypeDecl* td2 = rt2->FieldDecl(i);
+
+				if ( same_attrs(td1->attrs, td2->attrs) )
+					// Everything matches.
+					return true;
+				}
+			}
+
+		// Need to coerce.
+		op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType());
+		return true;
+		}
+
 	if ( ! same_type(op1->Type(), op2->Type()) )
 		{
-		if ( op1->Type()->Tag() == TYPE_RECORD &&
+		ExprError("type clash in assignment");
-		     op2->Type()->Tag() == TYPE_RECORD )
+		return false;
-			op2 = new RecordCoerceExpr(op2, op1->Type()->AsRecordType());
-		else
-			{
-			ExprError("type clash in assignment");
-			return false;
-			}
 		}
 
 	return true;
@@ -3290,48 +3311,12 @@ RecordConstructorExpr::RecordConstructorExpr(ListExpr* constructor_list)
 
 Val* RecordConstructorExpr::InitVal(const BroType* t, Val* aggr) const
 	{
-	if ( ! aggr )
+	RecordVal* rv = Eval(0)->AsRecordVal();
-		aggr = new RecordVal(const_cast<RecordType*>(t->AsRecordType()));
+	RecordVal* ar = rv->CoerceTo(t->AsRecordType(), aggr);
 
-	if ( record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) )
+	if ( ar )
 		{
-		RecordVal* ar = aggr->AsRecordVal();
-		RecordType* ar_t = aggr->Type()->AsRecordType();
-
-		RecordVal* rv = Eval(0)->AsRecordVal();
-		RecordType* rv_t = rv->Type()->AsRecordType();
-
-		int i;
-		for ( i = 0; i < rv_t->NumFields(); ++i )
-			{
-			int t_i = ar_t->FieldOffset(rv_t->FieldName(i));
-
-			if ( t_i < 0 )
-				{
-				char buf[512];
-				safe_snprintf(buf, sizeof(buf),
-					      "orphan field \"%s\" in initialization",
-					      rv_t->FieldName(i));
-				Error(buf);
-				break;
-				}
-
-			else
-				ar->Assign(t_i, rv->Lookup(i)->Ref());
-			}
-
-		for ( i = 0; i < ar_t->NumFields(); ++i )
-			if ( ! ar->Lookup(i) &&
-			     ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) )
-				{
-				char buf[512];
-				safe_snprintf(buf, sizeof(buf),
-					      "non-optional field \"%s\" missing in initialization", ar_t->FieldName(i));
-				Error(buf);
-				}
-
 		Unref(rv);
-
 		return ar;
 		}
 
@@ -3400,7 +3385,7 @@ TableConstructorExpr::TableConstructorExpr(ListExpr* constructor_list,
 			SetError("values in table(...) constructor do not specify a table");
 		}
 
-	attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0;
+	attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0;
 	}
 
 Val* TableConstructorExpr::Eval(Frame* f) const
@@ -3466,7 +3451,7 @@ SetConstructorExpr::SetConstructorExpr(ListExpr* constructor_list,
 	else if ( type->Tag() != TYPE_TABLE || ! type->AsTableType()->IsSet() )
 		SetError("values in set(...) constructor do not specify a set");
 
-	attrs = arg_attrs ? new Attributes(arg_attrs, type) : 0;
+	attrs = arg_attrs ? new Attributes(arg_attrs, type, false) : 0;
 	}
 
 Val* SetConstructorExpr::Eval(Frame* f) const
@@ -3523,6 +3508,13 @@ VectorConstructorExpr::VectorConstructorExpr(ListExpr* constructor_list)
 	if ( IsError() )
 		return;
 
+	if ( constructor_list->Exprs().length() == 0 )
+		{
+		// vector().
+		SetType(new ::VectorType(base_type(TYPE_ANY)));
+		return;
+		}
+
 	BroType* t = merge_type_list(constructor_list);
 	if ( t )
 		{
@@ -3994,15 +3986,8 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r)
 			{
 			int t_i = t_r->FieldOffset(sub_r->FieldName(i));
 			if ( t_i < 0 )
-				{
+				// Orphane field in rhs, that's ok.
-				// Same as in RecordConstructorExpr::InitVal.
-				char buf[512];
-				safe_snprintf(buf, sizeof(buf),
-					      "orphan record field \"%s\"",
-					      sub_r->FieldName(i));
-				Error(buf);
 				continue;
-				}
 
 			BroType* sub_t_i = sub_r->FieldType(i);
 			BroType* sup_t_i = t_r->FieldType(t_i);
@@ -4047,7 +4032,23 @@ Val* RecordCoerceExpr::Fold(Val* v) const
 	for ( int i = 0; i < map_size; ++i )
 		{
 		if ( map[i] >= 0 )
-			val->Assign(i, rv->Lookup(map[i])->Ref());
+			{
+			Val* rhs = rv->Lookup(map[i]);
+			if ( ! rhs )
+				{
+				const Attr* def = rv->Type()->AsRecordType()->FieldDecl(
+					map[i])->FindAttr(ATTR_DEFAULT);
+
+				if ( def )
+					rhs = def->AttrExpr()->Eval(0);
+				}
+
+			if ( rhs )
+				rhs = rhs->Ref();
+
+			assert(rhs || Type()->AsRecordType()->FieldDecl(i)->FindAttr(ATTR_OPTIONAL));
+			val->Assign(i, rhs);
+			}
 		else
 			val->Assign(i, 0);
 		}
@@ -4132,6 +4133,50 @@ bool TableCoerceExpr::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+VectorCoerceExpr::VectorCoerceExpr(Expr* op, VectorType* v)
+: UnaryExpr(EXPR_VECTOR_COERCE, op)
+	{
+	if ( IsError() )
+		return;
+
+	SetType(v->Ref());
+
+	if ( Type()->Tag() != TYPE_VECTOR )
+		ExprError("coercion to non-vector");
+
+	else if ( op->Type()->Tag() != TYPE_VECTOR )
+		ExprError("coercion of non-vector to vector");
+	}
+
+
+VectorCoerceExpr::~VectorCoerceExpr()
+	{
+	}
+
+Val* VectorCoerceExpr::Fold(Val* v) const
+	{
+	VectorVal* vv = v->AsVectorVal();
+
+	if ( vv->Size() > 0 )
+		Internal("coercion of non-empty vector");
+
+	return new VectorVal(Type()->Ref()->AsVectorType());
+	}
+
+IMPLEMENT_SERIAL(VectorCoerceExpr, SER_VECTOR_COERCE_EXPR);
+
+bool VectorCoerceExpr::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_VECTOR_COERCE_EXPR, UnaryExpr);
+	return true;
+	}
+
+bool VectorCoerceExpr::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(UnaryExpr);
+	return true;
+	}
+
 FlattenExpr::FlattenExpr(Expr* arg_op)
 : UnaryExpr(EXPR_FLATTEN, arg_op)
 	{
@@ -5308,27 +5353,52 @@ int check_and_promote_expr(Expr*& e, BroType* t)
 		return 1;
 		}
 
-	else if ( ! same_type(t, et) )
+	if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD )
 		{
-		if ( t->Tag() == TYPE_RECORD && et->Tag() == TYPE_RECORD )
+		RecordType* t_r = t->AsRecordType();
-			{
+		RecordType* et_r = et->AsRecordType();
-			RecordType* t_r = t->AsRecordType();
-			RecordType* et_r = et->AsRecordType();
 
-			if ( record_promotion_compatible(t_r, et_r) )
+		if ( same_type(t, et) )
+			{
+			// Make sure the attributes match as well.
+			for ( int i = 0; i < t_r->NumFields(); ++i )
 				{
-				e = new RecordCoerceExpr(e, t_r);
+				const TypeDecl* td1 = t_r->FieldDecl(i);
-				return 1;
+				const TypeDecl* td2 = et_r->FieldDecl(i);
+
+				if ( same_attrs(td1->attrs, td2->attrs) )
+					// Everything matches perfectly.
+					return 1;
 				}
 			}
 
-		else if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE &&
+		if ( record_promotion_compatible(t_r, et_r) )
+			{
+			e = new RecordCoerceExpr(e, t_r);
+			return 1;
+			}
+
+		t->Error("incompatible record types", e);
+		return 0;
+		}
+
+
+	if ( ! same_type(t, et) )
+		{
+		if ( t->Tag() == TYPE_TABLE && et->Tag() == TYPE_TABLE &&
 			  et->AsTableType()->IsUnspecifiedTable() )
 			{
 			e = new TableCoerceExpr(e, t->AsTableType());
 			return 1;
 			}
 
+		if ( t->Tag() == TYPE_VECTOR && et->Tag() == TYPE_VECTOR &&
+		     et->AsVectorType()->IsUnspecifiedVector() )
+			{
+			e = new VectorCoerceExpr(e, t->AsVectorType());
+			return 1;
+			}
+
 		t->Error("type clash", e);
 		return 0;
 		}

src/Expr.h

@@ -40,7 +40,10 @@ typedef enum {
 	EXPR_CALL,
 	EXPR_EVENT,
 	EXPR_SCHEDULE,
-	EXPR_ARITH_COERCE, EXPR_RECORD_COERCE, EXPR_TABLE_COERCE,
+	EXPR_ARITH_COERCE,
+	EXPR_RECORD_COERCE,
+	EXPR_TABLE_COERCE,
+	EXPR_VECTOR_COERCE,
 	EXPR_SIZE,
 	EXPR_FLATTEN,
 #define NUM_EXPRS (int(EXPR_FLATTEN) + 1)
@@ -895,6 +898,20 @@ protected:
 	DECLARE_SERIAL(TableCoerceExpr);
 };
 
+class VectorCoerceExpr : public UnaryExpr {
+public:
+	VectorCoerceExpr(Expr* op, VectorType* v);
+	~VectorCoerceExpr();
+
+protected:
+	friend class Expr;
+	VectorCoerceExpr()	{ }
+
+	Val* Fold(Val* v) const;
+
+	DECLARE_SERIAL(VectorCoerceExpr);
+};
+
 // An internal operator for flattening array indices that are records
 // into a list of individual values.
 class FlattenExpr : public UnaryExpr {

src/Func.cc

@@ -239,11 +239,15 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits,
 : Func(BRO_FUNC)
 	{
 	id = arg_id;
-	Body b;
-	b.stmts = AddInits(arg_body, aggr_inits);
-	b.priority = 0;
-	bodies.push_back(b);
 	frame_size = arg_frame_size;
+
+	if ( arg_body )
+		{
+		Body b;
+		b.stmts = AddInits(arg_body, aggr_inits);
+		b.priority = 0;
+		bodies.push_back(b);
+		}
 	}
 
 BroFunc::~BroFunc()
@@ -267,6 +271,13 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 #ifdef PROFILE_BRO_FUNCTIONS
 	DEBUG_MSG("Function: %s\n", id->Name());
 #endif
+	if ( ! bodies.size() ) 
+		{
+		// Can only happen for events.
+		assert(IsEvent());
+		return 0 ;
+		}
+
 	SegmentProfiler(segment_logger, location);
 	Frame* f = new Frame(frame_size, this, args);
 
@@ -497,9 +508,11 @@ void builtin_run_time(const char* msg, BroObj* arg)
 	}
 
 #include "bro.bif.func_h"
+#include "logging.bif.func_h"
 #include "strings.bif.func_h"
 
 #include "bro.bif.func_def"
+#include "logging.bif.func_def"
 #include "strings.bif.func_def"
 
 void init_builtin_funcs()
@@ -511,6 +524,7 @@ void init_builtin_funcs()
 	gap_info = internal_type("gap_info")->AsRecordType();
 
 #include "bro.bif.func_init"
+#include "logging.bif.func_init"
 #include "strings.bif.func_init"
 
 	did_builtin_init = true;

src/Func.h

@@ -15,6 +15,7 @@ class FuncType;
 class Stmt;
 class Frame;
 class ID;
+class CallExpr;
 
 class Func : public BroObj {
 public:
@@ -36,7 +37,8 @@ public:
 			{ return priority > other.priority; } // reverse sort
 	};
 
-	virtual const vector<Body>& GetBodies() const	{ return bodies; }
+	const vector<Body>& GetBodies() const	{ return bodies; }
+	bool HasBodies() const	{ return bodies.size(); }
 
 	// virtual Val* Call(ListExpr* args) const = 0;
 	virtual Val* Call(val_list* args, Frame* parent = 0) const = 0;

src/ID.cc

@@ -235,6 +235,25 @@ void ID::UpdateValAttrs()
 				}
 			}
 		}
+
+	if ( Type()->Tag() == TYPE_RECORD )
+		{
+		Attr* attr = attrs->FindAttr(ATTR_LOG);
+		if ( attr )
+			{
+			// Apply &log to all record fields.
+			RecordType* rt = Type()->AsRecordType();
+			for ( int i = 0; i < rt->NumFields(); ++i )
+				{
+				TypeDecl* fd = rt->FieldDecl(i);
+
+				if ( ! fd->attrs )
+					fd->attrs = new Attributes(new attr_list, rt->FieldType(i), true);
+
+				fd->attrs->AddAttr(new Attr(ATTR_LOG));
+				}
+			}
+		}
 	}
 
 void ID::AddAttrs(Attributes* a)

src/LogMgr.h

@@ -0,0 +1,133 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// A class managing log writers and filters.
+
+#ifndef LOGMGR_H
+#define LOGMGR_H
+
+#include "Val.h"
+#include "EventHandler.h"
+#include "RemoteSerializer.h"
+
+class SerializationFormat;
+
+// Description of a log field.
+struct LogField {
+	string name;
+	TypeTag type;
+
+	LogField()	{ }
+	LogField(const LogField& other)
+		: name(other.name), type(other.type)	{ }
+
+	// (Un-)serialize.
+	bool Read(SerializationFormat* fmt);
+	bool Write(SerializationFormat* fmt) const;
+};
+
+// Values as logged by a writer.
+struct LogVal {
+	TypeTag type;
+	bool present; // False for unset fields.
+
+	// The following union is a subset of BroValUnion, including only the
+	// types we can log directly.
+	struct set_t { bro_int_t size; LogVal** vals; };
+	typedef set_t vec_t;
+
+	union _val {
+		bro_int_t int_val;
+		bro_uint_t uint_val;
+		addr_type addr_val;
+		subnet_type subnet_val;
+		double double_val;
+		string* string_val;
+		set_t set_val;
+		vec_t vector_val;
+	} val;
+
+	LogVal(TypeTag arg_type = TYPE_ERROR, bool arg_present = true)
+		: type(arg_type), present(arg_present)	{}
+	~LogVal();
+
+	// (Un-)serialize.
+	bool Read(SerializationFormat* fmt);
+	bool Write(SerializationFormat* fmt) const;
+
+	// Returns true if the type can be logged the framework. If
+	// `atomic_only` is true, will not permit composite types.
+	static bool IsCompatibleType(BroType* t, bool atomic_only=false);
+
+private:
+	LogVal(const LogVal& other)	{ }
+};
+
+class LogWriter;
+class RemoteSerializer;
+class RotationTimer;
+
+class LogMgr {
+public:
+	LogMgr();
+	~LogMgr();
+
+	// These correspond to the BiFs visible on the scripting layer. The
+	// actual BiFs just forward here.
+	bool CreateStream(EnumVal* id, RecordVal* stream);
+	bool EnableStream(EnumVal* id);
+	bool DisableStream(EnumVal* id);
+	bool AddFilter(EnumVal* id, RecordVal* filter);
+	bool RemoveFilter(EnumVal* id, StringVal* name);
+	bool RemoveFilter(EnumVal* id, string name);
+	bool Write(EnumVal* id, RecordVal* columns);
+	bool SetBuf(EnumVal* id, bool enabled);	// Adjusts all writers.
+	bool Flush(EnumVal* id);		// Flushes all writers..
+
+protected:
+	friend class LogWriter;
+	friend class RemoteSerializer;
+	friend class RotationTimer;
+
+	//// Function also used by the RemoteSerializer.
+
+	// Takes ownership of fields.
+	LogWriter* CreateWriter(EnumVal* id, EnumVal* writer, string path,
+				int num_fields, LogField** fields);
+
+	// Takes ownership of values..
+	bool Write(EnumVal* id, EnumVal* writer, string path,
+		   int num_fields, LogVal** vals);
+
+	// Announces all instantiated writers to peer.
+	void SendAllWritersTo(RemoteSerializer::PeerID peer);
+
+	//// Functions safe to use by writers.
+
+	// Reports an error for the given writer.
+	void Error(LogWriter* writer, const char* msg);
+
+private:
+	struct Filter;
+	struct Stream;
+	struct WriterInfo;
+
+	bool TraverseRecord(Stream* stream, Filter* filter, RecordType* rt,
+			    TableVal* include, TableVal* exclude, string path, list<int> indices);
+
+	LogVal** RecordToFilterVals(Stream* stream, Filter* filter,
+				    RecordVal* columns);
+
+	LogVal* ValToLogVal(Val* val, BroType* ty = 0);
+	Stream* FindStream(EnumVal* id);
+	void RemoveDisabledWriters(Stream* stream);
+	void InstallRotationTimer(WriterInfo* winfo);
+	void Rotate(WriterInfo* info);
+	RecordVal* LookupRotationControl(EnumVal* writer, string path);
+	Filter* FindFilter(EnumVal* id, StringVal* filter);
+
+	vector<Stream *> streams;	// Indexed by stream enum.
+};
+
+extern LogMgr* log_mgr;
+
+#endif

src/LogWriter.cc

@@ -0,0 +1,188 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include "util.h"
+#include "LogWriter.h"
+
+LogWriter::LogWriter()
+	{
+	buf = 0;
+	buf_len = 1024;
+	buffering = true;
+	disabled = false;
+	}
+
+LogWriter::~LogWriter()
+	{
+	if ( buf )
+		free(buf);
+
+	delete [] fields;
+	}
+
+bool LogWriter::Init(string arg_path, int arg_num_fields,
+		     const LogField* const * arg_fields)
+	{
+	path = arg_path;
+	num_fields = arg_num_fields;
+	fields = arg_fields;
+
+	if ( ! DoInit(arg_path, arg_num_fields, arg_fields) )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+bool LogWriter::Write(int arg_num_fields, LogVal** vals)
+	{
+	// Double-check that the arguments match. If we get this from remote,
+	// something might be mixed up.
+	if ( num_fields != arg_num_fields )
+		{
+		DBG_LOG(DBG_LOGGING, "Number of fields don't match in LogWriter::Write() (%d vs. %d)",
+			arg_num_fields, num_fields);
+
+		return false;
+		}
+
+	for ( int i = 0; i < num_fields; ++i )
+		{
+		if ( vals[i]->type != fields[i]->type )
+			{
+			DBG_LOG(DBG_LOGGING, "Field type doesn't match in LogWriter::Write() (%d vs. %d)",
+				vals[i]->type, fields[i]->type);
+			return false;
+			}
+		}
+
+	bool result = DoWrite(num_fields, fields, vals);
+
+	DeleteVals(vals);
+
+	if ( ! result )
+		disabled = true;
+
+	return result;
+	}
+
+bool LogWriter::SetBuf(bool enabled)
+	{
+	if ( enabled == buffering )
+		// No change.
+		return true;
+
+	buffering = enabled;
+
+	if ( ! DoSetBuf(enabled) )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+bool LogWriter::Rotate(string rotated_path, string postprocessor, double open,
+		       double close, bool terminating)
+	{
+	if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+bool LogWriter::Flush()
+	{
+	if ( ! DoFlush() )
+		{
+		disabled = true;
+		return false;
+		}
+
+	return true;
+	}
+
+void LogWriter::Finish()
+	{
+	DoFinish();
+	}
+
+const char* LogWriter::Fmt(const char* format, ...)
+	{
+	if ( ! buf )
+		buf = (char*) malloc(buf_len);
+
+	va_list al;
+	va_start(al, format);
+	int n = safe_vsnprintf(buf, buf_len, format, al);
+	va_end(al);
+
+	if ( (unsigned int) n >= buf_len )
+		{ // Not enough room, grow the buffer.
+		buf_len = n + 32;
+		buf = (char*) realloc(buf, buf_len);
+
+		// Is it portable to restart?
+		va_start(al, format);
+		n = safe_vsnprintf(buf, buf_len, format, al);
+		va_end(al);
+		}
+
+	return buf;
+	}
+
+void LogWriter::Error(const char *msg)
+	{
+	log_mgr->Error(this, msg);
+	}
+
+void LogWriter::DeleteVals(LogVal** vals)
+	{
+	for ( int i = 0; i < num_fields; i++ )
+		delete vals[i];
+	}
+
+bool LogWriter::RunPostProcessor(string fname, string postprocessor,
+				 string old_name, double open, double close,
+				 bool terminating)
+	{
+	// This function operates in a way that is backwards-compatible with
+	// the old Bro log rotation scheme.
+
+	if ( ! postprocessor.size() )
+		return true;
+
+	const char* const fmt = "%y-%m-%d_%H.%M.%S";
+
+	struct tm tm1;
+	struct tm tm2;
+
+	time_t tt1 = (time_t)open;
+	time_t tt2 = (time_t)close;
+
+	localtime_r(&tt1, &tm1);
+	localtime_r(&tt2, &tm2);
+
+	char buf1[128];
+	char buf2[128];
+
+	strftime(buf1, sizeof(buf1), fmt, &tm1);
+	strftime(buf2, sizeof(buf2), fmt, &tm2);
+
+	string cmd = postprocessor;
+	cmd += " " + fname;
+	cmd += " " + old_name;
+	cmd += " " + string(buf1);
+	cmd += " " + string(buf2);
+	cmd += " " + string(terminating ? "1" : "0");
+	cmd += " &";
+
+	system(cmd.c_str());
+
+	return true;
+	}

src/LogWriter.h

@@ -0,0 +1,188 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Interface API for a log writer backend. The LogMgr creates a separate
+// writer instance of pair of (writer type, output path).
+//
+// Note thay classes derived from LogWriter must be fully thread-safe and not
+// use any non-thread-safe Bro functionality (which includes almost
+// everything ...). In particular, do not use fmt() but LogWriter::Fmt()!.
+//
+// The one exception to this rule is the constructor: it is guaranteed to be
+// executed inside the main thread and can thus in particular access global
+// script variables.
+
+#ifndef LOGWRITER_H
+#define LOGWRITER_H
+
+#include "LogMgr.h"
+#include "BroString.h"
+
+class LogWriter {
+public:
+	LogWriter();
+	virtual ~LogWriter();
+
+	//// Interface methods to interact with the writer. Note that these
+	//// methods are not necessarily thread-safe and must be called only
+	//// from the main thread (which will typically mean only from the
+	//// LogMgr). In particular, they must not be called from the
+	//// writer's derived implementation.
+
+	// One-time initialization of the writer to define the logged fields.
+	// Interpretation of "path" is left to the writer, and will be
+	// corresponding the value configured on the script-level.
+	// 
+	// Returns false if an error occured, in which case the writer must
+	// not be used further.
+	//
+	// The new instance takes ownership of "fields", and will delete them
+	// when done.
+	bool Init(string path, int num_fields, const LogField* const * fields);
+
+	// Writes one log entry. The method takes ownership of "vals" and
+	// will return immediately after queueing the write request, which is
+	// potentially before output has actually been written out.
+	//
+	// num_fields and the types of the LogVals must match what was passed
+	// to Init().
+	//
+	// Returns false if an error occured, in which case the writer must
+	// not be used any further.
+	bool Write(int num_fields, LogVal** vals);
+
+	// Sets the buffering status for the writer, if the writer supports
+	// that. (If not, it will be ignored).
+	bool SetBuf(bool enabled);
+
+	// Flushes any currently buffered output, if the writer supports 
+	// that. (If not, it will be ignored).
+	bool Flush();
+
+	// Triggers rotation, if the writer supports that. (If not, it will
+	// be ignored).
+	bool Rotate(string rotated_path, string postprocessor, double open,
+		    double close, bool terminating);
+
+	// Finishes writing to this logger regularly. Must not be called if
+	// an error has been indicated earlier. After calling this, no
+	// further writing must be performed.
+	void Finish();
+
+	//// Thread-safe methods that may be called from the writer
+	//// implementation.
+
+	// The following methods return the information as passed to Init().
+	const string Path() const	{ return path; }
+	int NumFields() const	{ return num_fields; }
+	const LogField* const * Fields() const	{ return fields; }
+
+protected:
+
+	// Methods for writers to override. If any of these returs false, it
+	// will be assumed that a fatal error has occured that prevents the
+	// writer from further operation. It will then be disabled and
+	// deleted. When return false, the writer should also report the
+	// error via Error(). Note that even if a writer does not support the
+	// functionality for one these methods (like rotation), it must still
+	// return true if that is not to be considered a fatal error.
+	//
+	// Called once for initialization of the writer.
+	virtual bool DoInit(string path, int num_fields,
+			    const LogField* const * fields) = 0;
+
+	// Called once per log entry to record.
+	virtual bool DoWrite(int num_fields, const LogField* const * fields,
+			     LogVal** vals) = 0;
+
+	// Called when the buffering status for this writer is changed. If
+	// buffering is disabled, the writer should attempt to write out
+	// information as quickly as possible even if doing so may have a
+	// performance impact. If enabled (which is the default), it may
+	// buffer data as helpful and write it out later in a way optimized
+	// for performance. The current buffering state can be queried via
+	// IsBuf().
+	//
+	// A writer may ignore buffering changes if it doesn't fit with its
+	// semantics (but must still return true in that case).
+	virtual bool DoSetBuf(bool enabled) = 0;
+
+	// Called to flush any currently buffered output.
+	//
+	// A writer may ignore flush requests if it doesn't fit with its
+	// semantics (but must still return true in that case).
+	virtual bool DoFlush() = 0;
+
+	// Called when a log output is to be rotated. Most directly this only
+	// applies to writers writing into files, which should then close the
+	// current file and open a new one.  However, a writer may also
+	// trigger other apppropiate actions if semantics are similar.
+	//
+	// "rotate_path" reflects the path to where the rotated output is to
+	// be moved, with specifics depending on the writer. It should
+	// generally be interpreted in a way consistent with that of "path"
+	// as passed into DoInit(). As an example, for file-based output, 
+	// "rotate_path" could be the original filename extended with a
+	// timestamp indicating the time of the rotation.
+
+	// "postprocessor" is the name of a command to execute on the rotated
+	// file. If empty, no postprocessing should take place; if given but
+	// the writer doesn't support postprocessing, it can be ignored (but
+	// the method must still return true in that case).
+
+	// "open" and "close" are the network time's when the *current* file
+	// was opened and closed, respectively.
+	//
+	// "terminating" indicated whether the rotation request occurs due
+	// the main Bro prcoess terminating (and not because we've reach a
+	// regularly scheduled time for rotation).
+	//
+	// A writer may ignore rotation requests if it doesn't fit with its
+	// semantics (but must still return true in that case).
+	virtual bool DoRotate(string rotated_path, string postprocessor,
+			      double open, double close, bool terminating) = 0;
+
+	// Called once on termination. Not called when any of the other
+	// methods has previously signaled an error, i.e., executing this
+	// method signals a regular shutdown of the writer.
+	virtual void DoFinish() = 0;
+
+	//// Methods for writers to use. These are thread-safe.
+
+	// A thread-safe version of fmt().
+	const char* Fmt(const char* format, ...);
+
+	// Returns the current buffering state.
+	bool IsBuf()	{ return buffering; }
+
+	// Reports an error to the user.
+	void Error(const char *msg);
+
+	// Runs a post-processor on the given file. Parameters correspond to
+	// those of DoRotate(). 
+	bool RunPostProcessor(string fname, string postprocessor,
+			      string old_name, double open, double close,
+			      bool terminating);
+
+private:
+	friend class LogMgr;
+
+	// When an error occurs, we call this method to set a flag marking
+	// the writer as disabled. The LogMgr will check the flag later and
+	// remove the writer.
+	bool Disabled()	{ return disabled; }
+
+	// Deletes the values as passed into Write().
+	void DeleteVals(LogVal** vals);
+
+	string path;
+	int num_fields;
+	const LogField* const * fields;
+	bool buffering;
+	bool disabled;
+
+	// For implementing Fmt().
+	char* buf;
+	unsigned int buf_len;
+};
+
+#endif

src/LogWriterAscii.cc

@@ -0,0 +1,266 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <string>
+#include <errno.h>
+
+#include "LogWriterAscii.h"
+#include "NetVar.h"
+
+LogWriterAscii::LogWriterAscii()
+	{
+	file = 0;
+
+	output_to_stdout = BifConst::LogAscii::output_to_stdout;
+	include_header = BifConst::LogAscii::include_header;
+
+	separator_len = BifConst::LogAscii::separator->Len();
+	separator = new char[separator_len];
+	memcpy(separator, BifConst::LogAscii::separator->Bytes(),
+	       separator_len);
+
+	set_separator_len = BifConst::LogAscii::set_separator->Len();
+	set_separator = new char[set_separator_len];
+	memcpy(set_separator, BifConst::LogAscii::set_separator->Bytes(),
+	       set_separator_len);
+
+	empty_field_len = BifConst::LogAscii::empty_field->Len();
+	empty_field = new char[empty_field_len];
+	memcpy(empty_field, BifConst::LogAscii::empty_field->Bytes(),
+	       empty_field_len);
+
+	unset_field_len = BifConst::LogAscii::unset_field->Len();
+	unset_field = new char[unset_field_len];
+	memcpy(unset_field, BifConst::LogAscii::unset_field->Bytes(),
+	       unset_field_len);
+
+	header_prefix_len = BifConst::LogAscii::header_prefix->Len();
+	header_prefix = new char[header_prefix_len];
+	memcpy(header_prefix, BifConst::LogAscii::header_prefix->Bytes(),
+	       header_prefix_len);
+
+	}
+
+LogWriterAscii::~LogWriterAscii()
+	{
+	if ( file )
+		fclose(file);
+
+	delete [] separator;
+	delete [] set_separator;
+	delete [] empty_field;
+	delete [] unset_field;
+	delete [] header_prefix;
+	}
+
+bool LogWriterAscii::DoInit(string path, int num_fields,
+			    const LogField* const * fields)
+	{
+	if ( output_to_stdout )
+		path = "/dev/stdout";
+
+	fname = IsSpecial(path) ? path : path + ".log";
+
+	if ( ! (file = fopen(fname.c_str(), "w")) )
+		{
+		Error(Fmt("cannot open %s: %s", fname.c_str(),
+			  strerror(errno)));
+
+		return false;
+		}
+
+	if ( include_header )
+		{
+		if ( fwrite(header_prefix, header_prefix_len, 1, file) != 1 )
+			goto write_error;
+
+		for ( int i = 0; i < num_fields; i++ )
+			{
+			if ( i > 0 &&
+			     fwrite(separator, separator_len, 1, file) != 1 )
+				goto write_error;
+
+			const LogField* field = fields[i];
+
+			if ( fputs(field->name.c_str(), file) == EOF )
+				goto write_error;
+			}
+
+		if ( fputc('\n', file) == EOF )
+			goto write_error;
+		}
+
+	return true;
+
+write_error:
+	Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
+	return false;
+	}
+
+bool LogWriterAscii::DoFlush()
+	{
+	fflush(file);
+	return true;
+	}
+
+void LogWriterAscii::DoFinish()
+	{
+	}
+
+bool LogWriterAscii::DoWriteOne(ODesc* desc, LogVal* val, const LogField* field)
+	{
+	if ( ! val->present )
+		{
+		desc->AddN(unset_field, unset_field_len);
+		return true;
+		}
+
+	switch ( val->type ) {
+
+	case TYPE_BOOL:
+		desc->Add(val->val.int_val ? "T" : "F");
+		break;
+
+	case TYPE_INT:
+		desc->Add(val->val.int_val);
+		break;
+
+	case TYPE_COUNT:
+	case TYPE_COUNTER:
+	case TYPE_PORT:
+		desc->Add(val->val.uint_val);
+		break;
+
+	case TYPE_SUBNET:
+		desc->Add(dotted_addr(val->val.subnet_val.net));
+		desc->Add("/");
+		desc->Add(val->val.subnet_val.width);
+		break;
+
+	case TYPE_NET:
+	case TYPE_ADDR:
+		desc->Add(dotted_addr(val->val.addr_val));
+		break;
+
+	case TYPE_DOUBLE:
+	case TYPE_TIME:
+	case TYPE_INTERVAL:
+		desc->Add(val->val.double_val);
+	break;
+
+	case TYPE_ENUM:
+	case TYPE_STRING:
+	case TYPE_FILE:
+		{
+		int size = val->val.string_val->size();
+		if ( size )
+			desc->AddN(val->val.string_val->data(), val->val.string_val->size());
+		else
+			desc->AddN(empty_field, empty_field_len);
+		break;
+		}
+
+	case TYPE_TABLE:
+		{
+		if ( ! val->val.set_val.size )
+			{
+			desc->AddN(empty_field, empty_field_len);
+			break;
+			}
+
+		for ( int j = 0; j < val->val.set_val.size; j++ )
+			{
+			if ( j > 0 )
+				desc->AddN(set_separator, set_separator_len);
+
+			if ( ! DoWriteOne(desc, val->val.set_val.vals[j], field) )
+				return false;
+			}
+
+		break;
+		}
+
+	case TYPE_VECTOR:
+		{
+		if ( ! val->val.vector_val.size )
+			{
+			desc->AddN(empty_field, empty_field_len);
+			break;
+			}
+
+		for ( int j = 0; j < val->val.vector_val.size; j++ )
+			{
+			if ( j > 0 )
+				desc->AddN(set_separator, set_separator_len);
+
+			if ( ! DoWriteOne(desc, val->val.vector_val.vals[j], field) )
+				return false;
+			}
+
+		break;
+		}
+
+	default:
+		Error(Fmt("unsupported field format %d for %s", val->type,
+			  field->name.c_str()));
+		return false;
+	}
+
+	return true;
+	}
+
+bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
+			     LogVal** vals)
+	{
+	ODesc desc(DESC_READABLE);
+	desc.SetEscape(separator, separator_len);
+
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( i > 0 )
+			desc.AddRaw(separator, separator_len);
+
+		if ( ! DoWriteOne(&desc, vals[i], fields[i]) )
+			return false;
+		}
+
+	desc.Add("\n");
+
+	if ( fwrite(desc.Bytes(), desc.Len(), 1, file) != 1 )
+		{
+		Error(Fmt("error writing to %s: %s", fname.c_str(), strerror(errno)));
+		return false;
+		}
+
+	if ( IsBuf() )
+		fflush(file);
+
+	return true;
+	}
+
+bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open,
+			      double close, bool terminating)
+	{
+	if ( IsSpecial(Path()) )
+		// Don't rotate special files.
+		return true;
+
+	fclose(file);
+
+	string nname = rotated_path + ".log";
+	rename(fname.c_str(), nname.c_str());
+
+	if ( postprocessor.size() &&
+	     ! RunPostProcessor(nname, postprocessor, fname.c_str(),
+				open, close, terminating) )
+		return false;
+
+	return DoInit(Path(), NumFields(), Fields());
+	}
+
+bool LogWriterAscii::DoSetBuf(bool enabled)
+	{
+	// Nothing to do.
+	return true;
+	}
+
+

src/LogWriterAscii.h

@@ -0,0 +1,55 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+//
+// Log writer for delimiter-separated ASCII logs.
+
+#ifndef LOGWRITERASCII_H
+#define LOGWRITERASCII_H
+
+#include "LogWriter.h"
+
+class LogWriterAscii : public LogWriter {
+public:
+	LogWriterAscii();
+	~LogWriterAscii();
+
+	static LogWriter* Instantiate()	{ return new LogWriterAscii; }
+
+protected:
+	virtual bool DoInit(string path, int num_fields,
+			    const LogField* const * fields);
+	virtual bool DoWrite(int num_fields, const LogField* const * fields,
+			     LogVal** vals);
+	virtual bool DoSetBuf(bool enabled);
+	virtual bool DoRotate(string rotated_path, string postprocessr,
+			      double open, double close, bool terminating);
+	virtual bool DoFlush();
+	virtual void DoFinish();
+
+private:
+	bool IsSpecial(string path) 	{ return path.find("/dev/") == 0; }
+	bool DoWriteOne(ODesc* desc, LogVal* val, const LogField* field);
+
+	FILE* file;
+	string fname;
+
+	// Options set from the script-level.
+	bool output_to_stdout;
+	bool include_header;
+
+	char* separator;
+	int separator_len;
+
+	char* set_separator;
+	int set_separator_len;
+
+	char* empty_field;
+	int empty_field_len;
+
+	char* unset_field;
+	int unset_field_len;
+
+	char* header_prefix;
+	int header_prefix_len;
+};
+
+#endif

src/Logger.h

@@ -1,5 +1,3 @@
-// $Id: Logger.h 6219 2008-10-01 05:39:07Z vern $
-//
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #ifndef logger_h

src/Login.h

@@ -1,5 +1,3 @@
-// $Id: Login.h 6219 2008-10-01 05:39:07Z vern $
-//
 // See the file "COPYING" in the main distribution directory for copyright.
 
 #ifndef login_h

src/NetVar.cc

@@ -263,6 +263,7 @@ TableType* id_table;
 #include "const.bif.netvar_def"
 #include "types.bif.netvar_def"
 #include "event.bif.netvar_def"
+#include "logging.bif.netvar_def"
 
 void init_event_handlers()
 	{
@@ -318,6 +319,7 @@ void init_net_var()
 	{
 #include "const.bif.netvar_init"
 #include "types.bif.netvar_init"
+#include "logging.bif.netvar_init"
 
 	conn_id = internal_type("conn_id")->AsRecordType();
 	endpoint = internal_type("endpoint")->AsRecordType();

src/NetVar.h

@@ -273,5 +273,6 @@ extern void init_net_var();
 #include "const.bif.netvar_h"
 #include "types.bif.netvar_h"
 #include "event.bif.netvar_h"
+#include "logging.bif.netvar_h"
 
 #endif

src/RemoteSerializer.cc

@@ -183,6 +183,7 @@
 #include "Sessions.h"
 #include "File.h"
 #include "Conn.h"
+#include "LogMgr.h"
 
 extern "C" {
 #include "setsignal.h"
@@ -190,7 +191,7 @@ extern "C" {
 
 // Gets incremented each time there's an incompatible change
 // to the communication internals.
-static const unsigned short PROTOCOL_VERSION = 0x06;
+static const unsigned short PROTOCOL_VERSION = 0x07;
 
 static const char MSG_NONE = 0x00;
 static const char MSG_VERSION = 0x01;
@@ -216,9 +217,12 @@ static const char MSG_SYNC_POINT = 0x14;
 static const char MSG_TERMINATE = 0x15;
 static const char MSG_DEBUG_DUMP = 0x16;
 static const char MSG_REMOTE_PRINT = 0x17;
+static const char MSG_LOG_CREATE_WRITER = 0x18;
+static const char MSG_LOG_WRITE = 0x19;
+static const char MSG_REQUEST_LOGS = 0x20;
 
 // Update this one whenever adding a new ID:
-static const char MSG_ID_MAX = MSG_REMOTE_PRINT;
+static const char MSG_ID_MAX = MSG_REQUEST_LOGS;
 
 static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U;
 
@@ -226,6 +230,9 @@ static const uint32 FINAL_SYNC_POINT = /* UINT32_MAX */ 4294967295U;
 static const int PRINT_BUFFER_SIZE = 10 * 1024;
 static const int SOCKBUF_SIZE = 1024 * 1024;
 
+// Buffer size for remote-log data.
+static const int LOG_BUFFER_SIZE = 50 * 1024;
+
 struct ping_args {
 	uint32 seq;
 	double time1; // Round-trip time parent1<->parent2
@@ -303,6 +310,9 @@ static const char* msgToStr(int msg)
 	MSG_STR(MSG_TERMINATE)
 	MSG_STR(MSG_DEBUG_DUMP)
 	MSG_STR(MSG_REMOTE_PRINT)
+	MSG_STR(MSG_LOG_CREATE_WRITER)
+	MSG_STR(MSG_LOG_WRITE)
+	MSG_STR(MSG_REQUEST_LOGS)
 	default:
 		return "UNKNOWN_MSG";
 	}
@@ -469,7 +479,10 @@ static inline bool is_peer_msg(int msg)
 		msg == MSG_CAPS ||
 		msg == MSG_COMPRESS ||
 		msg == MSG_SYNC_POINT ||
-		msg == MSG_REMOTE_PRINT;
+		msg == MSG_REMOTE_PRINT ||
+		msg == MSG_LOG_CREATE_WRITER ||
+		msg == MSG_LOG_WRITE ||
+		msg == MSG_REQUEST_LOGS;
 	}
 
 bool RemoteSerializer::IsConnectedPeer(PeerID id)
@@ -704,6 +717,7 @@ bool RemoteSerializer::CloseConnection(Peer* peer)
 		return true;
 
 	FlushPrintBuffer(peer);
+	FlushLogBuffer(peer);
 
 	Log(LogInfo, "closing connection", peer);
 
@@ -738,6 +752,31 @@ bool RemoteSerializer::RequestSync(PeerID id, bool auth)
 	return true;
 	}
 
+bool RemoteSerializer::RequestLogs(PeerID id)
+	{
+	if ( ! using_communication )
+		return true;
+
+	Peer* peer = LookupPeer(id, true);
+	if ( ! peer )
+		{
+		run_time(fmt("unknown peer id %d for request logs", int(id)));
+		return false;
+		}
+
+	if ( peer->phase != Peer::HANDSHAKE )
+		{
+		run_time(fmt("can't request logs from peer; wrong phase %d",
+			     peer->phase));
+		return false;
+		}
+
+	if ( ! SendToChild(MSG_REQUEST_LOGS, peer, 0) )
+		return false;
+
+	return true;
+	}
+
 bool RemoteSerializer::RequestEvents(PeerID id, RE_Matcher* pattern)
 	{
 	if ( ! using_communication )
@@ -1443,6 +1482,7 @@ bool RemoteSerializer::Poll(bool may_block)
 		case MSG_PHASE_DONE:
 		case MSG_TERMINATE:
 		case MSG_DEBUG_DUMP:
+		case MSG_REQUEST_LOGS:
 			{
 			// No further argument chunk.
 			msgstate = TYPE;
@@ -1465,6 +1505,8 @@ bool RemoteSerializer::Poll(bool may_block)
 		case MSG_LOG:
 		case MSG_SYNC_POINT:
 		case MSG_REMOTE_PRINT:
+		case MSG_LOG_CREATE_WRITER:
+		case MSG_LOG_WRITE:
 			{
 			// One further argument chunk.
 			msgstate = ARGS;
@@ -1601,6 +1643,15 @@ bool RemoteSerializer::DoMessage()
 	case MSG_REMOTE_PRINT:
 		return ProcessRemotePrint();
 
+	case MSG_LOG_CREATE_WRITER:
+		return ProcessLogCreateWriter();
+
+	case MSG_LOG_WRITE:
+		return ProcessLogWrite();
+
+	case MSG_REQUEST_LOGS:
+		return ProcessRequestLogs();
+
 	default:
 		DEBUG_COMM(fmt("unexpected msg type: %d",
 					int(current_msgtype)));
@@ -1663,6 +1714,7 @@ void RemoteSerializer::PeerConnected(Peer* peer)
 	peer->cache_out->Clear();
 	peer->our_runtime = int(current_time(true) - bro_start_time);
 	peer->sync_point = 0;
+	peer->logs_requested = false;
 
 	if ( ! SendCMsgToChild(MSG_VERSION, peer) )
 		return;
@@ -1725,6 +1777,7 @@ RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port,
 	peer->orig = false;
 	peer->accept_state = false;
 	peer->send_state = false;
+	peer->logs_requested = false;
 	peer->caps = 0;
 	peer->comp_level = 0;
 	peer->suspended_processing = false;
@@ -1735,6 +1788,8 @@ RemoteSerializer::Peer* RemoteSerializer::AddPeer(uint32 ip, uint16 port,
 	peer->sync_point = 0;
 	peer->print_buffer = 0;
 	peer->print_buffer_used = 0;
+	peer->log_buffer = 0;
+	peer->log_buffer_used = 0;
 
 	peers.append(peer);
 	Log(LogInfo, "added peer", peer);
@@ -1767,6 +1822,7 @@ void RemoteSerializer::RemovePeer(Peer* peer)
 	int id = peer->id;
 	Unref(peer->val);
 	delete [] peer->print_buffer;
+	delete [] peer->log_buffer;
 	delete peer->cache_in;
 	delete peer->cache_out;
 	delete peer;
@@ -1960,6 +2016,19 @@ bool RemoteSerializer::ProcessRequestSyncMsg()
 	return true;
 	}
 
+bool RemoteSerializer::ProcessRequestLogs()
+	{
+	if ( ! current_peer )
+		return false;
+
+	Log(LogInfo, "peer requested logs", current_peer);
+
+	current_peer->logs_requested = true;
+	current_peer->log_buffer = new char[LOG_BUFFER_SIZE];
+	current_peer->log_buffer_used = 0;
+	return true;
+	}
+
 bool RemoteSerializer::ProcessPhaseDone()
 	{
 	switch ( current_peer->phase ) {
@@ -2024,6 +2093,9 @@ bool RemoteSerializer::HandshakeDone(Peer* peer)
 	if ( (peer->caps & Peer::NEW_CACHE_STRATEGY) )
 		Log(LogInfo, "peer supports keep-in-cache; using that", peer);
 
+	if ( peer->logs_requested )
+		log_mgr->SendAllWritersTo(peer->id);
+
 	if ( peer->sync_requested != Peer::NONE )
 		{
 		if ( in_sync )
@@ -2377,6 +2449,260 @@ bool RemoteSerializer::ProcessRemotePrint()
 	return true;
 	}
 
+bool RemoteSerializer::SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields)
+	{
+	loop_over_list(peers, i)
+		{
+		SendLogCreateWriter(peers[i]->id, id, writer, path, num_fields, fields);
+		}
+
+	return true;
+	}
+
+bool RemoteSerializer::SendLogCreateWriter(PeerID peer_id, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields)
+	{
+	SetErrorDescr("logging");
+
+	ChunkedIO::Chunk* c = 0;
+
+	Peer* peer = LookupPeer(peer_id, true);
+	if ( ! peer )
+		return false;
+
+	if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING )
+		return false;
+
+	BinarySerializationFormat fmt;
+
+	fmt.StartWrite();
+
+	bool success = fmt.Write(id->AsEnum(), "id") &&
+		fmt.Write(writer->AsEnum(), "writer") &&
+		fmt.Write(path, "path") &&
+		fmt.Write(num_fields, "num_fields");
+
+	if ( ! success )
+		goto error;
+
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( ! fields[i]->Write(&fmt) )
+			goto error;
+		}
+
+	c = new ChunkedIO::Chunk;
+	c->len = fmt.EndWrite(&c->data);
+
+	if ( ! SendToChild(MSG_LOG_CREATE_WRITER, peer, 0) )
+		goto error;
+
+	if ( ! SendToChild(c) )
+		goto error;
+
+	return true;
+
+error:
+	FatalError(io->Error());
+	return false;
+	}
+
+bool RemoteSerializer::SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals)
+	{
+	loop_over_list(peers, i)
+		{
+		SendLogWrite(peers[i], id, writer, path, num_fields, vals);
+		}
+
+	return true;
+	}
+
+bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals)
+	{
+	if ( peer->phase != Peer::HANDSHAKE && peer->phase != Peer::RUNNING )
+		return false;
+
+	if ( ! peer->logs_requested )
+		return false;
+
+	assert(peer->log_buffer);
+
+	// Serialize the log record entry.
+
+	BinarySerializationFormat fmt;
+
+	fmt.StartWrite();
+
+	bool success = fmt.Write(id->AsEnum(), "id") &&
+		fmt.Write(writer->AsEnum(), "writer") &&
+		fmt.Write(path, "path") &&
+		fmt.Write(num_fields, "num_fields");
+
+	if ( ! success )
+		goto error;
+
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		if ( ! vals[i]->Write(&fmt) )
+			goto error;
+		}
+
+	// Ok, we have the binary data now.
+	char* data;
+	int len;
+
+	len = fmt.EndWrite(&data);
+
+	// Do we have enough space in the buffer? If not, flush first.
+	if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) )
+		FlushLogBuffer(peer);
+
+	// If the data is actually larger than our complete buffer, just send it out.
+	if ( len > LOG_BUFFER_SIZE )
+		return SendToChild(MSG_LOG_WRITE, peer, data, len);
+
+	// Now we have space in the buffer, copy it into there.
+	memcpy(peer->log_buffer + peer->log_buffer_used, data, len);
+	peer->log_buffer_used += len;
+	assert(peer->log_buffer_used <= LOG_BUFFER_SIZE);
+
+	FlushLogBuffer(peer);
+	return false;
+
+error:
+	FatalError(io->Error());
+	return false;
+	}
+
+bool RemoteSerializer::FlushLogBuffer(Peer* p)
+	{
+	if ( p->state == Peer::CLOSING )
+		return false;
+
+	if ( ! p->log_buffer )
+		return true;
+
+	SendToChild(MSG_LOG_WRITE, p, p->log_buffer, p->log_buffer_used);
+
+	p->log_buffer = new char[LOG_BUFFER_SIZE];
+	p->log_buffer_used = 0;
+	return true;
+	}
+
+bool RemoteSerializer::ProcessLogCreateWriter()
+	{
+	if ( current_peer->state == Peer::CLOSING )
+		return false;
+
+	assert(current_args);
+
+	EnumVal* id_val = 0;
+	EnumVal* writer_val = 0;
+	LogField** fields = 0;
+
+	BinarySerializationFormat fmt;
+	fmt.StartRead(current_args->data, current_args->len);
+
+	int id, writer;
+	string path;
+	int num_fields;
+
+	bool success = fmt.Read(&id, "id") &&
+		fmt.Read(&writer, "writer") &&
+		fmt.Read(&path, "path") &&
+		fmt.Read(&num_fields, "num_fields");
+
+	if ( ! success )
+		goto error;
+
+	fields = new LogField* [num_fields];
+
+	for ( int i = 0; i < num_fields; i++ )
+		{
+		fields[i] = new LogField;
+		if ( ! fields[i]->Read(&fmt) )
+			goto error;
+		}
+
+	fmt.EndRead();
+
+	id_val = new EnumVal(id, BifType::Enum::Log::ID);
+	writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
+
+	if ( ! log_mgr->CreateWriter(id_val, writer_val, path, num_fields, fields) )
+		goto error;
+
+	Unref(id_val);
+	Unref(writer_val);
+
+	return true;
+
+error:
+	Unref(id_val);
+	Unref(writer_val);
+
+	Error("write error for creating writer");
+	return false;
+	}
+
+bool RemoteSerializer::ProcessLogWrite()
+	{
+	if ( current_peer->state == Peer::CLOSING )
+		return false;
+
+	assert(current_args);
+
+	BinarySerializationFormat fmt;
+	fmt.StartRead(current_args->data, current_args->len);
+
+	while ( fmt.BytesRead() != (int)current_args->len )
+		{
+		// Unserialize one entry.
+		EnumVal* id_val = 0;
+		EnumVal* writer_val = 0;
+		LogVal** vals = 0;
+
+		int id, writer;
+		string path;
+		int num_fields;
+
+		bool success = fmt.Read(&id, "id") &&
+			fmt.Read(&writer, "writer") &&
+			fmt.Read(&path, "path") &&
+			fmt.Read(&num_fields, "num_fields");
+
+		if ( ! success )
+			goto error;
+
+		vals = new LogVal* [num_fields];
+
+		for ( int i = 0; i < num_fields; i++ )
+			{
+			vals[i] = new LogVal;
+			if ( ! vals[i]->Read(&fmt) )
+				goto error;
+			}
+
+		id_val = new EnumVal(id, BifType::Enum::Log::ID);
+		writer_val = new EnumVal(writer, BifType::Enum::Log::Writer);
+
+		success = log_mgr->Write(id_val, writer_val, path, num_fields, vals);
+
+		Unref(id_val);
+		Unref(writer_val);
+
+		if ( ! success )
+			goto error;
+
+		}
+
+	fmt.EndRead();
+
+	return true;
+
+error:
+	Error("write error for log entry");
+	return false;
+	}
 
 void RemoteSerializer::GotEvent(const char* name, double time,
 				EventHandlerPtr event, val_list* args)
@@ -3048,6 +3374,7 @@ bool SocketComm::ProcessParentMessage()
 		case MSG_TERMINATE:
 		case MSG_PHASE_DONE:
 		case MSG_DEBUG_DUMP:
+		case MSG_REQUEST_LOGS:
 			{
 			// No further argument chunk.
 			parent_msgstate = TYPE;
@@ -3067,6 +3394,8 @@ bool SocketComm::ProcessParentMessage()
 		case MSG_CAPS:
 		case MSG_SYNC_POINT:
 		case MSG_REMOTE_PRINT:
+		case MSG_LOG_CREATE_WRITER:
+		case MSG_LOG_WRITE:
 			{
 			// One further argument chunk.
 			parent_msgstate = ARGS;
@@ -3167,18 +3496,6 @@ bool SocketComm::DoParentMessage()
 		return true;
 		}
 
-	case MSG_PHASE_DONE:
-		{
-		// No argument block follows.
-		if ( parent_peer && parent_peer->connected )
-			{
-			DEBUG_COMM("child: forwarding with MSG_PHASE_DONE to peer");
-			if ( ! SendToPeer(parent_peer, MSG_PHASE_DONE, 0) )
-				return false;
-			}
-		return true;
-		}
-
 	case MSG_LISTEN:
 		return ProcessListen();
 
@@ -3206,6 +3523,20 @@ bool SocketComm::DoParentMessage()
 		return ForwardChunkToPeer();
 		}
 
+	case MSG_PHASE_DONE:
+	case MSG_REQUEST_LOGS:
+		{
+		// No argument block follows.
+		if ( parent_peer && parent_peer->connected )
+			{
+			DEBUG_COMM(fmt("child: forwarding %s to peer", msgToStr(parent_msgtype)));
+			if ( ! SendToPeer(parent_peer, parent_msgtype, 0) )
+				return false;
+			}
+
+		return true;
+		}
+
 	case MSG_REQUEST_EVENTS:
 	case MSG_REQUEST_SYNC:
 	case MSG_SERIAL:
@@ -3214,6 +3545,8 @@ bool SocketComm::DoParentMessage()
 	case MSG_CAPS:
 	case MSG_SYNC_POINT:
 	case MSG_REMOTE_PRINT:
+	case MSG_LOG_CREATE_WRITER:
+	case MSG_LOG_WRITE:
 		assert(parent_args);
 		return ForwardChunkToPeer();
 
@@ -3331,6 +3664,7 @@ bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer)
 
 		switch ( msg->Type() ) {
 		case MSG_PHASE_DONE:
+		case MSG_REQUEST_LOGS:
 			// No further argument block.
 			DEBUG_COMM("child: forwarding with 0 args to parent");
 			if ( ! SendToParent(msg->Type(), peer, 0) )
@@ -3387,6 +3721,8 @@ bool SocketComm::ProcessRemoteMessage(SocketComm::Peer* peer)
 	case MSG_CAPS:
 	case MSG_SYNC_POINT:
 	case MSG_REMOTE_PRINT:
+	case MSG_LOG_CREATE_WRITER:
+	case MSG_LOG_WRITE:
 		{
 		// Messages with one further argument block which we simply
 		// forward to our parent.

src/RemoteSerializer.h

@@ -16,6 +16,8 @@
 // FIXME: Change this to network byte order
 
 class IncrementalSendTimer;
+class LogField;
+class LogVal;
 
 // This class handles the communication done in Bro's main loop.
 class RemoteSerializer : public Serializer, public IOSource {
@@ -42,6 +44,9 @@ public:
 	// the peer right after the handshake.
 	bool RequestSync(PeerID peer, bool auth);
 
+	// Requests logs from the remote side.
+	bool RequestLogs(PeerID id);
+
 	// Sets flag whether we're accepting state from this peer
 	// (default: yes).
 	bool SetAcceptState(PeerID peer, bool accept);
@@ -92,6 +97,15 @@ public:
 	// Broadcast remote print.
 	bool SendPrintHookEvent(BroFile* f, const char* txt);
 
+	// Send a request to create a writer on a remote side.
+	bool SendLogCreateWriter(PeerID peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields);
+
+	// Broadcasts a request to create a writer.
+	bool SendLogCreateWriter(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogField* const * fields);
+
+	// Broadcast a log entry to everybody interested.
+	bool SendLogWrite(EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals);
+
 	// Synchronzizes time with all connected peers. Returns number of
 	// current sync-point, or -1 on error.
 	uint32 SendSyncPoint();
@@ -205,6 +219,7 @@ protected:
 		bool accept_state;	// True if we accept state from peer.
 		bool send_state; // True if we're supposed to initially sent our state.
 		int comp_level; // Compression level.
+		bool logs_requested; // True if the peer has requested logs.
 
 		// True if this peer triggered a net_suspend_processing().
 		bool suspended_processing;
@@ -217,6 +232,8 @@ protected:
 		uint32 sync_point;	// Highest sync-point received so far
 		char* print_buffer;	// Buffer for remote print or null.
 		int print_buffer_used;	// Number of bytes used in buffer.
+		char* log_buffer;	// Buffer for remote log or null.
+		int log_buffer_used;	// Number of bytes used in buffer.
 	};
 
 	// Shuts down remote serializer.
@@ -255,6 +272,9 @@ protected:
 	bool ProcessCapsMsg();
 	bool ProcessSyncPointMsg();
 	bool ProcessRemotePrint();
+	bool ProcessLogCreateWriter();
+	bool ProcessLogWrite();
+	bool ProcessRequestLogs();
 
 	Peer* AddPeer(uint32 ip, uint16 port, PeerID id = PEER_NONE);
 	Peer* LookupPeer(PeerID id, bool only_if_connected);
@@ -282,11 +302,13 @@ protected:
 	bool SendID(SerialInfo* info, Peer* peer, const ID& id);
 	bool SendCapabilities(Peer* peer);
 	bool SendPacket(SerialInfo* info, Peer* peer, const Packet& p);
+	bool SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, string path, int num_fields, const LogVal* const * vals);
 
 	void UnregisterHandlers(Peer* peer);
 	void RaiseEvent(EventHandlerPtr event, Peer* peer, const char* arg = 0);
 	bool EnterPhaseRunning(Peer* peer);
 	bool FlushPrintBuffer(Peer* p);
+	bool FlushLogBuffer(Peer* p);
 
 	void ChildDied();
 	void InternalCommError(const char* msg);

src/Scope.cc

@@ -113,9 +113,11 @@ TraversalCode Scope::Traverse(TraversalCallback* cb) const
 	}
 
 
-ID* lookup_ID(const char* name, const char* curr_module, bool no_global)
+ID* lookup_ID(const char* name, const char* curr_module, bool no_global,
+	      bool same_module_only)
 	{
 	string fullname = make_full_var_name(curr_module, name);
+
 	string ID_module = extract_module_name(fullname.c_str());
 	bool need_export = ID_module != GLOBAL_MODULE_NAME &&
 				ID_module != curr_module;
@@ -134,7 +136,8 @@ ID* lookup_ID(const char* name, const char* curr_module, bool no_global)
 			}
 		}
 
-	if ( ! no_global )
+	if ( ! no_global && (strcmp(GLOBAL_MODULE_NAME, curr_module) == 0 ||
+			     ! same_module_only) )
 		{
 		string globalname = make_full_var_name(GLOBAL_MODULE_NAME, name);
 		ID* id = global_scope()->Lookup(globalname.c_str());

src/Scope.h

@@ -65,7 +65,7 @@ extern bool in_debug;
 
 // If no_global is true, don't search in the default "global" namespace.
 extern ID* lookup_ID(const char* name, const char* module,
-			bool no_global = false);
+		     bool no_global = false, bool same_module_only=false);
 extern ID* install_ID(const char* name, const char* module_name,
 			bool is_global, bool is_export);
 

src/SerialTypes.h

@@ -146,6 +146,7 @@ SERIAL_EXPR(TABLE_CONSTRUCTOR_EXPR, 40)
 SERIAL_EXPR(SET_CONSTRUCTOR_EXPR, 41)
 SERIAL_EXPR(VECTOR_CONSTRUCTOR_EXPR, 42)
 SERIAL_EXPR(TABLE_COERCE_EXPR, 43)
+SERIAL_EXPR(VECTOR_COERCE_EXPR, 44)
 
 #define SERIAL_STMT(name, val) SERIAL_CONST(name, val, STMT)
 SERIAL_STMT(STMT, 1)

src/SerializationFormat.cc

@@ -44,6 +44,7 @@ void SerializationFormat::StartWrite()
 
 	output_pos = 0;
 	bytes_written = 0;
+	bytes_read = 0;
 	}
 
 uint32 SerializationFormat::EndWrite(char** data)
@@ -64,6 +65,8 @@ bool SerializationFormat::ReadData(void* b, size_t count)
 
 	memcpy(b, input + input_pos, count);
 	input_pos += count;
+	bytes_read += count;
+
 	return true;
 	}
 
@@ -214,6 +217,20 @@ bool BinarySerializationFormat::Read(char** str, int* len, const char* tag)
 	return true;
 	}
 
+bool BinarySerializationFormat::Read(string* v, const char* tag)
+	{
+	char* buffer;
+	int len;
+
+	if ( ! Read(&buffer, &len, tag) )
+		return false;
+
+	*v = string(buffer, len);
+
+	delete [] buffer;
+	return true;
+	}
+
 bool BinarySerializationFormat::Write(char v, const char* tag)
 	{
 	DBG_LOG(DBG_SERIAL, "Write char %s [%s]", fmt_bytes(&v, 1), tag);
@@ -278,6 +295,11 @@ bool BinarySerializationFormat::Write(const char* s, const char* tag)
 	return Write(s, strlen(s), tag);
 	}
 
+bool BinarySerializationFormat::Write(const string& s, const char* tag)
+	{
+	return Write(s.data(), s.size(), tag);
+	}
+
 bool BinarySerializationFormat::WriteOpenTag(const char* tag)
 	{
 	return true;
@@ -362,6 +384,12 @@ bool XMLSerializationFormat::Read(char** str, int* len, const char* tag)
 	return false;
 	}
 
+bool XMLSerializationFormat::Read(string* s, const char* tag)
+	{
+	internal_error("no reading of xml");
+	return false;
+	}
+
 bool XMLSerializationFormat::Write(char v, const char* tag)
 	{
 	return WriteElem(tag, "char", &v, 1);
@@ -416,6 +444,11 @@ bool XMLSerializationFormat::Write(const char* s, const char* tag)
 	return WriteElem(tag, "string", s, strlen(s));
 	}
 
+bool XMLSerializationFormat::Write(const string& s, const char* tag)
+	{
+	return WriteElem(tag, "string", s.data(), s.size());
+	}
+
 bool XMLSerializationFormat::WriteOpenTag(const char* tag)
 	{
 	return WriteData("<", 1) && WriteData(tag, strlen(tag) && WriteData(">", 1));

src/SerializationFormat.h

@@ -5,6 +5,10 @@
 #ifndef SERIALIZATION_FORMAT
 #define SERIALIZATION_FORMAT
 
+#include <string>
+
+using namespace std;
+
 #include "util.h"
 
 // Abstract base class.
@@ -25,6 +29,10 @@ public:
 	virtual bool Read(char* v, const char* tag) = 0;
 	virtual bool Read(bool* v, const char* tag) = 0;
 	virtual bool Read(double* d, const char* tag) = 0;
+	virtual bool Read(string* s, const char* tag) = 0;
+
+	// Returns number of raw bytes read since last call to StartRead().
+	int BytesRead() const	{ return bytes_read; }
 
 	// Passes ownership of string.
 	virtual bool Read(char** str, int* len, const char* tag) = 0;
@@ -43,6 +51,7 @@ public:
 	virtual bool Write(double d, const char* tag) = 0;
 	virtual bool Write(const char* s, const char* tag) = 0;
 	virtual bool Write(const char* buf, int len, const char* tag) = 0;
+	virtual bool Write(const string& s, const char* tag) = 0;
 
 	virtual bool WriteOpenTag(const char* tag) = 0;
 	virtual bool WriteCloseTag(const char* tag) = 0;
@@ -65,6 +74,7 @@ protected:
 	uint32 input_pos;
 
 	int bytes_written;
+	int bytes_read;
 };
 
 class BinarySerializationFormat : public SerializationFormat {
@@ -81,6 +91,7 @@ public:
 	virtual bool Read(bool* v, const char* tag);
 	virtual bool Read(double* d, const char* tag);
 	virtual bool Read(char** str, int* len, const char* tag);
+	virtual bool Read(string* s, const char* tag);
 	virtual bool Write(int v, const char* tag);
 	virtual bool Write(uint16 v, const char* tag);
 	virtual bool Write(uint32 v, const char* tag);
@@ -91,6 +102,7 @@ public:
 	virtual bool Write(double d, const char* tag);
 	virtual bool Write(const char* s, const char* tag);
 	virtual bool Write(const char* buf, int len, const char* tag);
+	virtual bool Write(const string& s, const char* tag);
 	virtual bool WriteOpenTag(const char* tag);
 	virtual bool WriteCloseTag(const char* tag);
 	virtual bool WriteSeparator();
@@ -112,6 +124,7 @@ public:
 	virtual bool Write(double d, const char* tag);
 	virtual bool Write(const char* s, const char* tag);
 	virtual bool Write(const char* buf, int len, const char* tag);
+	virtual bool Write(const string& s, const char* tag);
 	virtual bool WriteOpenTag(const char* tag);
 	virtual bool WriteCloseTag(const char* tag);
 	virtual bool WriteSeparator();
@@ -126,6 +139,7 @@ public:
 	virtual bool Read(bool* v, const char* tag);
 	virtual bool Read(double* d, const char* tag);
 	virtual bool Read(char** str, int* len, const char* tag);
+	virtual bool Read(string* s, const char* tag);
 
 private:
 	// Encodes non-printable characters.

src/StateAccess.cc

@@ -935,7 +935,7 @@ void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier)
 		attr_list* a = new attr_list;
 		Attr* attr = new Attr(ATTR_TRACKED);
 		a->append(attr);
-		id->SetAttrs(new Attributes(a, id->Type()));
+		id->SetAttrs(new Attributes(a, id->Type(), false));
 		Unref(attr);
 		}
 

src/Type.cc

@@ -33,6 +33,7 @@ const char* type_name(TypeTag t)
 		"func",
 		"file",
 		"vector",
+		"type",
 		"error",
 	};
 
@@ -102,6 +103,7 @@ BroType::BroType(TypeTag t, bool arg_base_type)
 	case TYPE_FUNC:
 	case TYPE_FILE:
 	case TYPE_VECTOR:
+	case TYPE_TYPE:
 		internal_tag = TYPE_INTERNAL_OTHER;
 		break;
 
@@ -781,10 +783,10 @@ bool FuncType::DoUnserialize(UnserialInfo* info)
 	return UNSERIALIZE(&is_event);
 	}
 
-TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs)
+TypeDecl::TypeDecl(BroType* t, const char* i, attr_list* arg_attrs, bool in_record)
 	{
 	type = t;
-	attrs = arg_attrs ? new Attributes(arg_attrs, t) : 0;
+	attrs = arg_attrs ? new Attributes(arg_attrs, t, in_record) : 0;
 	id = i;
 	}
 
@@ -902,6 +904,8 @@ RecordType::RecordType(TypeList* arg_base, type_decl_list* refinements)
 
 void RecordType::Init(TypeList* arg_base)
 	{
+	assert(false);  // Is this ever used?
+
 	base = arg_base;
 
 	if ( ! base )
@@ -1066,6 +1070,46 @@ void RecordType::DescribeReST(ODesc* d) const
 	DescribeFieldsReST(d, false);
 	}
 
+const char* RecordType::AddFields(type_decl_list* others, attr_list* attr)
+	{
+	assert(types);
+
+	bool log = false;
+
+	if ( attr )
+		{
+		loop_over_list(*attr, j)
+			{
+			if ( (*attr)[j]->Tag() == ATTR_LOG )
+				log = true;
+			}
+		}
+
+	loop_over_list(*others, i)
+		{
+		TypeDecl* td = (*others)[i];
+
+		if ( ! td->FindAttr(ATTR_DEFAULT) &&
+		     ! td->FindAttr(ATTR_OPTIONAL) )
+			return "extension field must be &optional or have &default";
+
+		if ( log )
+			{
+			if ( ! td->attrs )
+				td->attrs = new Attributes(new attr_list, td->type, true);
+
+			td->attrs->AddAttr(new Attr(ATTR_LOG));
+			}
+
+		types->append(td);
+		}
+
+	delete others;
+
+	num_fields = types->length();
+	return 0;
+	}
+
 void RecordType::DescribeFields(ODesc* d) const
 	{
 	if ( d->IsReadable() )
@@ -1518,6 +1562,11 @@ int VectorType::MatchesIndex(ListExpr*& index) const
 				MATCHES_INDEX_SCALAR : DOES_NOT_MATCH_INDEX;
 	}
 
+bool VectorType::IsUnspecifiedVector() const
+	{
+	return yield_type->Tag() == TYPE_ANY;
+	}
+
 IMPLEMENT_SERIAL(VectorType, SER_VECTOR_TYPE);
 
 bool VectorType::DoSerialize(SerialInfo* info) const
@@ -1591,7 +1640,9 @@ static int is_init_compat(const BroType* t1, const BroType* t2)
 
 int same_type(const BroType* t1, const BroType* t2, int is_init)
 	{
-	if ( t1 == t2 )
+	if ( t1 == t2 ||
+	     t1->Tag() == TYPE_ANY ||
+	     t2->Tag() == TYPE_ANY )
 		return 1;
 
 	t1 = flatten_type(t1);
@@ -1718,12 +1769,26 @@ int same_type(const BroType* t1, const BroType* t2, int is_init)
 	case TYPE_FILE:
 		return same_type(t1->YieldType(), t2->YieldType(), is_init);
 
+	case TYPE_TYPE:
+		return same_type(t1, t2, is_init);
+
 	case TYPE_UNION:
 		error("union type in same_type()");
 	}
 	return 0;
 	}
 
+int same_attrs(const Attributes* a1, const Attributes* a2)
+	{
+	if ( ! a1 )
+		return (a2 == 0);
+
+	if ( ! a2 )
+		return (a1 == 0);
+
+	return (*a1 == *a2);
+	}
+
 int record_promotion_compatible(const RecordType* /* super_rec */,
 				const RecordType* /* sub_rec */)
 	{
@@ -1795,6 +1860,7 @@ int is_assignable(BroType* t)
 	case TYPE_VECTOR:
 	case TYPE_FILE:
 	case TYPE_TABLE:
+	case TYPE_TYPE:
 		return 1;
 
 	case TYPE_VOID:
@@ -1862,6 +1928,7 @@ BroType* merge_types(const BroType* t1, const BroType* t2)
 	case TYPE_ADDR:
 	case TYPE_NET:
 	case TYPE_SUBNET:
+	case TYPE_BOOL:
 	case TYPE_ANY:
 	case TYPE_ERROR:
 		return base_type(tg1);

src/Type.h

@@ -32,6 +32,7 @@ typedef enum {
 	TYPE_FUNC,
 	TYPE_FILE,
 	TYPE_VECTOR,
+	TYPE_TYPE,
 	TYPE_ERROR
 #define NUM_TYPES (int(TYPE_ERROR) + 1)
 } TypeTag;
@@ -60,6 +61,7 @@ class ListExpr;
 class EnumType;
 class Serializer;
 class VectorType;
+class TypeType;
 
 const int DOES_NOT_MATCH_INDEX = 0;
 const int MATCHES_INDEX_SCALAR = 1;
@@ -153,6 +155,7 @@ public:
 		CHECK_TYPE_TAG(TYPE_SUBNET, "BroType::AsSubNetType");
 		return (const SubNetType*) this;
 		}
+
 	SubNetType* AsSubNetType()
 		{
 		CHECK_TYPE_TAG(TYPE_SUBNET, "BroType::AsSubNetType");
@@ -194,6 +197,18 @@ public:
 		return (VectorType*) this;
 		}
 
+	const TypeType* AsTypeType() const
+	        {
+		CHECK_TYPE_TAG(TYPE_TYPE, "BroType::AsTypeType");
+		return (TypeType*) this;
+		}
+
+	TypeType* AsTypeType()
+	        {
+		CHECK_TYPE_TAG(TYPE_TYPE, "BroType::AsTypeType");
+		return (TypeType*) this;
+		}
+
 	int IsSet() const
 		{
 		return tag == TYPE_TABLE && (YieldType() == 0);
@@ -371,9 +386,22 @@ protected:
 	ID* return_value;
 };
 
+class TypeType : public BroType {
+public:
+	TypeType(BroType* t) : BroType(TYPE_TYPE)	{ type = t->Ref(); }
+	~TypeType()	{ Unref(type); }
+
+	BroType* Type()	{ return type; }
+
+protected:
+	TypeType()	{}
+
+	BroType* type;
+};
+
 class TypeDecl {
 public:
-	TypeDecl(BroType* t, const char* i, attr_list* attrs = 0);
+	TypeDecl(BroType* t, const char* i, attr_list* attrs = 0, bool in_record = false);
 	virtual ~TypeDecl();
 
 	const Attr* FindAttr(attr_tag a) const
@@ -434,6 +462,10 @@ public:
 
 	int NumFields() const			{ return num_fields; }
 
+	// Returns 0 if all is ok, otherwise a pointer to an error message.
+	// Takes ownership of list.
+	const char* AddFields(type_decl_list* types, attr_list* attr);
+
 	void Describe(ODesc* d) const;
 	void DescribeReST(ODesc* d) const;
 	void DescribeFields(ODesc* d) const;
@@ -542,6 +574,10 @@ public:
 
 	int MatchesIndex(ListExpr*& index) const;
 
+	// Returns true if this table type is "unspecified", which is what one
+	// gets using an empty "vector()" constructor.
+	bool IsUnspecifiedVector() const;
+
 protected:
 	VectorType()	{ yield_type = 0; }
 
@@ -564,6 +600,9 @@ inline BroType* error_type()	{ return base_type(TYPE_ERROR); }
 // test is done in the context of an initialization.
 extern int same_type(const BroType* t1, const BroType* t2, int is_init=0);
 
+// True if the two attribute lists are equivalent.
+extern int same_attrs(const Attributes* a1, const Attributes* a2);
+
 // Returns true if the record sub_rec can be promoted to the record
 // super_rec.
 extern int record_promotion_compatible(const RecordType* super_rec,

src/Val.cc

@@ -2042,7 +2042,23 @@ Val* TableVal::Default(Val* index)
 		return 0;
 
 	if ( ! def_val )
-		def_val = def_attr->AttrExpr()->Eval(0);
+		{
+		BroType* ytype = Type()->YieldType();
+		BroType* dtype = def_attr->AttrExpr()->Type();
+
+		if ( dtype->Tag() == TYPE_RECORD && ytype->Tag() == TYPE_RECORD &&
+		     ! same_type(dtype, ytype) &&
+		     record_promotion_compatible(dtype->AsRecordType(),
+						 ytype->AsRecordType()) )
+			{
+			Expr* coerce = new RecordCoerceExpr(def_attr->AttrExpr(), ytype->AsRecordType());
+			def_val = coerce->Eval(0);
+			Unref(coerce);
+			}
+
+		else
+			def_val = def_attr->AttrExpr()->Eval(0);
+		}
 
 	if ( ! def_val )
 		{
@@ -2912,6 +2928,83 @@ Val* RecordVal::Lookup(int field) const
 	return (*AsRecord())[field];
 	}
 
+Val* RecordVal::LookupWithDefault(int field) const
+	{
+	Val* val = (*AsRecord())[field];
+
+	if ( val )
+		return val->Ref();
+
+	// Check for &default.
+	const Attr* def_attr =
+		record_type->FieldDecl(field)->attrs->FindAttr(ATTR_DEFAULT);
+
+	return def_attr ? def_attr->AttrExpr()->Eval(0) : 0;
+	}
+
+RecordVal* RecordVal::CoerceTo(const RecordType* t, Val* aggr) const
+	{
+	if ( ! record_promotion_compatible(t->AsRecordType(), Type()->AsRecordType()) )
+		return 0;
+
+	if ( ! aggr )
+		aggr = new RecordVal(const_cast<RecordType*>(t->AsRecordType()));
+
+	RecordVal* ar = aggr->AsRecordVal();
+	RecordType* ar_t = aggr->Type()->AsRecordType();
+
+	const RecordType* rv_t = Type()->AsRecordType();
+
+	int i;
+	for ( i = 0; i < rv_t->NumFields(); ++i )
+		{
+		int t_i = ar_t->FieldOffset(rv_t->FieldName(i));
+
+		if ( t_i < 0 )
+			{
+			char buf[512];
+			safe_snprintf(buf, sizeof(buf),
+					"orphan field \"%s\" in initialization",
+					rv_t->FieldName(i));
+			Error(buf);
+			break;
+			}
+
+		if ( ar_t->FieldType(t_i)->Tag() == TYPE_RECORD
+				&& ! same_type(ar_t->FieldType(t_i), Lookup(i)->Type()) )
+			{
+			Expr* rhs = new ConstExpr(Lookup(i)->Ref());
+			Expr* e = new RecordCoerceExpr(rhs, ar_t->FieldType(t_i)->AsRecordType());
+			ar->Assign(t_i, e->Eval(0));
+			}
+
+		ar->Assign(t_i, Lookup(i)->Ref());
+		}
+
+	for ( i = 0; i < ar_t->NumFields(); ++i )
+		if ( ! ar->Lookup(i) &&
+			 ! ar_t->FieldDecl(i)->FindAttr(ATTR_OPTIONAL) )
+			{
+			char buf[512];
+			safe_snprintf(buf, sizeof(buf),
+					"non-optional field \"%s\" missing in initialization", ar_t->FieldName(i));
+			Error(buf);
+			}
+
+	return ar;
+	}
+
+RecordVal* RecordVal::CoerceTo(RecordType* t)
+	{
+	if ( same_type(Type(), t) )
+		{
+		this->Ref();
+		return this;
+		}
+
+	return CoerceTo(t, 0);
+	}
+
 void RecordVal::Describe(ODesc* d) const
 	{
 	const val_list* vl = AsRecord();
@@ -3340,6 +3433,10 @@ Val* check_and_promote(Val* v, const BroType* t, int is_init)
 	TypeTag t_tag = t->Tag();
 	TypeTag v_tag = vt->Tag();
 
+	// More thought definitely needs to go into this.
+	if ( t_tag == TYPE_ANY || v_tag == TYPE_ANY )
+		return v;
+
 	if ( ! EitherArithmetic(t_tag, v_tag) ||
 	     /* allow sets as initializers */
 	     (is_init && v_tag == TYPE_TABLE) )

src/Val.h

@@ -39,6 +39,7 @@ class TableVal;
 class RecordVal;
 class ListVal;
 class StringVal;
+class EnumVal;
 class MutableVal;
 
 class StateAccess;
@@ -143,6 +144,15 @@ public:
 	// class has ref'd it.
 	Val(BroFile* f);
 
+	Val(BroType* t, bool type_type) // Extra arg to differentiate from protected version.
+		{
+		type = new TypeType(t->Ref());
+		attribs = 0;
+#ifdef DEBUG
+		bound_id = 0;
+#endif
+		}
+
 	Val()
 		{
 		val.int_val = 0;
@@ -225,6 +235,12 @@ public:
 		return &val.subnet_val;
 		}
 
+	BroType* AsType() const
+		{
+		CHECK_TAG(type->Tag(), TYPE_TYPE, "Val::Type", type_name)
+		return type;
+		}
+
 	// ... in network byte order
 	const addr_type AsAddr() const
 		{
@@ -275,6 +291,7 @@ public:
 	CONVERTER(TYPE_LIST, ListVal*, AsListVal)
 	CONVERTER(TYPE_STRING, StringVal*, AsStringVal)
 	CONVERTER(TYPE_VECTOR, VectorVal*, AsVectorVal)
+	CONVERTER(TYPE_ENUM, EnumVal*, AsEnumVal)
 
 #define CONST_CONVERTER(tag, ctype, name) \
 	const ctype name() const \
@@ -894,7 +911,8 @@ public:
 		{ return new Val(record_type->NumFields(), TYPE_COUNT); }
 
 	void Assign(int field, Val* new_val, Opcode op = OP_ASSIGN);
-	Val* Lookup(int field) const;
+	Val* Lookup(int field) const;	// Does not Ref() value.
+	Val* LookupWithDefault(int field) const;	// Does Ref() value.
 
 	void Describe(ODesc* d) const;
 
@@ -903,6 +921,17 @@ public:
 	void SetOrigin(BroObj* o)	{ origin = o; }
 	BroObj* GetOrigin() const	{ return origin; }
 
+	// Returns a new value representing the value coerced to the given
+	// type. If coercion is not possible, returns 0. The non-const
+	// version may return the current value ref'ed if its type matches
+	// directly.
+	//
+	// *aggr* is optional; if non-zero, we add to it. See
+	// Expr::InitVal(). We leave it out in the non-const version to make
+	// the choice unambigious.
+	RecordVal* CoerceTo(const RecordType* other, Val* aggr) const;
+	RecordVal* CoerceTo(RecordType* other);
+
 	unsigned int MemoryAllocation() const;
 	void DescribeReST(ODesc* d) const;
 

src/Var.cc

@@ -109,7 +109,7 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init,
 	id->SetType(t);
 
 	if ( attr )
-		id->AddAttrs(new Attributes(attr, t));
+		id->AddAttrs(new Attributes(attr, t, false));
 
 	if ( id->FindAttr(ATTR_PERSISTENT) || id->FindAttr(ATTR_SYNCHRONIZED) )
 		{
@@ -172,6 +172,16 @@ static void make_var(ID* id, BroType* t, init_class c, Expr* init,
 		}
 
 	id->UpdateValAttrs();
+
+	if ( t && t->Tag() == TYPE_FUNC && t->AsFuncType()->IsEvent() )
+		{
+		// For events, add a function value (without any body) here so that
+		// we can later access the ID even if no implementations have been
+		// defined.
+		Func* f = new BroFunc(id, 0, 0, 0);
+		id->SetVal(new Val(f));
+		id->SetConst();
+		}
 	}
 
 
@@ -256,7 +266,7 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */)
 	id->MakeType();
 
 	if ( attr )
-		id->SetAttrs(new Attributes(attr, tnew));
+		id->SetAttrs(new Attributes(attr, tnew, false));
 	}
 
 void begin_func(ID* id, const char* module_name, function_flavor flavor,

src/bro.bif

@@ -360,6 +360,26 @@ function cat%(...%): string
 	return new StringVal(s);
 	%}
 
+function record_type_to_vector%(rt: string%): string_vec
+	%{
+	VectorVal* result =
+		new VectorVal(internal_type("string_vec")->AsVectorType());
+
+	RecordType *type = internal_type(rt->CheckString())->AsRecordType();
+
+	if ( type )
+		{
+		for ( int i = 0; i < type->NumFields(); ++i )
+			{
+			StringVal* val = new StringVal(type->FieldName(i));
+			result->Assign(i+1, val, 0);
+			}
+		}
+
+	return result;
+	%}
+
+
 function cat_sep%(sep: string, def: string, ...%): string
 	%{
 	ODesc d;
@@ -2086,6 +2106,12 @@ function request_remote_sync%(p: event_peer, auth: bool%) : bool
 	return new Val(remote_serializer->RequestSync(id, auth), TYPE_BOOL);
 	%}
 
+function request_remote_logs%(p: event_peer%) : bool
+	%{
+	RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount();
+	return new Val(remote_serializer->RequestLogs(id), TYPE_BOOL);
+	%}
+
 function set_accept_state%(p: event_peer, accept: bool%) : bool
 	%{
 	RemoteSerializer::PeerID id = p->AsRecordVal()->Lookup(0)->AsCount();

src/logging.bif

@@ -0,0 +1,76 @@
+# Internal functions and types used by the logging framework.
+
+module Log;
+
+%%{
+#include "LogMgr.h"
+#include "NetVar.h"
+%%}
+
+type Filter: record;
+type Stream: record;
+type RotationInfo: record;
+type RotationControl: record;
+
+const Log::rotation_control: RotationControl;
+
+function Log::__create_stream%(id: Log::ID, stream: Log::Stream%) : bool
+	%{
+	bool result = log_mgr->CreateStream(id->AsEnumVal(), stream->AsRecordVal());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__enable_stream%(id: Log::ID%) : bool
+	%{
+	bool result = log_mgr->EnableStream(id->AsEnumVal());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__disable_stream%(id: Log::ID%) : bool
+	%{
+	bool result = log_mgr->DisableStream(id->AsEnumVal());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__add_filter%(id: Log::ID, filter: Log::Filter%) : bool
+	%{
+	bool result = log_mgr->AddFilter(id->AsEnumVal(), filter->AsRecordVal());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__remove_filter%(id: Log::ID, name: string%) : bool
+	%{
+	bool result = log_mgr->RemoveFilter(id->AsEnumVal(), name);
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__write%(id: Log::ID, columns: any%) : bool
+	%{
+	bool result = log_mgr->Write(id->AsEnumVal(), columns->AsRecordVal());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__set_buf%(id: Log::ID, buffered: bool%): bool
+	%{
+	bool result = log_mgr->SetBuf(id->AsEnumVal(), buffered);
+	return new Val(result, TYPE_BOOL);
+	%}
+
+function Log::__flush%(id: Log::ID%): bool
+	%{
+	bool result = log_mgr->Flush(id->AsEnumVal());
+	return new Val(result, TYPE_BOOL);
+	%}
+
+# Options for the ASCII writer.
+
+module LogAscii;
+
+const output_to_stdout: bool;
+const include_header: bool;
+const header_prefix: string;
+const separator: string;
+const set_separator: string;
+const empty_field: string;
+const unset_field: string;
+

src/main.cc

@@ -30,6 +30,7 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 #include "Event.h"
 #include "File.h"
 #include "Logger.h"
+#include "LogMgr.h"
 #include "Net.h"
 #include "NetVar.h"
 #include "Var.h"
@@ -71,6 +72,7 @@ name_list prefixes;
 DNS_Mgr* dns_mgr;
 TimerMgr* timer_mgr;
 Logger* bro_logger;
+LogMgr* log_mgr;
 Func* alarm_hook = 0;
 Stmt* stmts;
 EventHandlerPtr bro_signal = 0;
@@ -288,6 +290,7 @@ void terminate_bro()
 	delete conn_compressor;
 	delete remote_serializer;
 	delete dpm;
+	delete log_mgr;
 	}
 
 void termination_signal()
@@ -702,7 +705,8 @@ int main(int argc, char** argv)
 
 	persistence_serializer = new PersistenceSerializer();
 	remote_serializer = new RemoteSerializer();
-	event_registry = new EventRegistry;
+	event_registry = new EventRegistry();
+	log_mgr = new LogMgr();
 
 	if ( events_file )
 		event_player = new EventPlayer(events_file);

src/parse.y

@@ -3,7 +3,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 %}
 
-%expect 81
+%expect 85
 
 %token TOK_ADD TOK_ADD_TO TOK_ADDR TOK_ALARM TOK_ANY
 %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF
@@ -24,7 +24,7 @@
 %token TOK_ATTR_EXPIRE_CREATE TOK_ATTR_EXPIRE_READ TOK_ATTR_EXPIRE_WRITE
 %token TOK_ATTR_PERSISTENT TOK_ATTR_SYNCHRONIZED
 %token TOK_ATTR_DISABLE_PRINT_HOOK TOK_ATTR_RAW_OUTPUT TOK_ATTR_MERGEABLE
-%token TOK_ATTR_PRIORITY TOK_ATTR_GROUP
+%token TOK_ATTR_PRIORITY TOK_ATTR_GROUP TOK_ATTR_LOG
 
 %token TOK_DEBUG
 
@@ -45,7 +45,7 @@
 
 %type <str> TOK_ID TOK_PATTERN_TEXT single_pattern TOK_DOC TOK_POST_DOC
 %type <str_l> opt_doc_list opt_post_doc_list
-%type <id> local_id global_id event_id global_or_event_id resolve_id begin_func
+%type <id> local_id global_id def_global_id event_id global_or_event_id resolve_id begin_func
 %type <id_l> local_id_list
 %type <ic> init_class
 %type <expr> opt_init
@@ -112,8 +112,10 @@ extern Expr* g_curr_debug_expr;
 
 Expr* bro_this = 0;
 int in_init = 0;
+int in_record = 0;
 bool in_debug = false;
 bool resolving_global_ID = false;
+bool defining_global_ID = false;
 
 ID* func_id = 0;
 EnumType *cur_enum_type = 0;
@@ -464,6 +466,12 @@ expr:
 				$$ = $2;
 			}
 
+	|	'[' ']'
+			{
+			// We interpret this as an empty record constructor.
+			$$ = new RecordConstructorExpr(new ListExpr);
+			}
+
 
 	|	TOK_RECORD '(' expr_list ')'
 			{
@@ -805,7 +813,11 @@ type:
 				$$ = new SetType($3, 0);
 				}
 
-	|	TOK_RECORD '{' { do_doc_token_start(); } type_decl_list '}'
+	|	TOK_RECORD '{'
+			{ ++in_record; do_doc_token_start(); }
+		type_decl_list
+			{ --in_record; }
+		'}'
 				{
 				do_doc_token_stop();
 				set_location(@1, @5);
@@ -938,7 +950,7 @@ type_decl:
 					$4, $2, a_copy, concat_opt_docs($1, $7));
 				}
 
-			$$ = new TypeDecl($4, $2, $5);
+			$$ = new TypeDecl($4, $2, $5, (in_record > 0));
 			}
 	;
 
@@ -980,7 +992,7 @@ decl:
 	|	TOK_EXPORT '{' { is_export = true; } decl_list '}'
 			{ is_export = false; }
 
-	|	TOK_GLOBAL global_id opt_type init_class opt_init opt_attr ';'
+	|	TOK_GLOBAL def_global_id opt_type init_class opt_init opt_attr ';'
 			{
 			add_global($2, $3, $4, $5, $6, VAR_REGULAR);
 
@@ -1005,7 +1017,7 @@ decl:
 				}
 			}
 
-	|	TOK_CONST global_id opt_type init_class opt_init opt_attr ';'
+	|	TOK_CONST def_global_id opt_type init_class opt_init opt_attr ';'
 			{
 			add_global($2, $3, $4, $5, $6, VAR_CONST);
 
@@ -1054,6 +1066,25 @@ decl:
 				}
 			}
 
+	|	TOK_REDEF TOK_RECORD global_id TOK_ADD_TO
+			'{' type_decl_list '}' opt_attr ';'
+			{
+			if ( ! $3->Type() )
+				$3->Error("unknown identifier");
+			else
+				{
+				RecordType* add_to = $3->Type()->AsRecordType();
+				if ( ! add_to )
+					$3->Error("not a record type");
+				else
+					{
+					const char* error = add_to->AddFields($6, $8);
+					if ( error )
+					$3->Error(error);
+					}
+				}
+			}
+
 	|	TOK_TYPE global_id ':' refined_type opt_attr ';'
 			{
 			add_type($2, $4, $5, 0);
@@ -1113,7 +1144,7 @@ conditional:
 	;
 
 func_hdr:
-		TOK_FUNCTION global_id func_params
+		TOK_FUNCTION def_global_id func_params
 			{
 			begin_func($2, current_module.c_str(),
 				FUNC_FLAVOR_FUNCTION, 0, $3);
@@ -1263,6 +1294,8 @@ attr:
 			{ $$ = new Attr(ATTR_PRIORITY, $3); }
 	|	TOK_ATTR_GROUP '=' expr
 			{ $$ = new Attr(ATTR_GROUP, $3); }
+	|	TOK_ATTR_LOG
+			{ $$ = new Attr(ATTR_LOG); }
 	;
 
 stmt:
@@ -1501,6 +1534,11 @@ global_id:
 		{ $$ = $2; }
 	;
 
+def_global_id:
+	{ defining_global_ID = 1; } global_id { defining_global_ID = 0; } 
+		{ $$ = $2; }
+	;
+
 event_id:
 	{ resolving_global_ID = 0; } global_or_event_id
 		{ $$ = $2; }
@@ -1511,7 +1549,7 @@ global_or_event_id:
 			{
 			set_location(@1);
 
-			$$ = lookup_ID($1, current_module.c_str(), false);
+			$$ = lookup_ID($1, current_module.c_str(), false, defining_global_ID);
 			if ( $$ )
 				{
 				if ( ! $$->IsGlobal() )

src/scan.l

@@ -296,6 +296,7 @@ when	return TOK_WHEN;
 &encrypt	return TOK_ATTR_ENCRYPT;
 &expire_func	return TOK_ATTR_EXPIRE_FUNC;
 &group		return TOK_ATTR_GROUP;
+&log		return TOK_ATTR_LOG;
 &mergeable	return TOK_ATTR_MERGEABLE;
 &optional	return TOK_ATTR_OPTIONAL;
 &persistent	return TOK_ATTR_PERSISTENT;

src/strings.bif

@@ -136,6 +136,26 @@ function sort_string_array%(a: string_array%): string_array
 	return b;
 	%}
 	
+function join_string_vec%(vec: string_vec, sep: string%): string
+	%{
+	ODesc d;
+	VectorVal *v = vec->AsVectorVal();
+
+	for ( unsigned i = 0; i < v->Size(); ++i )
+		{
+		if ( i > 0 )
+			d.Add(sep->CheckString(), 0);
+
+		v->Lookup(i+1)->Describe(&d);
+		}
+
+	BroString* s = new BroString(1, d.TakeBytes(), d.Len());
+	s->SetUseFreeToDelete(true);
+
+	return new StringVal(s);
+	%}
+
+
 function edit%(arg_s: string, arg_edit_char: string%): string
 	%{
 	if ( arg_edit_char->Len() != 1 )

src/types.bif

@@ -50,3 +50,14 @@ enum rpc_status %{
 	RPC_AUTH_ERROR,
 	RPC_UNKNOWN_ERROR,
 %}
+
+module Log;
+
+enum Writer %{
+	WRITER_DEFAULT,
+	WRITER_ASCII,
+%}
+
+enum ID %{
+	Unknown,
+%}

testing/btest/Baseline/language.rec-nested-opt/output

@@ -0,0 +1,3 @@
+{
+[Wget/1.9+cvs-stable (Red Hat modified)] = [name=Wget, version=[major=1, minor=9, addl=+cvs], host=0.0.0.0, ts=0.0]
+}

testing/btest/Baseline/language.rec-table-default/output

@@ -0,0 +1,14 @@
+{
+[foo] = T
+}
+{
+
+}
+{
+B,
+A,
+C
+}
+{
+
+}

testing/btest/Baseline/language.record-extension/output

@@ -0,0 +1,2 @@
+[a=21, b=<uninitialized>, c=42, d=<uninitialized>]
+[a=21, b=<uninitialized>, c=42, d=XXX]

testing/btest/Baseline/language.record-ref-assign/output

@@ -0,0 +1 @@
+XXX, XXX

testing/btest/Baseline/language.vector-coerce-expr/output

@@ -0,0 +1,4 @@
+[]
+[1, 2, 3]
+[T, F, T]
+[]

testing/btest/Baseline/language.wrong-record-extension/output

@@ -0,0 +1 @@
+ error, extension field must be &optional or have &default

testing/btest/Baseline/logging.adapt-filter/ssh-new-default.log

@@ -0,0 +1,3 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.40319	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1299718503.40319	1.2.3.4	1234	2.3.4.5	80	failure	US

testing/btest/Baseline/logging.ascii-empty/output

@@ -0,0 +1,6 @@
+PREFIX<>t|id.orig_h|id.orig_p|id.resp_h|id.resp_p|status|country|b
+1299718506.56593|1.2.3.4|1234|2.3.4.5|80|success|unknown|NOT-SET
+1299718506.56593|1.2.3.4|1234|2.3.4.5|80|NOT-SET|US|NOT-SET
+1299718506.56593|1.2.3.4|1234|2.3.4.5|80|failure|UK|NOT-SET
+1299718506.56593|1.2.3.4|1234|2.3.4.5|80|NOT-SET|BR|NOT-SET
+1299718506.56593|1.2.3.4|1234|2.3.4.5|80|failure|EMPTY|T

testing/btest/Baseline/logging.ascii-options/output

@@ -0,0 +1,5 @@
+1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|unknown
+1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|US
+1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|UK
+1299718506.38074|1.2.3.4|1234|2.3.4.5|80|success|BR
+1299718506.38074|1.2.3.4|1234|2.3.4.5|80|failure|MX

testing/btest/Baseline/logging.attr-extend/ssh.log

@@ -0,0 +1,2 @@
+# status	country	a1	b1	b2
+success	unknown	1	3	4

testing/btest/Baseline/logging.attr/ssh.log

@@ -0,0 +1,6 @@
+# status	country
+success	unknown
+failure	US
+failure	UK
+success	BR
+failure	MX

testing/btest/Baseline/logging.empty-event/ssh.log

@@ -0,0 +1,6 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299809561.67372	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1299809561.67372	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299809561.67372	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299809561.67372	1.2.3.4	1234	2.3.4.5	80	success	BR
+1299809561.67372	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/logging.events/output

@@ -0,0 +1,2 @@
+[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=success, country=<uninitialized>]
+[t=1299718502.96511, id=[orig_h=1.2.3.4, orig_p=1234/tcp, resp_h=2.3.4.5, resp_p=80/tcp], status=failure, country=US]

testing/btest/Baseline/logging.exclude/ssh.log

@@ -0,0 +1,6 @@
+# id.orig_p	id.resp_h	id.resp_p	status	country
+1234	2.3.4.5	80	success	unknown
+1234	2.3.4.5	80	failure	US
+1234	2.3.4.5	80	failure	UK
+1234	2.3.4.5	80	success	BR
+1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/logging.file/ssh.log

@@ -0,0 +1,2 @@
+# t	f
+1303098703.62603	Foo.log

testing/btest/Baseline/logging.include/ssh.log

@@ -0,0 +1,6 @@
+# t	id.orig_h
+1303064007.48299	1.2.3.4
+1303064007.48299	1.2.3.4
+1303064007.48299	1.2.3.4
+1303064007.48299	1.2.3.4
+1303064007.48299	1.2.3.4

testing/btest/Baseline/logging.path-func/output

@@ -0,0 +1,13 @@
+static-prefix-0.log
+static-prefix-1.log
+static-prefix-2.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	success	BR
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	MX3
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	MX
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299718503.05867	1.2.3.4	1234	2.3.4.5	80	failure	MX2

testing/btest/Baseline/logging.pred/ssh.failure.log

@@ -0,0 +1,2 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.16177	1.2.3.4	1234	2.3.4.5	80	failure	US

testing/btest/Baseline/logging.pred/ssh.success.log

@@ -0,0 +1,2 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.16177	1.2.3.4	1234	2.3.4.5	80	success	-

testing/btest/Baseline/logging.remote-types/receiver.ssh.log

@@ -0,0 +1,2 @@
+# b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se	vc	ve
+T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1301360085.98852	100.0	hurz	4,1,3,2	CC,BB,AA	EMPTY	10,20,30	EMPTY

testing/btest/Baseline/logging.remote/sender.ssh.failure.log

@@ -0,0 +1,4 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/logging.remote/sender.ssh.log

@@ -0,0 +1,6 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	success	-
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	success	BR
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/logging.remote/sender.ssh.success.log

@@ -0,0 +1,3 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	success	-
+1299718503.72819	1.2.3.4	1234	2.3.4.5	80	success	BR

testing/btest/Baseline/logging.remove/ssh.failure.log

@@ -0,0 +1,3 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.28253	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718503.28253	1.2.3.4	1234	2.3.4.5	80	failure	UK

testing/btest/Baseline/logging.remove/ssh.log

@@ -0,0 +1,4 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718503.28253	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718503.28253	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299718503.28253	1.2.3.4	1234	2.3.4.5	80	failure	BR

testing/btest/Baseline/logging.rotate-custom/out

@@ -0,0 +1,134 @@
+2nd test2-11-03-06_19.00.05.log test2.log 11-03-06_19.00.05 11-03-06_19.59.55 0
+1st test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0
+2nd test2-11-03-06_19.59.55.log test2.log 11-03-06_19.59.55 11-03-06_20.00.05 0
+2nd test2-11-03-06_20.00.05.log test2.log 11-03-06_20.00.05 11-03-06_20.59.55 0
+1st test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0
+2nd test2-11-03-06_20.59.55.log test2.log 11-03-06_20.59.55 11-03-06_21.00.05 0
+2nd test2-11-03-06_21.00.05.log test2.log 11-03-06_21.00.05 11-03-06_21.59.55 0
+1st test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0
+2nd test2-11-03-06_21.59.55.log test2.log 11-03-06_21.59.55 11-03-06_22.00.05 0
+2nd test2-11-03-06_22.00.05.log test2.log 11-03-06_22.00.05 11-03-06_22.59.55 0
+1st test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0
+2nd test2-11-03-06_22.59.55.log test2.log 11-03-06_22.59.55 11-03-06_23.00.05 0
+2nd test2-11-03-06_23.00.05.log test2.log 11-03-06_23.00.05 11-03-06_23.59.55 0
+1st test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0
+2nd test2-11-03-06_23.59.55.log test2.log 11-03-06_23.59.55 11-03-07_00.00.05 0
+2nd test2-11-03-07_00.00.05.log test2.log 11-03-07_00.00.05 11-03-07_00.59.55 0
+1st test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0
+2nd test2-11-03-07_00.59.55.log test2.log 11-03-07_00.59.55 11-03-07_01.00.05 0
+2nd test2-11-03-07_01.00.05.log test2.log 11-03-07_01.00.05 11-03-07_01.59.55 0
+1st test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0
+2nd test2-11-03-07_01.59.55.log test2.log 11-03-07_01.59.55 11-03-07_02.00.05 0
+2nd test2-11-03-07_02.00.05.log test2.log 11-03-07_02.00.05 11-03-07_02.59.55 0
+1st test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0
+2nd test2-11-03-07_02.59.55.log test2.log 11-03-07_02.59.55 11-03-07_03.00.05 0
+2nd test2-11-03-07_03.00.05.log test2.log 11-03-07_03.00.05 11-03-07_03.59.55 0
+1st test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0
+2nd test2-11-03-07_03.59.55.log test2.log 11-03-07_03.59.55 11-03-07_04.00.05 0
+2nd test2-11-03-07_04.00.05.log test2.log 11-03-07_04.00.05 11-03-07_04.59.55 0
+1st test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1
+2nd test2-11-03-07_04.59.55.log test2.log 11-03-07_04.59.55 11-03-07_04.59.55 1
+> test-11-03-06_19.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299466805.0	10.0.0.1	20	10.0.0.2	1024
+1299470395.0	10.0.0.2	20	10.0.0.3	0
+> test-11-03-06_20.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299470405.0	10.0.0.1	20	10.0.0.2	1025
+1299473995.0	10.0.0.2	20	10.0.0.3	1
+> test-11-03-06_21.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299474005.0	10.0.0.1	20	10.0.0.2	1026
+1299477595.0	10.0.0.2	20	10.0.0.3	2
+> test-11-03-06_22.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299477605.0	10.0.0.1	20	10.0.0.2	1027
+1299481195.0	10.0.0.2	20	10.0.0.3	3
+> test-11-03-06_23.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299481205.0	10.0.0.1	20	10.0.0.2	1028
+1299484795.0	10.0.0.2	20	10.0.0.3	4
+> test-11-03-07_00.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299484805.0	10.0.0.1	20	10.0.0.2	1029
+1299488395.0	10.0.0.2	20	10.0.0.3	5
+> test-11-03-07_01.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299488405.0	10.0.0.1	20	10.0.0.2	1030
+1299491995.0	10.0.0.2	20	10.0.0.3	6
+> test-11-03-07_02.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299492005.0	10.0.0.1	20	10.0.0.2	1031
+1299495595.0	10.0.0.2	20	10.0.0.3	7
+> test-11-03-07_03.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299495605.0	10.0.0.1	20	10.0.0.2	1032
+1299499195.0	10.0.0.2	20	10.0.0.3	8
+> test-11-03-07_04.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299499205.0	10.0.0.1	20	10.0.0.2	1033
+1299502795.0	10.0.0.2	20	10.0.0.3	9
+> test2-11-03-06_19.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299466805.0	10.0.0.1	20	10.0.0.2	1024
+> test2-11-03-06_19.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299470395.0	10.0.0.2	20	10.0.0.3	0
+> test2-11-03-06_20.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299470405.0	10.0.0.1	20	10.0.0.2	1025
+> test2-11-03-06_20.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299473995.0	10.0.0.2	20	10.0.0.3	1
+> test2-11-03-06_21.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299474005.0	10.0.0.1	20	10.0.0.2	1026
+> test2-11-03-06_21.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299477595.0	10.0.0.2	20	10.0.0.3	2
+> test2-11-03-06_22.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299477605.0	10.0.0.1	20	10.0.0.2	1027
+> test2-11-03-06_22.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299481195.0	10.0.0.2	20	10.0.0.3	3
+> test2-11-03-06_23.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299481205.0	10.0.0.1	20	10.0.0.2	1028
+> test2-11-03-06_23.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299484795.0	10.0.0.2	20	10.0.0.3	4
+> test2-11-03-07_00.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299484805.0	10.0.0.1	20	10.0.0.2	1029
+> test2-11-03-07_00.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299488395.0	10.0.0.2	20	10.0.0.3	5
+> test2-11-03-07_01.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299488405.0	10.0.0.1	20	10.0.0.2	1030
+> test2-11-03-07_01.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299491995.0	10.0.0.2	20	10.0.0.3	6
+> test2-11-03-07_02.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299492005.0	10.0.0.1	20	10.0.0.2	1031
+> test2-11-03-07_02.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299495595.0	10.0.0.2	20	10.0.0.3	7
+> test2-11-03-07_03.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299495605.0	10.0.0.1	20	10.0.0.2	1032
+> test2-11-03-07_03.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299499195.0	10.0.0.2	20	10.0.0.3	8
+> test2-11-03-07_04.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299499205.0	10.0.0.1	20	10.0.0.2	1033
+> test2-11-03-07_04.59.55.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299502795.0	10.0.0.2	20	10.0.0.3	9
+> test2.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+> test.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p

testing/btest/Baseline/logging.rotate/out

@@ -0,0 +1,50 @@
+test-11-03-06_19.00.05.log test.log 11-03-06_19.00.05 11-03-06_20.00.05 0
+test-11-03-06_20.00.05.log test.log 11-03-06_20.00.05 11-03-06_21.00.05 0
+test-11-03-06_21.00.05.log test.log 11-03-06_21.00.05 11-03-06_22.00.05 0
+test-11-03-06_22.00.05.log test.log 11-03-06_22.00.05 11-03-06_23.00.05 0
+test-11-03-06_23.00.05.log test.log 11-03-06_23.00.05 11-03-07_00.00.05 0
+test-11-03-07_00.00.05.log test.log 11-03-07_00.00.05 11-03-07_01.00.05 0
+test-11-03-07_01.00.05.log test.log 11-03-07_01.00.05 11-03-07_02.00.05 0
+test-11-03-07_02.00.05.log test.log 11-03-07_02.00.05 11-03-07_03.00.05 0
+test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0
+test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_04.59.55 1
+> test-11-03-06_19.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299466805.0	10.0.0.1	20	10.0.0.2	1024
+1299470395.0	10.0.0.2	20	10.0.0.3	0
+> test-11-03-06_20.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299470405.0	10.0.0.1	20	10.0.0.2	1025
+1299473995.0	10.0.0.2	20	10.0.0.3	1
+> test-11-03-06_21.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299474005.0	10.0.0.1	20	10.0.0.2	1026
+1299477595.0	10.0.0.2	20	10.0.0.3	2
+> test-11-03-06_22.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299477605.0	10.0.0.1	20	10.0.0.2	1027
+1299481195.0	10.0.0.2	20	10.0.0.3	3
+> test-11-03-06_23.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299481205.0	10.0.0.1	20	10.0.0.2	1028
+1299484795.0	10.0.0.2	20	10.0.0.3	4
+> test-11-03-07_00.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299484805.0	10.0.0.1	20	10.0.0.2	1029
+1299488395.0	10.0.0.2	20	10.0.0.3	5
+> test-11-03-07_01.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299488405.0	10.0.0.1	20	10.0.0.2	1030
+1299491995.0	10.0.0.2	20	10.0.0.3	6
+> test-11-03-07_02.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299492005.0	10.0.0.1	20	10.0.0.2	1031
+1299495595.0	10.0.0.2	20	10.0.0.3	7
+> test-11-03-07_03.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299495605.0	10.0.0.1	20	10.0.0.2	1032
+1299499195.0	10.0.0.2	20	10.0.0.3	8
+> test-11-03-07_04.00.05.log
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
+1299499205.0	10.0.0.1	20	10.0.0.2	1033
+1299502795.0	10.0.0.2	20	10.0.0.3	9

testing/btest/Baseline/logging.stdout/output

@@ -0,0 +1,6 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718506.28824	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1299718506.28824	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718506.28824	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299718506.28824	1.2.3.4	1234	2.3.4.5	80	success	BR
+1299718506.28824	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/logging.test-logging/ssh.log

@@ -0,0 +1,6 @@
+# t	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	country
+1299718506.1313	1.2.3.4	1234	2.3.4.5	80	success	unknown
+1299718506.1313	1.2.3.4	1234	2.3.4.5	80	failure	US
+1299718506.1313	1.2.3.4	1234	2.3.4.5	80	failure	UK
+1299718506.1313	1.2.3.4	1234	2.3.4.5	80	success	BR
+1299718506.1313	1.2.3.4	1234	2.3.4.5	80	failure	MX

testing/btest/Baseline/logging.types/ssh.log

@@ -0,0 +1,2 @@
+# b	i	e	c	p	sn	n	a	d	t	iv	s	sc	ss	se	vc	ve
+T	-42	SSH::SSH	21	123	10.0.0.0/24	10.0.0.0	1.2.3.4	3.14	1301359781.8203	100.0	hurz	4,1,3,2	CC,BB,AA	EMPTY	10,20,30	EMPTY

testing/btest/Baseline/logging.vec/ssh.log

@@ -0,0 +1,2 @@
+# vec
+-,2,-,-,5

testing/btest/btest.cfg

@@ -1,10 +1,9 @@
-
 [btest]
-TestDirs    = doc bifs
+TestDirs    = doc bifs logging language
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
-IgnoreFiles = *.tmp *.swp #*
+IgnoreFiles = *.tmp *.swp #* *.trace
 
 [environment]
 BROPATH=`bash -c %(testbase)s/../../build/bro-path-dev`

testing/btest/language/rec-nested-opt.bro

@@ -0,0 +1,25 @@
+
+# @TEST-EXEC: bro %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+type Version: record {
+        major:  count &optional;    ##< Major version number
+        minor:  count &optional;    ##< Minor version number
+        addl:   string &optional;  ##< Additional version string (e.g. "beta42")
+} &log;
+
+type Info: record {
+	name: string;
+	version: Version;
+	host: addr;
+	ts: time;
+};
+
+
+# Important thing to note here is that $minor2 is not include in the $version field.
+global matched_software: table[string] of Info = {
+        ["Wget/1.9+cvs-stable (Red Hat modified)"] =
+                [$name="Wget", $version=[$major=1,$minor=9,$addl="+cvs"], $host=0.0.0.0, $ts=network_time()],
+};
+
+print matched_software;

testing/btest/language/rec-table-default.bro

@@ -0,0 +1,19 @@
+
+# @TEST-EXEC: bro %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+type X: record {
+     a: table[string] of bool &default=table( ["foo"] = T );
+     b: table[string] of bool &default=table();
+     c: set[string] &default=set("A", "B", "C");
+     d: set[string] &default=set();
+};
+
+global x: X;
+global y: table[string] of bool &default=T;
+
+print x$a;
+print x$b;
+print x$c;
+print x$d;
+

testing/btest/language/record-extension.bro

@@ -0,0 +1,19 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+type Foo: record {
+        a: count;
+        b: count &optional;
+};
+
+redef record Foo += {
+        c: count &default=42;
+        d: count &optional;
+};
+
+global f1: Foo = [$a=21];
+global f2: Foo = [$a=21, $d="XXX"];
+
+print f1;
+print f2;
+

testing/btest/language/record-ref-assign.bro

@@ -0,0 +1,12 @@
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+type State: record {
+	host: string &default="NOT SET";
+};
+
+global session: State;
+global s: State;
+s = session;
+s$host = "XXX";
+print s$host, session$host;

testing/btest/language/vector-coerce-expr.bro

@@ -0,0 +1,20 @@
+# @TEST-EXEC: bro %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+type X: record {
+     a: vector of bool &default=vector(T, F, T);
+     b: vector of bool &default=vector();
+};
+
+global x: X;
+
+global a: vector of count;
+
+a = vector();
+print a;
+
+a = vector(1,2,3);
+print a;
+
+print x$a;
+print x$b;

testing/btest/language/wrong-record-extension.bro

@@ -0,0 +1,14 @@
+# @TEST-EXEC-FAIL: bro %INPUT  >output.tmp 2>&1 
+# @TEST-EXEC: sed 's#^.*:##g' <output.tmp >output
+# @TEST-EXEC: btest-diff output
+
+type Foo: record {
+        a: count;
+        b: count &optional;
+};
+
+redef record Foo += {
+        c: count;
+        d: string &optional;
+};
+

testing/btest/logging/adapt-filter.bro

@@ -0,0 +1,35 @@
+
+# @TEST-EXEC: bro %INPUT 
+# @TEST-EXEC: btest-diff ssh-new-default.log
+# @TEST-EXEC: test '!' -e ssh.log
+
+module SSH;
+
+@load logging
+
+export {
+	# Create a new ID for our log stream
+	redef enum Log::ID += { SSH };
+
+	# Define a record with all the columns the log file can have.
+	# (I'm using a subset of fields from ssh-ext for demonstration.)
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+	local filter = Log::get_filter(SSH, "default");
+	filter$path= "ssh-new-default";
+	Log::add_filter(SSH, filter);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+}

testing/btest/logging/ascii-empty.bro

@@ -0,0 +1,38 @@
+#
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+redef LogAscii::output_to_stdout = T;
+redef LogAscii::separator = "|";
+redef LogAscii::empty_field = "EMPTY";
+redef LogAscii::unset_field = "NOT-SET";
+redef LogAscii::header_prefix = "PREFIX<>";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+		b: bool &optional;
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $b=T, $status="failure", $country=""]);
+	
+}
+

testing/btest/logging/ascii-escape.bro

@@ -0,0 +1,32 @@
+#
+# @TEST-EXEC: bro %INPUT 
+# @TEST-EXEC: btest-diff ssh.log
+
+redef LogAscii::separator = "||";
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="fa||ure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="su||ess", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);	
+}
+

testing/btest/logging/ascii-options.bro

@@ -0,0 +1,35 @@
+#
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+redef LogAscii::output_to_stdout = T;
+redef LogAscii::separator = "|";
+redef LogAscii::include_header = F;
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id; # Will be rolled out into individual columns.
+		status: string &optional;
+		country: string &default="unknown";
+	} &log;
+}
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="US"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="UK"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $country="BR"]);
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="failure", $country="MX"]);
+	
+}
+

testing/btest/logging/attr-extend.bro

@@ -0,0 +1,37 @@
+#
+# @TEST-EXEC: bro %INPUT
+# @TEST-EXEC: btest-diff ssh.log
+
+module SSH;
+
+export {
+	redef enum Log::ID += { SSH };
+
+	type Log: record {
+		t: time;
+		id: conn_id;
+		status: string &optional &log;
+		country: string &default="unknown" &log;
+	};
+}
+
+redef record Log += {
+	a1: count &log &optional;
+	a2: count &optional;
+};
+
+redef record Log += {
+	b1: count &optional;
+	b2: count &optional;
+} &log;
+
+
+event bro_init()
+{
+	Log::create_stream(SSH, [$columns=Log]);
+
+    local cid = [$orig_h=1.2.3.4, $orig_p=1234/tcp, $resp_h=2.3.4.5, $resp_p=80/tcp];
+
+	Log::write(SSH, [$t=network_time(), $id=cid, $status="success", $a1=1, $a2=2, $b1=3, $b2=4]);
+}
+