Merge branch 'master' into topic/jsiwek/autodoc-fixes

Conflicts:
	scripts/CMakeLists.txt
	scripts/base/frameworks/cluster/setup-connections.bro
	scripts/base/frameworks/communication/__load__.bro
	scripts/base/frameworks/metrics/conn-example.bro
	scripts/base/frameworks/metrics/http-example.bro
	scripts/site/local.bro

CHANGES

@@ -1,4 +1,58 @@
 
+1.6-dev-1061 | 2011-08-08 18:25:27 -0700
+
+  * A set of new/changed tests regarding the new policy script
+    organisation. (Robin Sommer)
+
+1.6-dev-1058 | 2011-08-08 16:15:18 -0700
+
+  * Reorganisation of the scripts that Bro loads by default. (Seth
+    Hall)
+
+    - policy/ renamed to scripts/
+
+    - By default BROPATH now contains:
+    	- scripts/
+    	- scripts/policy
+    	- scripts/site
+
+    - The scripts in scripts/base/protocols/ only do logging and state
+      building.
+
+    - All of scripts/base/ is loaded by by default. This can however
+      be disabled by switching Bro into "bare mode" using the new
+      command-line option --bare-mode (or -b). The cripts in
+      scripts/base/ don't use relative path loading to ease use of
+      bare mode (to copy and paste that script).
+
+    - The scripts in scripts/base/frameworks/ add functionality
+      without causing any additional overhead.
+
+    - All "detection" activity happens through scripts in
+      scripts/policy/.
+
+    - bro.init was renamed to base/init-bare.bro, and base/all.bro was
+      renamed to init-default.bro.
+
+    - local.bro now loads more functionality from policy/ and adds
+      more documentation. (Seth Hall)
+
+  * Adding default_path_func() to the logging framework that makes the
+    default naming scheme script-level controlled. (Robin Sommer)
+
+  * Reworking logging's postprocessor logic so that postprocessor
+    commands are no longer run by the log writers themselves, but
+    instead by a script level function. (Robin Sommer)
+
+  * The communication subsystem is now by default off and must be
+    enabled explicitly with a new BiF, enable_communication(). Closes
+    #540. (Robin Sommer)
+
+  * The hostname notice email extension now only add hostnames for
+    emailed noticed. (Seth Hall)
+
+  * Cleaning up doc generation. (Seth Hall)
+
 1.6-dev-1044 | 2011-08-05 19:07:32 -0700
 
   * Fixing memory (and CPU) leak in log writer.

VERSION

@@ -1 +1 @@
-1.6-dev-1044
+1.6-dev-1061

aux/binpac

@@ -1 +1 @@
-Subproject commit 4fc13f7c6987b4163609e3df7a31f38501411cb7
+Subproject commit 7cdd9c39d97c2984293fbe4a6dbe9ac0b33ecbfa

aux/broccoli

@@ -1 +1 @@
-Subproject commit 2455dbebc15f06b3cc5eccb701727baf5472cf24
+Subproject commit 45f577240089d63dd0dc58be564280725a97acec

aux/broctl

@@ -1 +1 @@
-Subproject commit 870ee782bfeb3a60bac40fce4273436e5f2d280b
+Subproject commit dbbe6c81ef40666338c950d8f69dc8597f2adc70

scripts/CMakeLists.txt

@@ -1,17 +1,27 @@
 include(InstallPackageConfigFile)
 
 install(DIRECTORY ./ DESTINATION ${BRO_SCRIPT_INSTALL_PATH} FILES_MATCHING
-        PATTERN "all.bro" EXCLUDE
+        PATTERN "site/local*" EXCLUDE
-        PATTERN "site/local.bro" EXCLUDE
         PATTERN "*.bro"
         PATTERN "*.sig"
-        PATTERN "*.osf"
+        PATTERN "*.fp"
 )
 
-# Install as a config file since the local.bro script is meant to be
+# Install all local* scripts as config files since they are meant to be
 # user modify-able.
 InstallPackageConfigFile(
     ${CMAKE_CURRENT_SOURCE_DIR}/site/local.bro
     ${BRO_SCRIPT_INSTALL_PATH}/site
     local.bro)
-
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-manager.bro
+    ${BRO_SCRIPT_INSTALL_PATH}/site
+    local-manager.bro)
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-proxy.bro
+    ${BRO_SCRIPT_INSTALL_PATH}/site
+    local-proxy.bro)
+InstallPackageConfigFile(
+    ${CMAKE_CURRENT_SOURCE_DIR}/site/local-worker.bro
+    ${BRO_SCRIPT_INSTALL_PATH}/site
+    local-worker.bro)

scripts/base/frameworks/cluster/__load__.bro

@@ -9,10 +9,6 @@ redef peer_description = Cluster::node;
 # Add a cluster prefix.
 @prefixes += cluster
 
-# Make this a controllable node since all cluster nodes are inherently 
-# controllable.
-@load frameworks/control/controllee
-
 ## If this script isn't found anywhere, the cluster bombs out.
 ## Loading the cluster framework requires that a script by this name exists
 ## somewhere in the BROPATH.  The only thing in the file should be the

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -10,11 +10,14 @@
 
 @prefixes += cluster-manager
 
+## Load the script for local site configuration for the manager node.
+@load site/local-manager
+
 ## Turn off remote logging since this is the manager and should only log here.
 redef Log::enable_remote_logging = F;
 
 ## Use the cluster's archive logging script.
-redef Log::default_rotation_postprocessor = "archive-log";
+redef Log::default_rotation_postprocessor_cmd = "archive-log";
 
 ## We're processing essentially *only* remote events.
 redef max_remote_events_processed = 10000;

scripts/base/frameworks/cluster/nodes/proxy.bro

@@ -1,6 +1,9 @@
 
 @prefixes += cluster-proxy
 
+## Load the script for local site configuration for proxy nodes.
+@load site/local-proxy
+
 ## The proxy only syncs state; does not forward events.
 redef forward_remote_events = F;
 redef forward_remote_state_changes = T;
@@ -12,5 +15,5 @@ redef Log::enable_local_logging = F;
 redef Log::enable_remote_logging = T;
 
 ## Use the cluster's delete-log script.
-redef Log::default_rotation_postprocessor = "delete-log";
+redef Log::default_rotation_postprocessor_cmd = "delete-log";
 

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,6 +1,9 @@
 
 @prefixes += cluster-worker
 
+## Load the script for local site configuration for the worker nodes.
+@load site/local-worker
+
 ## Don't do any local logging.
 redef Log::enable_local_logging = F;
 
@@ -8,7 +11,7 @@ redef Log::enable_local_logging = F;
 redef Log::enable_remote_logging = T;
 
 ## Use the cluster's delete-log script.
-redef Log::default_rotation_postprocessor = "delete-log";
+redef Log::default_rotation_postprocessor_cmd = "delete-log";
 
 ## Record all packets into trace file.
 # TODO: should we really be setting this to T?

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,4 +1,5 @@
-@load base/frameworks/communication
+@load ./main
+@load base/frameworks/communication/main
 
 module Cluster;
 

scripts/base/frameworks/communication/__load__.bro

@@ -1,5 +1 @@
-
-# TODO: get rid of this as soon as the Expr.cc hack is changed.
-@if ( getenv("ENABLE_COMMUNICATION") != "" )
 @load ./main
-@endif

scripts/base/frameworks/communication/main.bro

@@ -108,6 +108,9 @@ const src_names = {
 event bro_init()
 	{
 	Log::create_stream(COMMUNICATION, [$columns=Info]);
+
+	if ( |nodes| > 0 )
+		enable_communication();
 	}
 
 function do_script_log_common(level: count, src: count, msg: string)

scripts/base/frameworks/logging/main.bro

@@ -27,6 +27,17 @@ export {
 		ev: any &optional;
 	};
 
+	## Default function for building the path values for log filters if not
+	## speficied otherwise by a filter. The default implementation uses ``id``
+	## to derive a name.
+	##
+	## id: The log stream.
+	## path: A suggested path value, which may be either the filter's ``path``
+	## if defined or a fall-back generated internally.
+	##
+	## Returns: The path to be used for the filter.
+	global default_path_func: function(id: ID, path: string) : string &redef;
+
 	## Filter customizing logging.
 	type Filter: record {
 		## Descriptive name to reference this filter.
@@ -82,35 +93,33 @@ export {
 	## Information passed into rotation callback functions.
 	type RotationInfo: record {
 		writer: Writer;		##< Writer.
+		fname: string;		##< Full name of the rotated file.
 		path: string;		##< Original path value.
 		open: time;			##< Time when opened.
 		close: time;		##< Time when closed.
+		terminating: bool;	##< True if rotation occured due to Bro shutting down.
 	};
 
 	## Default rotation interval. Zero disables rotation.
 	const default_rotation_interval = 0secs &redef;
 
-	## Default naming suffix format. Uses a strftime() style.
+	## Default naming format for timestamps embedded into filenames. Uses a strftime() style.
-	const default_rotation_date_format = "%y-%m-%d_%H.%M.%S" &redef;
+	const default_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;
 
-	## Default postprocessor for writers outputting into files.
+	## Default shell command to run on rotated files. Empty for none.
-	const default_rotation_postprocessor = "" &redef;
+	const default_rotation_postprocessor_cmd = "" &redef;
 
-	## Default function to construct the name of a rotated output file.
+	## Specifies the default postprocessor function per writer type. Entries in this
-	## The default implementation appends info$date_fmt to the original
+	## table are initialized by each writer type.
-	## file name.
+	const default_rotation_postprocessors: table[Writer] of function(info: RotationInfo) : bool &redef;
-	##
-	## info: Meta-data about the file to be rotated.
-	global default_rotation_path_func: function(info: RotationInfo) : string &redef;
 
 	## Type for controlling file rotation.
 	type RotationControl: record  {
 		## Rotation interval.
 		interv: interval &default=default_rotation_interval;
-		## Format for timestamps embedded into rotated file names.
+		## Callback function to trigger for rotated files. If not set, the default
-		date_fmt: string &default=default_rotation_date_format;
+		## comes out of default_rotation_postprocessors.
-		## Postprocessor process to run on rotate file.
+		postprocessor: function(info: RotationInfo) : bool &optional;
-		postprocessor: string &default=default_rotation_postprocessor;
 	};
 
 	## Specifies rotation parameters per ``(id, path)`` tuple.
@@ -133,6 +142,8 @@ export {
 	global flush: function(id: ID): bool;
 	global add_default_filter: function(id: ID) : bool;
 	global remove_default_filter: function(id: ID) : bool;
+
+	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
@@ -140,10 +151,39 @@ global filters: table[ID, string] of Filter;
 
 @load logging.bif.bro # Needs Filter and Stream defined.
 
-function default_rotation_path_func(info: RotationInfo) : string
+module Log;
+
+# Used internally by the log manager.
+function __default_rotation_postprocessor(info: RotationInfo) : bool
 	{
-	local date_fmt = rotation_control[info$writer, info$path]$date_fmt;
+	if ( info$writer in default_rotation_postprocessors )
-	return fmt("%s-%s", info$path, strftime(date_fmt, info$open));
+		return default_rotation_postprocessors[info$writer](info);
+	}
+
+function default_path_func(id: ID, path: string) : string
+	{
+	# TODO for Seth: Do what you want. :)
+	return path;
+	}
+
+# Run post-processor on file. If there isn't any postprocessor defined,
+# we move the file to a nicer name.
+function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : bool
+	{
+	local pp_cmd = default_rotation_postprocessor_cmd;
+
+	if ( pp_cmd == "" )
+		return T;
+
+	# The date format is hard-coded here to provide a standardized
+	# script interface.
+	system(fmt("%s %s %s %s %s %d",
+               pp_cmd, npath, info$path,
+               strftime("%y-%m-%d_%H.%M.%S", info$open),
+               strftime("%y-%m-%d_%H.%M.%S", info$close),
+               info$terminating));
+
+	return T;
 	}
 
 function create_stream(id: ID, stream: Stream) : bool
@@ -162,6 +202,12 @@ function disable_stream(id: ID) : bool
 
 function add_filter(id: ID, filter: Filter) : bool
 	{
+	# This is a work-around for the fact that we can't forward-declare
+	# the default_path_func and then use it as &default in the record
+	# definition.
+	if ( ! filter?$path_func )
+		filter$path_func = default_path_func;
+
 	filters[id, filter$name] = filter;
 	return __add_filter(id, filter);
 	}

scripts/base/frameworks/logging/writers/ascii.bro

@@ -26,4 +26,19 @@ export {
 	const unset_field = "-" &redef;
 }
 
+# Default function to postprocess a rotated ASCII log file. It moves the rotated
+# file to a new name that includes a timestamp with the opening time, and then
+# runs the writer's default postprocessor command on it.
+function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
+	{
+	# Move file to name including both opening and closing time.
+	local dst = fmt("%s.%s.log", info$path,
+			strftime(Log::default_rotation_date_format, info$open));
 
+	system(fmt("/bin/mv %s %s", info$fname, dst));
+
+	# Run default postprocessor.
+	return Log::run_rotation_postprocessor_cmd(info, dst);
+	}
+
+redef Log::default_rotation_postprocessors += { [Log::WRITER_ASCII] = default_rotation_postprocessor_func };

scripts/base/frameworks/metrics/conn-example.bro

@@ -1,19 +0,0 @@
-@load base/frameworks/metrics
-
-redef enum Metrics::ID += { 
-	CONNS_ORIGINATED, 
-	CONNS_RESPONDED 
-};
-
-event bro_init()
-	{
-	Metrics::configure(CONNS_ORIGINATED, [$aggregation_mask=24, $break_interval=5mins]);
-	Metrics::configure(CONNS_RESPONDED, [$aggregation_mask=24, $break_interval=5mins]);
-	}
-
-event connection_established(c: connection)
-	{
-	Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h], 1);
-	Metrics::add_data(CONNS_RESPONDED,  [$host=c$id$resp_h], 1);
-	}
-	

scripts/base/frameworks/metrics/http-example.bro

@@ -1,20 +0,0 @@
-@load base/frameworks/metrics
-
-redef enum Metrics::ID += {
-	HTTP_REQUESTS_BY_STATUS_CODE,
-	HTTP_REQUESTS_BY_HOST,
-};
-
-event bro_init()
-	{
-	Metrics::configure(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_mask=24, $break_interval=10secs]);
-	Metrics::configure(HTTP_REQUESTS_BY_HOST, [$break_interval=10secs]);
-	}
-
-event HTTP::log_http(rec: HTTP::Info)
-	{
-	if ( rec?$host )
-		Metrics::add_data(HTTP_REQUESTS_BY_HOST, [$index=rec$host], 1);
-	if ( rec?$status_code )
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $index=fmt("%d", rec$status_code)], 1);
-	}

scripts/base/frameworks/metrics/main.bro

@@ -1,4 +1,4 @@
-##! This is the implementation of the metrics framework
+##! This is the implementation of the metrics framework.
 
 module Metrics;
 
@@ -6,25 +6,23 @@ export {
 	redef enum Log::ID += { METRICS };
 	
 	type ID: enum {
-		ALL,
+		NOTHING,
 	};
 	
-	const default_aggregation_mask = 24 &redef;
+	## The default interval used for "breaking" metrics and writing the 
-	const default_break_interval = 5mins &redef;
+	## current value to the logging stream.
+	const default_break_interval = 15mins &redef;
 	
-	# TODO: configure a metrics filter logging stream to log the current
+	type Info: record {
-	#       metrics configuration in case someone is looking through
+		ts:           time   &log;
-	#       old logs and the configuration has changed since then.
+		metric_id:    ID     &log;
-	type Filter: record {
+		filter_name:  string &log;
-		name:              ID                      &optional;
+		agg_subnet:   string &log &optional;
-		## Global mask by which you'd like to aggregate traffic.
+		index:        string &log &optional;
-		aggregation_mask:  count                   &optional;
+		value:        count  &log;
-		## This is essentially applying names to various subnets.
-		aggregation_table: table[subnet] of string &optional;
-		break_interval:    interval                &default=default_break_interval;
 	};
 	
-	type Index: record {
+	type Entry: record {
 		## Host is the value to which this metric applies.
 		host:         addr &optional;
 		
@@ -36,36 +34,71 @@ export {
 		## metric since multiple IP addresses could respond for the same Host
 		## header value.
 		index:        string &default="";
+		
+		## The value by which the counter should be increased in each filter
+		## where this entry is accepted.
+		increment:    count &default=1;
 	};
 	
-	type Info: record {
+	# TODO: configure a metrics filter logging stream to log the current
-		ts:           time   &log;
+	#       metrics configuration in case someone is looking through
-		name:         ID     &log;
+	#       old logs and the configuration has changed since then.
-		index:        string &log &optional;
+	type Filter: record {
-		agg_subnet:   string &log &optional;
+		## The :bro:type:`Metrics::ID` that this filter applies to.
-		value:        count  &log;
+		id:                ID                      &optional;
+		## The name for this filter so that multiple filters can be
+		## applied to a single metrics to get a different view of the same
+		## metric data being collected (different aggregation, break, etc).
+		name:              string                  &default="default";
+		## A predicate so that you can decide per index if you would like
+		## to accept the data being inserted.
+		pred:              function(entry: Entry): bool &optional;
+		## Global mask by which you'd like to aggregate traffic.
+		aggregation_mask:  count                   &optional;
+		## This is essentially applying names to various subnets.
+		aggregation_table: table[subnet] of string &optional;
+		## The interval at which the metric should be "broken" and written
+		## to the logging stream.
+		break_interval:    interval                &default=default_break_interval;
+		## This determines if the result of this filter is sent to the metrics
+		## logging stream.  One use for the logging framework is as an internal
+		## thresholding and statistics gathering utility that is meant to
+		## never log but rather to generate notices and derive data.
+		log:               bool                    &default=T;
+		## A straight threshold for generating a notice.
+		notice_threshold:  count                   &optional;
+		## A series of thresholds at which to generate notices.
+		notice_thresholds: vector of count         &optional;
+		## If this and a $notice_threshold value are set, this notice type
+		## will be generated by the metrics framework.
+		note:              Notice::Type            &optional;
 	};
 	
-	global add_filter: function(name: ID, filter: Filter);
+	global add_filter: function(id: ID, filter: Filter);
-	global add_data: function(name: ID, index: Index, increment: count);
+	global add_data: function(id: ID, entry: Entry);
 	
 	global log_metrics: event(rec: Info);
 }
 
-global metric_filters: table[ID] of Filter = table();
+global metric_filters: table[ID] of vector of Filter = table();
 
 type MetricIndex: table[string] of count &default=0;
 type MetricTable: table[string] of MetricIndex;
-global store: table[ID] of MetricTable = table();
+# This is indexed by metric ID and stream filter name.
+global store: table[ID, string] of MetricTable = table();
 
-event bro_init()
+# This stores the current threshold index for filters using the
+# $notice_thresholds element.
+global thresholds: table[string] of count = {} &default=0;
+
+event bro_init() &priority=5
 	{
 	Log::create_stream(METRICS, [$columns=Info, $ev=log_metrics]);
 	}
 	
-function reset(name: ID)
+function reset(filter: Filter)
 	{
-	store[name] = table();
+	store[filter$id, filter$name] = table();
 	}
 
 event log_it(filter: Filter)
@@ -73,70 +106,128 @@ event log_it(filter: Filter)
 	# If this node is the manager in a cluster, this needs to request values
 	# for this metric from all of the workers.
 	
+	local id = filter$id;
 	local name = filter$name;
-	for ( agg_subnet in store[name] )
+	for ( agg_subnet in store[id, name] )
 		{
-		local metric_values = store[name][agg_subnet];
+		local metric_values = store[id, name][agg_subnet];
 		for ( index in metric_values )
 			{
 			local val = metric_values[index];
 			local m: Info = [$ts=network_time(), 
-			                 $name=name, 
+			                 $metric_id=id,
+			                 $filter_name=name,
 			                 $agg_subnet=fmt("%s", agg_subnet),
 			                 $index=index,
 			                 $value=val];
+			
+			if ( filter?$notice_threshold &&
+			     m$value >= filter$notice_threshold )
+				{
+				print m;
+				NOTICE([$note=filter$note, 
+				        $msg=fmt("Metrics threshold crossed by %s %d/%d", m$agg_subnet, m$value, filter$notice_threshold),
+				        $n=m$value]);
+				}
+			
+			else if ( filter?$notice_thresholds &&
+			          m$value >= filter$notice_thresholds[thresholds[cat(id,name)]] )
+				{
+				# TODO: implement this
+				}
+			
+			# If there wasn't an index, remove the field.
 			if ( index == "" )
 				delete m$index;
+			
+			# If there wasn't an aggregation subnet, remove the field.
 			if ( agg_subnet == "" )
 				delete m$agg_subnet;
+				
 			Log::write(METRICS, m);
 			}
 		}
-	
+	reset(filter);
-	
-	reset(name);
 	
 	schedule filter$break_interval { log_it(filter) };
 	}
 
-function add_filter(name: ID, filter: Filter)
+function add_filter(id: ID, filter: Filter)
 	{
 	if ( filter?$aggregation_table && filter?$aggregation_mask )
 		{
 		print "INVALID Metric filter: Defined $aggregation_table and $aggregation_mask.";
 		return;
 		}
+	if ( [id, filter$name] in store )
+		{
+		print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
+		return;
+		}
+	if ( filter?$notice_threshold && filter?$notice_thresholds )
+		{
+		print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds";
+		return;
+		}
 	
-	filter$name = name;
+	if ( ! filter?$id )
-	metric_filters[name] = filter;
+		filter$id = id;
-	store[name] = table();
+	
+	if ( id !in metric_filters )
+		metric_filters[id] = vector();
+	metric_filters[id][|metric_filters[id]|] = filter;
+
+	store[id, filter$name] = table();
 	
 	# Only do this on the manager if in a cluster.
 	schedule filter$break_interval { log_it(filter) };
 	}
 	
-function add_data(name: ID, index: Index, increment: count)
+function add_data(id: ID, entry: Entry)
 	{
-	local conf = metric_filters[name];
+	if ( id !in metric_filters )
+		return;
+	
+	local filters = metric_filters[id];
+	
+	# Add the data to any of the defined filters.
+	for ( filter_id in filters )
+		{
+		local filter = filters[filter_id];
+		
+		# If this filter has a predicate, run the predicate and skip this
+		# entry if the predicate return false.
+		if ( filter?$pred &&
+		     ! filter$pred(entry) )
+			next;
 		
 		local agg_subnet = "";
-	if ( index?$host )
+		local filt_store = store[id, filter$name];
+		if ( entry?$host )
 			{
-		if ( conf?$aggregation_mask )
+			if ( filter?$aggregation_mask )
 				{
-			local agg_mask = conf$aggregation_mask;
+				local agg_mask = filter$aggregation_mask;
-			agg_subnet = fmt("%s", mask_addr(index$host, agg_mask));
+				agg_subnet = fmt("%s", mask_addr(entry$host, agg_mask));
+				}
+			else if ( filter?$aggregation_table )
+				{
+				agg_subnet = fmt("%s", filter$aggregation_table[entry$host]);
+				# if an aggregation table is being used and the value isn't 
+				# in the table, that means we aren't interested in it.
+				if ( agg_subnet == "" )
+					next;
 				}
-		else if ( conf?$aggregation_table )	
-			agg_subnet = fmt("%s", conf$aggregation_table[index$host]);
 			else
-			agg_subnet = fmt("%s", index$host);
+				agg_subnet = fmt("%s", entry$host);
 			}
 
-	if ( agg_subnet !in store[name] )
+		if ( agg_subnet !in filt_store )
-		store[name][agg_subnet] = table();
+			filt_store[agg_subnet] = table();
 
-	if ( index$index !in store[name][agg_subnet] )
+		local fs = filt_store[agg_subnet];
-		store[name][agg_subnet][index$index] = 0;
+		if ( entry$index !in fs )
-	store[name][agg_subnet][index$index] = store[name][agg_subnet][index$index] + increment;
+			fs[entry$index] = 0;
+		fs[entry$index] = fs[entry$index] + entry$increment;
+		}
 	}

scripts/base/frameworks/notice/__load__.bro

@@ -6,7 +6,8 @@
 @load ./actions/drop
 @load ./actions/email_admin
 @load ./actions/page
+@load ./actions/add-geodata
 
-# Load the script to add hostnames to emails by default.
+# There shouldn't be any defaul toverhead from loading these since they 
-# NOTE: this exposes a memleak in async DNS lookups.
+# *should* only do anything when notices have the ACTION_EMAIL action applied.
-#@load ./extend-email/hostnames
+@load ./extend-email/hostnames

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -0,0 +1,47 @@
+##! This script adds geographic location data to notices for the "remote"
+##! host in a connection.  It does make the assumption that one of the 
+##! addresses in a connection is "local" and one is "remote" which is 
+##! probably a safe assumption to make in most cases.  If both addresses
+##! are remote, it will use the $src address.
+
+module Notice;
+
+export {
+	redef enum Action += {
+		## Indicates that the notice should have geodata added for the
+		## "remote" host.  :bro:id:`Site::local_nets` must be defined
+		## in order for this to work.
+		ACTION_ADD_GEODATA
+	};
+	
+	redef record Info += {
+		## If libGeoIP support is built in, notices can have geographic
+		## information attached to them.
+		remote_location: geo_location  &log &optional;
+	};
+	
+	## Notice types which should have the "remote" location looked up.
+	## If GeoIP support is not built in, this does nothing.
+	const lookup_location_types: set[Notice::Type] = {} &redef;
+	
+	## Add a helper to the notice policy for looking up GeoIP data.
+	redef Notice::policy += {
+		[$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); },
+		 $priority = 10],
+	};
+}
+
+# This is handled at a high priority in case other notice handlers 
+# want to use the data.
+event notice(n: Notice::Info) &priority=10
+	{
+	if ( ACTION_ADD_GEODATA in n$actions &&
+	     |Site::local_nets| > 0 &&
+	     ! n?$remote_location )
+		{
+		if ( n?$src && ! Site::is_local_addr(n$src) )
+			n$remote_location = lookup_location(n$src);
+		else if ( n?$dst && ! Site::is_local_addr(n$dst) )
+			n$remote_location = lookup_location(n$dst);
+		}
+	}

scripts/base/init-bare.bro

@@ -509,7 +509,7 @@ const tunnel_port = 0/udp &redef;
 # packets and IP-level bytes transfered by each endpoint. If
 # true, these values are returned in the connection's endpoint
 # record val.
-const use_conn_size_analyzer = F &redef;
+const use_conn_size_analyzer = T &redef;
 
 const UDP_INACTIVE = 0;
 const UDP_ACTIVE = 1;	# means we've seen something from this endpoint

scripts/base/protocols/http/main.bro

@@ -214,7 +214,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr
 			c$http$response_content_length = extract_count(value);
 		else if ( name == "CONTENT-DISPOSITION" &&
 		          /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value )
-			c$http$filename = sub(value, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+			c$http$filename = extract_filename_from_content_disposition(value);
 		}
 	}
 	

scripts/base/protocols/mime/file-extract.bro

@@ -16,11 +16,13 @@ export {
 		extract_file:         bool    &default=F;
 	
 		## Store the file handle here for the file currently being extracted.
-		extraction_file:      file    &optional;
+		extraction_file:      file    &log &optional;
+	};
 
+	redef record State += {
 		## Store a count of the number of files that have been transferred in
 		## this conversation to create unique file names on disk.
-		num_extracted_files:  count   &optional;
+		num_extracted_files:  count   &default=0;
 	};
 }
 
@@ -34,7 +36,7 @@ event mime_segment_data(c: connection, length: count, data: string) &priority=3
 	{
 	if ( c$mime$extract_file && c$mime$content_len == 0 )
 		{
-		local suffix = fmt("%d.dat", ++c$mime$num_extracted_files);
+		local suffix = fmt("%d.dat", ++c$mime_state$num_extracted_files);
 		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
 		c$mime$extraction_file = open(fname);
 		enable_raw_output(c$mime$extraction_file);

scripts/base/protocols/smtp/__load__.bro

@@ -1 +1,3 @@
 @load ./main
+@load ./entities
+@load ./entities-excerpt

scripts/base/protocols/smtp/entities-excerpt.bro

@@ -0,0 +1,52 @@
+##! This script is for optionally adding a body excerpt to the SMTP
+##! entities log.
+
+@load ./entities
+
+module SMTP;
+
+export {
+	redef record SMTP::EntityInfo += {
+		## The entity body excerpt.
+		excerpt:    string &log &default="";
+		
+		## Internal tracking to know how much of the body should be included
+		## in the excerpt.
+		excerpt_len: count &optional;
+	};
+	
+	## This is the default value for how much of the entity body should be
+	## included for all MIME entities.
+	const default_entity_excerpt_len = 0 &redef;
+	
+	## This table defines how much of various entity bodies should be 
+	## included in excerpts.
+	const entity_excerpt_len: table[string] of count = {} 
+		&redef
+		&default = default_entity_excerpt_len;
+}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-1
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$excerpt_len = entity_excerpt_len[c$smtp$current_entity$mime_type];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-2
+	{
+	if ( ! c?$smtp ) return;
+	
+	local ent = c$smtp$current_entity;
+	if ( ent$content_len < ent$excerpt_len )
+		{
+		if ( ent$content_len + length < ent$excerpt_len )
+			ent$excerpt = cat(ent$excerpt, data);
+		else
+			{
+			local x_bytes = ent$excerpt_len - ent$content_len;
+			ent$excerpt = cat(ent$excerpt, sub_bytes(data, 1, x_bytes));
+			}
+		}
+	}

scripts/base/protocols/smtp/entities.bro

@@ -0,0 +1,234 @@
+##! Analysis and logging for MIME entities found in SMTP sessions.
+
+@load base/utils/strings
+@load base/utils/files
+@load ./main
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that an MD5 sum was calculated for a MIME message.
+		MD5,
+	};
+
+	redef enum Log::ID += { SMTP_ENTITIES };
+
+	type EntityInfo: record {
+		## This is the timestamp of when the MIME content transfer began.
+		ts:               time    &log;
+		uid:              string  &log;
+		id:               conn_id &log;
+		## Internally generated "message id" that ties back to the particular
+		## message in the SMTP log where this entity was seen.
+		mid:              string  &log;
+		## The filename seen in the Content-Disposition header.
+		filename:         string  &log &optional;
+		## Track how many bytes of the MIME encoded file have been seen.
+		content_len:      count   &log &default=0;
+		## The mime type of the entity discovered through magic bytes identification.
+		mime_type:        string  &log &optional;
+		
+		## The calculated MD5 sum for the MIME entity.
+		md5:              string  &log &optional;
+		## Optionally calculate the file's MD5 sum.  Must be set prior to the 
+		## first data chunk being see in an event.
+		calc_md5:         bool    &default=F;
+		## This boolean value indicates if an MD5 sum is being calculated 
+		## for the current file transfer.
+		calculating_md5:  bool    &default=F;
+		
+		## Optionally write the file to disk.  Must be set prior to first 
+		## data chunk being seen in an event.
+		extract_file:     bool    &default=F;
+		## Store the file handle here for the file currently being extracted.
+		extraction_file:  file    &log &optional;
+	};
+
+	redef record Info += {
+		## The in-progress entity information.
+		current_entity:   EntityInfo &optional;
+	};
+
+	redef record State += {
+		## Store a count of the number of files that have been transferred in
+		## a conversation to create unique file names on disk.
+		num_extracted_files:  count   &default=0;
+		## Track the number of MIME encoded files transferred during a session.
+		mime_level:           count   &default=0;
+	};
+
+	## Generate MD5 sums for these filetypes.
+	const generate_md5 = /application\/x-dosexec/    # Windows and DOS executables
+	                   | /application\/x-executable/ # *NIX executable binary
+	                   &redef;
+
+	## Pattern of file mime types to extract from MIME bodies.
+	const extract_file_types = /NO_DEFAULT/ &redef;
+
+	## The on-disk prefix for files to be extracted from MIME entity bodies.
+	const extraction_prefix = "smtp-entity" &redef;
+
+	global log_mime: event(rec: EntityInfo);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(SMTP_ENTITIES, [$columns=EntityInfo, $ev=log_mime]);
+	}
+
+function set_session(c: connection, new_entity: bool)
+	{
+	if ( ! c$smtp?$current_entity || new_entity )
+		{
+		local info: EntityInfo;
+		info$ts=network_time();
+		info$uid=c$uid;
+		info$id=c$id;
+		info$mid=c$smtp$mid;
+		
+		c$smtp$current_entity = info;
+		++c$smtp_state$mime_level;
+		}
+	}
+
+event mime_begin_entity(c: connection) &priority=10
+	{
+	if ( ! c?$smtp ) return;
+	
+	set_session(c, T);
+	}
+
+# This has priority -10 because other handlers need to know the current
+# content_len before it's updated by this handler.
+event mime_segment_data(c: connection, length: count, data: string) &priority=-10
+	{
+	if ( ! c?$smtp ) return;
+	
+	c$smtp$current_entity$content_len = c$smtp$current_entity$content_len + length;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=7
+    {
+	if ( ! c?$smtp ) return;
+	if ( c$smtp$current_entity$content_len == 0 )
+		c$smtp$current_entity$mime_type = split1(identify_data(data, T), /;/)[1];
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	if ( c$smtp$current_entity$content_len == 0 )
+		{
+		if ( generate_md5 in c$smtp$current_entity$mime_type )
+			c$smtp$current_entity$calc_md5 = T;
+
+		if ( c$smtp$current_entity$calc_md5 )
+			{
+			c$smtp$current_entity$calculating_md5 = T;
+			md5_hash_init(c$id);
+			}
+		}
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		md5_hash_update(c$id, data);
+}
+
+## In the event of a content gap during the MIME transfer, detect the state for
+## the MD5 sum calculation and stop calculating the MD5 since it would be
+## incorrect anyway.
+event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5
+	{
+	if ( is_orig || ! c?$smtp || ! c$smtp?$current_entity ) return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$calculating_md5 = F;
+		md5_hash_finish(c$id);
+		}
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+    {
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c?$smtp || ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity$calculating_md5 )
+		{
+		c$smtp$current_entity$md5 = md5_hash_finish(c$id);
+
+		NOTICE([$note=MD5, $msg=fmt("Calculated a hash for a MIME entity from %s", c$id$orig_h),
+				$sub=c$smtp$current_entity$md5, $conn=c]);
+		}
+	}
+
+event mime_one_header(c: connection, h: mime_header_rec)
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( h$name == "CONTENT-DISPOSITION" &&
+	     /[fF][iI][lL][eE][nN][aA][mM][eE]/ in h$value )
+		c$smtp$current_entity$filename = extract_filename_from_content_disposition(h$value);
+	}
+
+event mime_end_entity(c: connection) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+
+	# This check and the delete below are just to cope with a bug where
+	# mime_end_entity can be generated multiple times for the same event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	# Only log is there was some content.
+	if ( c$smtp$current_entity$content_len > 0 )
+		Log::write(SMTP_ENTITIES, c$smtp$current_entity);
+
+	delete c$smtp$current_entity;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( extract_file_types in c$smtp$current_entity$mime_type )
+		c$smtp$current_entity$extract_file = T;
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=3
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && 
+	     c$smtp$current_entity$content_len == 0 )
+		{
+		local suffix = fmt("%d.dat", ++c$smtp_state$num_extracted_files);
+		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
+		c$smtp$current_entity$extraction_file = open(fname);
+		enable_raw_output(c$smtp$current_entity$extraction_file);
+		}
+	}
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=-5
+	{
+	if ( ! c?$smtp ) return;
+	
+	if ( c$smtp$current_entity$extract_file && c$smtp$current_entity?$extraction_file )
+		print c$smtp$current_entity$extraction_file, data;
+	}
+
+event mime_end_entity(c: connection) &priority=-3
+	{
+	if ( ! c?$smtp ) return;
+	
+	# TODO: this check is only due to a bug in mime_end_entity that
+	#       causes the event to be generated twice for the same real event.
+	if ( ! c$smtp?$current_entity )
+		return;
+
+	if ( c$smtp$current_entity?$extraction_file )
+		close(c$smtp$current_entity$extraction_file);
+	}

scripts/base/protocols/smtp/main.bro

@@ -4,17 +4,14 @@ module SMTP;
 export {
 	redef enum Log::ID += { SMTP };
 
-	redef enum Notice::Type += { 
-		## Indicates that the server sent a reply mentioning an SMTP block list.
-		BL_Error_Message, 
-		## Indicates the client's address is seen in the block list error message.
-		BL_Blocked_Host, 
-	};
-
 	type Info: record {
 		ts:                time            &log;
 		uid:               string          &log;
 		id:                conn_id         &log;
+		## This is an internally generated "message id" that can be used to
+		## map between SMTP messages and MIME entities in the SMTP entities
+		## log.
+		mid:               string          &log;
 		helo:              string          &log &optional;
 		mailfrom:          string          &log &optional;
 		rcptto:            set[string]     &log &optional;
@@ -30,19 +27,13 @@ export {
 		second_received:   string          &log &optional;
 		## The last message the server sent to the client.
 		last_reply:        string          &log &optional;
-		files:             set[string]     &log &optional;
 		path:              vector of addr  &log &optional;
 		user_agent:        string          &log &optional;
 		
-		## Indicate if this session is currently transmitting SMTP message 
-		## envelope headers.
-		in_headers:        bool            &default=F;
 		## Indicate if the "Received: from" headers should still be processed.
 		process_received_from: bool        &default=T;
-		## Maintain the current header for cases where there is header wrapping.
+		## Indicates if client activity has been seen, but not yet logged
-		current_header:    string          &default="";
+		has_client_activity:  bool            &default=F;
-		## Indicate when the message is logged and no longer applicable.
-		done:              bool            &default=F;
 	};
 	
 	type State: record {
@@ -62,25 +53,6 @@ export {
 	##    NO_HOSTS - never capture the path.
 	const mail_path_capture = ALL_HOSTS &redef;
 		
-	# This matches content in SMTP error messages that indicate some
-	# block list doesn't like the connection/mail.
-	const bl_error_messages = 
-	    /spamhaus\.org\//
-	  | /sophos\.com\/security\//
-	  | /spamcop\.net\/bl/
-	  | /cbl\.abuseat\.org\// 
-	  | /sorbs\.net\// 
-	  | /bsn\.borderware\.com\//
-	  | /mail-abuse\.com\//
-	  | /b\.barracudacentral\.com\//
-	  | /psbl\.surriel\.com\// 
-	  | /antispam\.imp\.ch\// 
-	  | /dyndns\.com\/.*spam/
-	  | /rbl\.knology\.net\//
-	  | /intercept\.datapacket\.net\//
-	  | /uceprotect\.net\//
-	  | /hostkarma\.junkemailfilter\.com\// &redef;
-	
 	global log_smtp: event(rec: Info);
 	
 	## Configure the default ports for SMTP analysis.
@@ -121,6 +93,7 @@ function new_smtp_log(c: connection): Info
 	l$ts=network_time();
 	l$uid=c$uid;
 	l$id=c$id;
+	l$mid=unique_id("@");
 	if ( c?$smtp_state && c$smtp_state?$helo )
 		l$helo = c$smtp_state$helo;
 	
@@ -136,20 +109,14 @@ function set_smtp_session(c: connection)
 	if ( ! c?$smtp_state )
 		c$smtp_state = [];
 	
-	if ( ! c?$smtp || c$smtp$done )
+	if ( ! c?$smtp )
-		{
 		c$smtp = new_smtp_log(c);
 	}
-	}
-
 
 function smtp_message(c: connection)
 	{
+	if ( c$smtp$has_client_activity )
 		Log::write(SMTP, c$smtp);
-
-	c$smtp$done = T;
-	# Track the number of messages seen in this session.
-	++c$smtp_state$messages_transferred;
 	}
 	
 event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &priority=5
@@ -157,6 +124,9 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 	set_smtp_session(c);
 	local upper_command = to_upper(command);
 
+	if ( upper_command != "QUIT" )
+		c$smtp$has_client_activity = T;
+	
 	if ( upper_command == "HELO" || upper_command == "EHLO" )
 		{
 		c$smtp_state$helo = arg;
@@ -172,25 +142,10 @@ event smtp_request(c: connection, is_orig: bool, command: string, arg: string) &
 
 	else if ( upper_command == "MAIL" && /^[fF][rR][oO][mM]:/ in arg )
 		{
-		# In case this is not the first message in a session we want to 
-		# essentially write out a log, clear the session tracking, and begin
-		# new session tracking.
-		if ( c$smtp_state$messages_transferred > 0 )
-			{
-			smtp_message(c);
-			set_smtp_session(c);
-			}
-		
 		local partially_done = split1(arg, /:[[:blank:]]*/)[2];
 		c$smtp$mailfrom = split1(partially_done, /[[:blank:]]?/)[1];
 		}
-		
-	else if ( upper_command == "DATA" )
-		{
-		c$smtp$in_headers = T;
 	}
-	}
-	
 	
 event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
                  msg: string, cont_resp: bool) &priority=5
@@ -199,152 +154,83 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 	
 	# This continually overwrites, but we want the last reply,
 	# so this actually works fine.
-	if ( code != 421 && code >= 400 )
-		{
 	c$smtp$last_reply = fmt("%d %s", code, msg);
+	}
 
-		# Raise a notice when an SMTP error about a block list is discovered.
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
-		if ( bl_error_messages in msg )
+                 msg: string, cont_resp: bool) &priority=-5
 	{
-			local note = BL_Error_Message;
-			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
-
-			# Determine if the originator's IP address is in the message.
-			local ips = find_ip_addresses(msg);
-			local text_ip = "";
-			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
-				{
-				note = BL_Blocked_Host;
-				message = fmt("%s is on an SMTP block list", c$id$orig_h);
-				}
-			
-			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
-			}
-		}
-	}
-
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=5
-	{
-	# Is there something we should be handling from the server?
-	if ( ! is_orig ) return;
-	
 	set_smtp_session(c);
-	
+	if ( cmd == "." )
-	if ( ! c$smtp$in_headers )
 		{
-		if ( /^[cC][oO][nN][tT][eE][nN][tT]-[dD][iI][sS].*[fF][iI][lL][eE][nN][aA][mM][eE]/ in data )
+		# Track the number of messages seen in this session.
-			{
+		++c$smtp_state$messages_transferred;
-			if ( ! c$smtp?$files )
+		smtp_message(c);
-				c$smtp$files = set();
+		c$smtp = new_smtp_log(c);
-			data = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
-			add c$smtp$files[data];
 		}
-		return;
 	}
 
-	if ( /^[[:blank:]]*$/ in data )
+event mime_one_header(c: connection, h: mime_header_rec) &priority=5
-		c$smtp$in_headers = F;
-
-	# This is to reconstruct headers that tend to wrap around.
-	if ( /^[[:blank:]]/ in data )
 	{
-		# Remove all but a single space at the beginning (this seems to follow
+	if ( ! c?$smtp ) return;
-		# the most common behavior).
+	c$smtp$has_client_activity = T;
-		data = sub(data, /^[[:blank:]]*/, " ");
-		if ( c$smtp$current_header == "MESSAGE-ID" )
-			c$smtp$msg_id += data;
-		else if ( c$smtp$current_header == "RECEIVED" )
-			c$smtp$first_received += data;
-		else if ( c$smtp$current_header == "IN-REPLY-TO" )
-			c$smtp$in_reply_to += data;
-		else if ( c$smtp$current_header == "SUBJECCT" )
-			c$smtp$subject += data;
-		else if ( c$smtp$current_header == "FROM" )
-			c$smtp$from += data;
-		else if ( c$smtp$current_header == "REPLY-TO" )
-			c$smtp$reply_to += data;
-		else if ( c$smtp$current_header == "USER-AGENT" )
-			c$smtp$user_agent += data;
-		return;
-		}
-	# Once there isn't a line starting with a blank, we're not continuing a 
-	# header anymore.
-	c$smtp$current_header = "";
 
-	local header_parts = split1(data, /:[[:blank:]]*/);
+	if ( h$name == "MESSAGE-ID" )
-	# TODO: do something in this case?  This would definitely be odd.
+		c$smtp$msg_id = h$value;
-	# Header wrapping needs to be handled more elegantly.  This will happen
-	# if the header value is wrapped immediately after the header key.
-	if ( |header_parts| != 2 )
-		return;
 
-	local header_key = to_upper(header_parts[1]);
+	else if ( h$name == "RECEIVED" )
-	c$smtp$current_header = header_key;
-	
-	local header_val = header_parts[2];
-	
-	if ( header_key == "MESSAGE-ID" )
-		c$smtp$msg_id = header_val;
-	
-	else if ( header_key == "RECEIVED" )
 		{
 		if ( c$smtp?$first_received )
 			c$smtp$second_received = c$smtp$first_received;
-		c$smtp$first_received = header_val;
+		c$smtp$first_received = h$value;
 		}
 
-	else if ( header_key == "IN-REPLY-TO" )
+	else if ( h$name == "IN-REPLY-TO" )
-		c$smtp$in_reply_to = header_val;
+		c$smtp$in_reply_to = h$value;
 
-	else if ( header_key == "DATE" )
+	else if ( h$name == "SUBJECT" )
-		c$smtp$date = header_val;
+		c$smtp$subject = h$value;
 
-	else if ( header_key == "FROM" )
+	else if ( h$name == "FROM" )
-		c$smtp$from = header_val;
+		c$smtp$from = h$value;
 
-	else if ( header_key == "TO" )
+	else if ( h$name == "REPLY-TO" )
+		c$smtp$reply_to = h$value;
+
+	else if ( h$name == "DATE" )
+		c$smtp$date = h$value;
+
+	else if ( h$name == "TO" )
 		{
 		if ( ! c$smtp?$to )
 			c$smtp$to = set();
-		add c$smtp$to[header_val];
+		add c$smtp$to[h$value];
 		}
 
-	else if ( header_key == "REPLY-TO" )
+	else if ( h$name == "X-ORIGINATING-IP" )
-		c$smtp$reply_to = header_val;
-	
-	else if ( header_key == "SUBJECT" )
-		c$smtp$subject = header_val;
-
-	else if ( header_key == "X-ORIGINATING-IP" )
 		{
-		local addresses = find_ip_addresses(header_val);
+		local addresses = find_ip_addresses(h$value);
 		if ( 1 in addresses )
 			c$smtp$x_originating_ip = to_addr(addresses[1]);
 		}
 	
-	else if ( header_key == "X-MAILER" || 
+	else if ( h$name == "X-MAILER" ||
-	          header_key == "USER-AGENT" ||
+	          h$name == "USER-AGENT" ||
-	          header_key == "X-USER-AGENT" )
+	          h$name == "X-USER-AGENT" )
-		{
+		c$smtp$user_agent = h$value;
-		c$smtp$user_agent = header_val;
-		# Explicitly set the current header here because there are several 
-		# headers bulked under this same key.
-		c$smtp$current_header = "USER-AGENT";
-		}
 	}
 	
 # This event handler builds the "Received From" path by reading the 
 # headers in the mail
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=3
+event mime_one_header(c: connection, h: mime_header_rec) &priority=3
 	{
 	# If we've decided that we're done watching the received headers for
 	# whatever reason, we're done.  Could be due to only watching until 
 	# local addresses are seen in the received from headers.
-	if ( c$smtp$current_header != "RECEIVED" ||
+	if ( ! c?$smtp || h$name != "RECEIVED" || ! c$smtp$process_received_from )
-	     ! c$smtp$process_received_from )
 		return;
 
-	local text_ip = find_address_in_smtp_header(data);
+	local text_ip = find_address_in_smtp_header(h$value);
 	if ( text_ip == "" )
 		return;
 	local ip = to_addr(text_ip);
@@ -354,14 +240,12 @@ event smtp_data(c: connection, is_orig: bool, data: string) &priority=3
 		{
 		c$smtp$process_received_from = F;
 		}
-	
 	if ( c$smtp$path[|c$smtp$path|-1] != ip )
 		c$smtp$path[|c$smtp$path|] = ip;
 	}
 
-
 event connection_state_remove(c: connection) &priority=-5
 	{
-	if ( c?$smtp && ! c$smtp$done )
+	if ( c?$smtp )
 		smtp_message(c);
 	}

scripts/base/protocols/ssh/main.bro

@@ -1,74 +1,58 @@
+##! Base SSH analysis script.  The heuristic to blindly determine success or 
+##! failure for SSH connections is implemented here.  At this time, it only
+##! uses the size of the data being returned from the server to make the
+##! heuristic determination about success of the connection.  
+##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
+##! is not attempted if the connection size analyzer isn't enabled.
 
 module SSH;
 
 export {
 	redef enum Log::ID += { SSH };
 
-	redef enum Notice::Type += {
-		Login,
-		Password_Guessing,
-		Login_By_Password_Guesser,
-		Login_From_Interesting_Hostname,
-		Bytecount_Inconsistency,
-	};
-
 	type Info: record {
 		ts:              time         &log;
 		uid:             string       &log;
 		id:              conn_id      &log;
+		## Indicates if the login was heuristically guessed to be "success"
+		## or "failure".
 		status:          string       &log &optional;
-		direction:       string       &log &optional;
+		## Direction of the connection.  If the client was a local host 
-		remote_location: geo_location &log &optional;
+		## logging into an external host, this would be OUTBOUD.  INBOUND
+		## would be set for the opposite situation.
+		# TODO: handle local-local and remote-remote better.
+		direction:       Direction    &log &optional;
+		## The software string given by the client.
 		client:          string       &log &optional;
+		## The software string given by the server.
 		server:          string       &log &optional;
+		## The amount of data returned from the server.  This is currently
+		## the only measure of the success heuristic and it is logged to 
+		## assist analysts looking at the logs to make their own determination
+		## about the success on a case-by-case basis.
 		resp_size:       count        &log &default=0;
 		
 		## Indicate if the SSH session is done being watched.
 		done:            bool         &default=F;
 	};
 	
-	const password_guesses_limit = 30 &redef;
+	## The size in bytes at which the SSH connection is presumed to be
-	
+	## successful.
-	# The size in bytes at which the SSH connection is presumed to be
-	# successful.
 	const authentication_data_size = 5500 &redef;
 	
-	# The amount of time to remember presumed non-successful logins to build
+	## If true, we tell the event engine to not look at further data
-	# model of a password guesser.
+	## packets after the initial SSH handshake. Helps with performance
-	const guessing_timeout = 30 mins &redef;
+	## (especially with large file transfers) but precludes some
-
+	## kinds of analyses (e.g., tracking connection size).
-	# The set of countries for which you'd like to throw notices upon successful login
-	#   requires Bro compiled with libGeoIP support
-	const watched_countries: set[string] = {"RO"} &redef;
-
-	# Strange/bad host names to originate successful SSH logins
-	const interesting_hostnames =
-			/^d?ns[0-9]*\./ |
-			/^smtp[0-9]*\./ |
-			/^mail[0-9]*\./ |
-			/^pop[0-9]*\./  |
-			/^imap[0-9]*\./ |
-			/^www[0-9]*\./  |
-			/^ftp[0-9]*\./  &redef;
-
-	# This is a table with orig subnet as the key, and subnet as the value.
-	const ignore_guessers: table[subnet] of subnet &redef;
-	
-	# If true, we tell the event engine to not look at further data
-	# packets after the initial SSH handshake. Helps with performance
-	# (especially with large file transfers) but precludes some
-	# kinds of analyses (e.g., tracking connection size).
 	const skip_processing_after_detection = F &redef;
 	
-	# Keeps count of how many rejections a host has had
+	## This event is generated when the heuristic thinks that a login
-	global password_rejections: table[addr] of TrackCount 
+	## was successful.
-		&write_expire=guessing_timeout
+	global heuristic_successful_login: event(c: connection);
-		&synchronized;
 	
-	# Keeps track of hosts identified as guessing passwords
+	## This event is generated when the heuristic thinks that a login
-	# TODO: guessing_timeout doesn't work correctly here.  If a user redefs
+	## failed.
-	#       the variable, it won't take effect.
+	global heuristic_failed_login: event(c: connection);
-	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
 	
 	global log_ssh: event(rec: Info);
 }
@@ -106,116 +90,51 @@ function check_ssh_connection(c: connection, done: bool)
 	
 	# If this is still a live connection and the byte count has not
 	# crossed the threshold, just return and let the resheduled check happen later.
-	if ( !done && c$resp$size < authentication_data_size )
+	if ( !done && c$resp$num_bytes_ip < authentication_data_size )
 		return;
 
 	# Make sure the server has sent back more than 50 bytes to filter out
 	# hosts that are just port scanning.  Nothing is ever logged if the server
 	# doesn't send back at least 50 bytes.
-	if ( c$resp$size < 50 )
+	if ( c$resp$num_bytes_ip < 50 )
 		return;
 
-	local status = "failure";
+	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
-	local direction = Site::is_local_addr(c$id$orig_h) ? "to" : "from";
+	c$ssh$resp_size = c$resp$num_bytes_ip;
-	local location: geo_location;
-	location = (direction == "to") ? lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
 	
-	if ( done && c$resp$size < authentication_data_size )
+	if ( c$resp$num_bytes_ip < authentication_data_size )
 		{
-		# presumed failure
+		c$ssh$status  = "failure";
-		if ( c$id$orig_h !in password_rejections )
+		event SSH::heuristic_failed_login(c);
-			password_rejections[c$id$orig_h] = new_track_count();
-			
-		# Track the number of rejections
-		if ( !(c$id$orig_h in ignore_guessers &&
-		       c$id$resp_h in ignore_guessers[c$id$orig_h]) )
-			++password_rejections[c$id$orig_h]$n;
-			
-		if ( default_check_threshold(password_rejections[c$id$orig_h]) )
-			{
-			add password_guessers[c$id$orig_h];
-			NOTICE([$note=Password_Guessing,
-			        $conn=c,
-			        $msg=fmt("SSH password guessing by %s", c$id$orig_h),
-			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n),
-			        $n=password_rejections[c$id$orig_h]$n]);
 		}
-		} 
+	else
-	# TODO: This is to work around a quasi-bug in Bro which occasionally 
-	#       causes the byte count to be oversized.
-	#   Watch for Gregors work that adds an actual counter of bytes transferred.
-	else if ( c$resp$size < 20000000 ) 
 		{ 
 		# presumed successful login
-		status = "success";
+		c$ssh$status = "success";
-		c$ssh$done = T;
+		event SSH::heuristic_successful_login(c);
-
-		if ( c$id$orig_h in password_rejections &&
-		     password_rejections[c$id$orig_h]$n > password_guesses_limit &&
-		     c$id$orig_h !in password_guessers )
-			{
-			add password_guessers[c$id$orig_h];
-			NOTICE([$note=Login_By_Password_Guesser,
-			        $conn=c,
-			        $n=password_rejections[c$id$orig_h]$n,
-			        $msg=fmt("Successful SSH login by password guesser %s", c$id$orig_h),
-			        $sub=fmt("%d failed logins", password_rejections[c$id$orig_h]$n)]);
 		}
 	
-		local message = fmt("SSH login %s %s \"%s\" \"%s\" %f %f %s (triggered with %d bytes)",
-		              direction, location$country_code, location$region, location$city,
-		              location$latitude, location$longitude,
-		              id_string(c$id), c$resp$size);
-		NOTICE([$note=Login,
-		        $conn=c,
-		        $msg=message,
-		        $sub=location$country_code]);
-		
-		# Check to see if this login came from an interesting hostname
-		when ( local hostname = lookup_addr(c$id$orig_h) )
-			{
-			if ( interesting_hostnames in hostname )
-				{
-				NOTICE([$note=Login_From_Interesting_Hostname,
-				        $conn=c,
-				        $msg=fmt("Strange login from %s", hostname),
-				        $sub=hostname]);
-				}
-			}
-			
-		if ( location$country_code in watched_countries )
-			{
-			
-			}
-			
-		}
-	else if ( c$resp$size >= 200000000 ) 
-		{
-		NOTICE([$note=Bytecount_Inconsistency,
-		        $conn=c,
-		        $msg="During byte counting in SSH analysis, an overly large value was seen.",
-		        $sub=fmt("%d",c$resp$size)]);
-		}
-
-	c$ssh$remote_location = location;
-	c$ssh$status = status;
-	c$ssh$direction = direction;
-	c$ssh$resp_size = c$resp$size;
-	
-	Log::write(SSH, c$ssh);
-	
 	# Set the "done" flag to prevent the watching event from rescheduling
 	# after detection is done.
-	c$ssh$done;
+	c$ssh$done=T;
 	
-	# Stop watching this connection, we don't care about it anymore.
 	if ( skip_processing_after_detection )
 		{
+		# Stop watching this connection, we don't care about it anymore.
 		skip_further_processing(c$id);
 		set_record_packets(c$id, F);
 		}
 	}
 
+event SSH::heuristic_successful_login(c: connection) &priority=-5
+	{
+	Log::write(SSH, c$ssh);
+	}
+event SSH::heuristic_failed_login(c: connection) &priority=-5
+	{
+	Log::write(SSH, c$ssh);
+	}
+
 event connection_state_remove(c: connection) &priority=-5
 	{
 	if ( c?$ssh )
@@ -244,5 +163,9 @@ event ssh_client_version(c: connection, version: string) &priority=5
 	{
 	set_session(c);
 	c$ssh$client = version;
+	
+	# The heuristic detection for SSH relies on the ConnSize analyzer.
+	# Don't do the heuristics if it's disabled.
+	if ( use_conn_size_analyzer )
 		schedule +15secs { ssh_watcher(c) };
 	}

scripts/base/utils/files.bro

@@ -13,3 +13,14 @@ function generate_extraction_filename(prefix: string, c: connection, suffix: str
 		
 	return conn_info;
 	}
+	
+## For CONTENT-DISPOSITION headers, this function can be used to extract 
+## the filename.
+function extract_filename_from_content_disposition(data: string): string
+	{
+	local filename = sub(data, /^.*[fF][iI][lL][eE][nN][aA][mM][eE]=/, "");
+	# Remove quotes around the filename if they are there.
+	if ( /^\"/ in filename )
+		filename =  split_n(filename, /\"/, F, 2)[2];
+	return filename;
+	}

scripts/base/utils/site.bro

@@ -17,6 +17,10 @@ export {
 	## Networks that are considered "local".
 	const local_nets: set[subnet] &redef;
 	
+	## This is used for mapping between local networks and string
+	## values for the CIDRs represented.
+	global local_nets_table: table[subnet] of string = {};
+
 	## Networks that are considered "neighbors".
 	const neighbor_nets: set[subnet] &redef;
 	
@@ -138,4 +142,9 @@ event bro_init() &priority=10
 	# Double backslashes are needed due to string parsing.
 	local_dns_suffix_regex = set_to_regex(local_zones, "(^\\.?|\\.)(~~)$");
 	local_dns_neighbor_suffix_regex = set_to_regex(neighbor_zones, "(^\\.?|\\.)(~~)$");
+
+	# Create the local_nets mapping table.
+	for ( cidr in Site::local_nets )
+		local_nets_table[cidr] = fmt("%s", cidr);
+
 	}

scripts/policy/frameworks/communication/listen-clear.bro

@@ -15,5 +15,6 @@ export {
 
 event bro_init() &priority=-10
 	{
+	enable_communication();
 	listen(listen_if_clear, listen_port_clear, F);
 	}

scripts/policy/frameworks/communication/listen-ssl.bro

@@ -16,5 +16,6 @@ export {
 
 event bro_init() &priority=-10
 	{
+	enable_communication();
 	listen(listen_if_ssl, listen_port_ssl, T);
 	}

scripts/policy/frameworks/metrics/conn-example.bro

@@ -0,0 +1,20 @@
+
+redef enum Metrics::ID += { 
+	CONNS_ORIGINATED, 
+	CONNS_RESPONDED 
+};
+
+event bro_init()
+	{
+	Metrics::add_filter(CONNS_ORIGINATED, [$aggregation_mask=24, $break_interval=1mins]);
+	
+	# Site::local_nets must be defined in order for this to actually do anything.
+	Metrics::add_filter(CONNS_RESPONDED,  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
+	}
+
+event connection_established(c: connection)
+	{
+	Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h]);
+	Metrics::add_data(CONNS_RESPONDED,  [$host=c$id$resp_h]);
+	}
+	

scripts/policy/frameworks/metrics/http-example.bro

@@ -0,0 +1,22 @@
+
+
+redef enum Metrics::ID += {
+	HTTP_REQUESTS_BY_STATUS_CODE,
+	HTTP_REQUESTS_BY_HOST_HEADER,
+};
+
+event bro_init()
+	{
+	Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER, [$break_interval=5mins]);
+	
+	# Site::local_nets must be defined in order for this to actually do anything.
+	Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table, $break_interval=5mins]);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( rec?$host )
+		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$index=rec$host]);
+	if ( rec?$status_code )
+		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $index=fmt("%d", rec$status_code)]);
+	}

scripts/policy/frameworks/metrics/ssl-example.bro

@@ -0,0 +1,22 @@
+
+
+redef enum Metrics::ID += {
+	SSL_SERVERNAME,
+};
+
+event bro_init()
+	{
+	Metrics::add_filter(SSL_SERVERNAME, 
+		[$name="no-google-ssl-servers",
+		 $pred(entry: Metrics::Entry) = { 
+		    return (/google\.com$/ !in entry$index); 
+		 },
+		 $break_interval=10secs
+		]);
+	}
+
+event SSL::log_ssl(rec: SSL::Info)
+	{
+	if ( rec?$server_name )
+		Metrics::add_data(SSL_SERVERNAME, [$index=rec$server_name]);
+	}

scripts/policy/misc/profiling.bro

@@ -0,0 +1,19 @@
+##! Turns on profiling of Bro resource consumption.
+
+module Profiling;
+
+redef profiling_file = open_log_file("prof");
+
+export {
+	## Cheap profiling every 15 seconds.
+	redef profiling_interval = 15 secs &redef;
+}
+
+# Expensive profiling every 5 minutes.
+redef expensive_profiling_multiple = 20;
+
+event bro_init()
+	{
+	set_buf(profiling_file, F);
+	}
+

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,7 +1,7 @@
 ##! This script takes MD5 sums of files transferred over HTTP and checks them with
 ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
-##! documentation for the protocols/http/file-hash.bro script to see how to 
+##! documentation for the base/protocols/http/file-hash.bro script to see how to 
 ##! configure which transfers will have hashes calculated.
 
 export {

scripts/policy/protocols/http/detect-sqli.bro

@@ -9,17 +9,17 @@ export {
 	};
 	
 	redef enum Metrics::ID += {
-		SQL_ATTACKER,
+		SQL_ATTACKS,
-		SQL_ATTACK_AGAINST,
+		SQL_ATTACKS_AGAINST,
 	};
 
 	redef enum Tags += {
 		## Indicator of a URI based SQL injection attack.
 		URI_SQLI,
 		## Indicator of client body based SQL injection attack.  This is 
-		## typically the body content of a POST request. Not implemented yet!
+		## typically the body content of a POST request. Not implemented yet.
 		POST_SQLI,
-		## Indicator of a cookie based SQL injection attack. Not implemented yet!
+		## Indicator of a cookie based SQL injection attack. Not implemented yet.
 		COOKIE_SQLI,
 	};
 
@@ -30,13 +30,18 @@ export {
 		| /[\?&][^[:blank:]\x00-\x37]+?=[\-0-9%]*([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/)*(-|=|\+|\|\|)([[:blank:]\x00-\x37]|\/\*.*?\*\/)*([0-9]|\(?[cC][oO][nN][vV][eE][rR][tT]|[cC][aA][sS][tT])/
 		| /[\?&][^[:blank:]\x00-\x37\|]+?=([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]([[:blank:]\x00-\x37]|\/\*.*?\*\/|;)*([xX]?[oO][rR]|[nN]?[aA][nN][dD]|[hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[rR][eE][gG][eE][xX][pP]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/|[\[(])+[a-zA-Z&]{2,}/
 		| /[\?&][^[:blank:]\x00-\x37]+?=[^\.]*?([cC][hH][aA][rR]|[aA][sS][cC][iI][iI]|[sS][uU][bB][sS][tT][rR][iI][nN][gG]|[tT][rR][uU][nN][cC][aA][tT][eE]|[vV][eE][rR][sS][iI][oO][nN]|[lL][eE][nN][gG][tT][hH])\(/
-		| /\/\*![[:digit:]]{5}.*?\*\//;
+		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
 event bro_init()
 	{
-	Metrics::add_filter(SQL_ATTACKER, [$break_interval=5mins, $note=SQL_Injection_Attack]);
+	Metrics::add_filter(SQL_ATTACKS, [$log=T, 
-	Metrics::add_filter(SQL_ATTACK_AGAINST, [$break_interval=5mins, $note=SQL_Injection_Attack]);
+	                                  $break_interval=1mins, 
+	                                  $note=SQL_Injection_Attacker]);
+	Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=T, 
+	                                          $break_interval=1mins, 
+	                                          $note=SQL_Injection_Attack, 
+	                                          $notice_thresholds=vector(10,100)]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -46,7 +51,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 		
-		Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1);
+		Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h]);
-		Metrics::add_data(SQL_ATTACK_AGAINST, [$host=c$id$resp_h], 1);
+		Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h]);
 		}
 	}

scripts/policy/protocols/smtp/blocklists.bro

@@ -0,0 +1,58 @@
+
+@load base/protocols/smtp
+
+module SMTP;
+
+export {
+	redef enum Notice::Type += { 
+		## Indicates that the server sent a reply mentioning an SMTP block list.
+		Blocklist_Error_Message,
+		## Indicates the client's address is seen in the block list error message.
+		Blocklist_Blocked_Host,
+	};
+
+	# This matches content in SMTP error messages that indicate some
+	# block list doesn't like the connection/mail.
+	const blocklist_error_messages = 
+		  /spamhaus\.org\//
+		| /sophos\.com\/security\//
+		| /spamcop\.net\/bl/
+		| /cbl\.abuseat\.org\// 
+		| /sorbs\.net\// 
+		| /bsn\.borderware\.com\//
+		| /mail-abuse\.com\//
+		| /b\.barracudacentral\.com\//
+		| /psbl\.surriel\.com\// 
+		| /antispam\.imp\.ch\// 
+		| /dyndns\.com\/.*spam/
+		| /rbl\.knology\.net\//
+		| /intercept\.datapacket\.net\//
+		| /uceprotect\.net\//
+		| /hostkarma\.junkemailfilter\.com\// &redef;
+	
+}
+
+event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
+                 msg: string, cont_resp: bool) &priority=3
+	{
+	if ( code >= 400 && code != 421 )
+		{
+		# Raise a notice when an SMTP error about a block list is discovered.
+		if ( blocklist_error_messages in msg )
+			{
+			local note = Blocklist_Error_Message;
+			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
+
+			# Determine if the originator's IP address is in the message.
+			local ips = find_ip_addresses(msg);
+			local text_ip = "";
+			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
+				{
+				note = Blocklist_Blocked_Host;
+				message = fmt("%s is on an SMTP block list", c$id$orig_h);
+				}
+			
+			NOTICE([$note=note, $conn=c, $msg=message, $sub=msg]);
+			}
+		}
+	}

scripts/policy/protocols/smtp/software.bro

@@ -43,10 +43,10 @@ export {
 	                   | /ZimbraWebClient/ &redef;
 }
 
-event smtp_data(c: connection, is_orig: bool, data: string) &priority=4
+event mime_one_header(c: connection, h: mime_header_rec) &priority=4
 	{
-	if ( c$smtp$current_header == "USER-AGENT" &&
+	if ( ! c?$smtp ) return;
-	     webmail_user_agents in c$smtp$user_agent )
+	if ( h$name == "USER-AGENT" && webmail_user_agents in c$smtp$user_agent )
 		c$smtp$is_webmail = T;
 	}
 

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -0,0 +1,79 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates that a host has been identified as crossing the 
+		## :bro:id:`password_guesses_limit` threshold with heuristically
+		## determined failed logins.
+		Password_Guessing,
+		## Indicates that a host previously identified as a "password guesser"
+		## has now had a heuristically successful login attempt.
+		Login_By_Password_Guesser,
+	};
+
+	## The number of failed SSH connections before a host is designated as
+	## guessing passwords.
+	const password_guesses_limit = 30 &redef;
+
+	## The amount of time to remember presumed non-successful logins to build
+	## model of a password guesser.
+	const guessing_timeout = 30 mins &redef;
+
+	## This value can be used to exclude hosts or entire networks from being 
+	## tracked as potential "guessers".  There are cases where the success
+	## heuristic fails and this acts as the whitelist.  The index represents 
+	## client subnets and the yield value represents server subnets.
+	const ignore_guessers: table[subnet] of subnet &redef;
+
+	## Keeps count of how many rejections a host has had.
+	global password_rejections: table[addr] of TrackCount 
+		&write_expire=guessing_timeout
+		&synchronized;
+
+	## Keeps track of hosts identified as guessing passwords.
+	global password_guessers: set[addr] &read_expire=guessing_timeout+1hr &synchronized;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	local id = c$id;
+	
+	# TODO: this should be migrated to the metrics framework.
+	if ( id$orig_h in password_rejections &&
+	     password_rejections[id$orig_h]$n > password_guesses_limit &&
+	     id$orig_h !in password_guessers )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Login_By_Password_Guesser,
+		        $conn=c,
+		        $n=password_rejections[id$orig_h]$n,
+		        $msg=fmt("Successful SSH login by password guesser %s", id$orig_h),
+		        $sub=fmt("%d failed logins", password_rejections[id$orig_h]$n)]);
+		}
+	}
+
+event SSH::heuristic_failed_login(c: connection)
+	{
+	local id = c$id;
+	
+	# presumed failure
+	if ( id$orig_h !in password_rejections )
+		password_rejections[id$orig_h] = new_track_count();
+	
+	# Track the number of rejections
+	# TODO: this should be migrated to the metrics framework.
+	if ( ! (id$orig_h in ignore_guessers &&
+	        id$resp_h in ignore_guessers[id$orig_h]) )
+		++password_rejections[id$orig_h]$n;
+	
+	if ( default_check_threshold(password_rejections[id$orig_h]) )
+		{
+		add password_guessers[id$orig_h];
+		NOTICE([$note=Password_Guessing,
+		        $conn=c,
+		        $msg=fmt("SSH password guessing by %s", id$orig_h),
+		        $sub=fmt("%d apparently failed logins", password_rejections[id$orig_h]$n),
+		        $n=password_rejections[id$orig_h]$n]);
+		}
+	}

scripts/policy/protocols/ssh/geo-data.bro

@@ -0,0 +1,39 @@
+##! This implements all of the additional information and geodata detections 
+##! for SSH analysis.
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## If an SSH login is seen to or from a "watched" country based on the
+		## :bro:id:`SSH::watched_countries` variable then this notice will
+		## be generated.
+		Login_From_Watched_Country,
+	};
+	
+	## The set of countries for which you'd like to throw notices upon 
+	## successful login
+	const watched_countries: set[string] = {"RO"} &redef;
+
+	redef record Info += {
+		## Add geographic data related to the "remote" host of the connection.
+		remote_location: geo_location &log &optional;
+	};
+}
+
+event SSH::heuristic_successful_login(c: connection) &priority=5
+	{
+	local location: geo_location;
+	location = (c$ssh$direction == OUTBOUND) ? 
+		lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h);
+	
+	# Add the location data to the SSH record.
+	c$ssh$remote_location = location;
+	
+	if ( location$country_code in watched_countries )
+		{
+		NOTICE([$note=Login_From_Watched_Country,
+		        $conn=c,
+		        $msg=fmt("SSH login from watched country: %s", location$country_code)]);
+		}
+	}

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -0,0 +1,50 @@
+
+module SSH;
+
+export {
+	redef enum Notice::Type += {
+		## Generated if a login originates from a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_From_Interesting_Hostname,
+		## Generated if a login goes to a host matched by the 
+		## :bro:id:`interesting_hostnames` regular expression.
+		Login_To_Interesting_Hostname,
+	};
+	
+	## Strange/bad host names to see successful SSH logins from or to.
+	const interesting_hostnames =
+			/^d?ns[0-9]*\./ |
+			/^smtp[0-9]*\./ |
+			/^mail[0-9]*\./ |
+			/^pop[0-9]*\./  |
+			/^imap[0-9]*\./ |
+			/^www[0-9]*\./  |
+			/^ftp[0-9]*\./  &redef;
+}
+
+event SSH::heuristic_successful_login(c: connection)
+	{
+	# Check to see if this login came from an interesting hostname.
+	when ( local orig_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in orig_hostname )
+			{
+			NOTICE([$note=Login_From_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login from hostname: %s", orig_hostname),
+			        $sub=orig_hostname]);
+			}
+		}
+	# Check to see if this login went to an interesting hostname.
+	when ( local resp_hostname = lookup_addr(c$id$orig_h) )
+		{
+		if ( interesting_hostnames in resp_hostname )
+			{
+			NOTICE([$note=Login_To_Interesting_Hostname,
+			        $conn=c,
+			        $msg=fmt("Interesting login to hostname: %s", resp_hostname),
+			        $sub=resp_hostname]);
+			}
+		}
+	}
+

scripts/policy/protocols/ssh/software.bro

@@ -3,8 +3,8 @@ module SSH;
 
 export {
 	redef enum Software::Type += {
-		SSH_SERVER,
+		SERVER,
-		SSH_CLIENT,
+		CLIENT,
 	};
 }
 
@@ -12,7 +12,7 @@ event ssh_client_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /^SSH[0-9\.\-]+/, "");
-	local si = Software::parse(cleaned_version, c$id$orig_h, SSH_CLIENT);
+	local si = Software::parse(cleaned_version, c$id$orig_h, CLIENT);
 	Software::found(c$id, si);
 	}
 
@@ -20,6 +20,6 @@ event ssh_server_version(c: connection, version: string) &priority=4
 	{
 	# Get rid of the protocol information when passing to the software framework.
 	local cleaned_version = sub(version, /SSH[0-9\.\-]{2,}/, "");
-	local si = Software::parse(cleaned_version, c$id$resp_h, SSH_SERVER);
+	local si = Software::parse(cleaned_version, c$id$resp_h, SERVER);
 	Software::found(c$id, si);
 	}

scripts/site/local-manager.bro

@@ -0,0 +1,7 @@
+##! Local site policy loaded only by the manager in a cluster.
+
+# If you are running a cluster you should define your Notice::policy here 
+# so that notice processing occurs on the manager.
+redef Notice::policy += {
+
+};

scripts/site/local-proxy.bro

@@ -0,0 +1,2 @@
+##! Local site policy loaded only by the proxies if Bro is running as a cluster.
+

scripts/site/local-worker.bro

@@ -0,0 +1 @@
+##! Local site policy loaded only by the workers if Bro is running as a cluster.

scripts/site/local.bro

@@ -22,6 +22,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
 # Load all of the scripts that detect software in various protocols.
 @load protocols/http/software
+#@load protocols/http/detect-webapps
 @load protocols/ftp/software
 @load protocols/smtp/software
 @load protocols/ssh/software
@@ -44,3 +45,24 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
 # Load the script to enable SSL/TLS certificate validation.
 @load protocols/ssl/validate-certs
+
+# If you have libGeoIP support built in, do some geographic detections and 
+# logging for SSH traffic.
+@load protocols/ssh/geo-data
+# Detect hosts doing SSH bruteforce attacks.
+@load protocols/ssh/detect-bruteforcing
+# Detect logins using "interesting" hostnames.
+@load protocols/ssh/interesting-hostnames
+
+# Detect MD5 sums in Team Cymru's Malware Hash Registry.
+@load protocols/http/detect-MHR
+# Detect SQL injection attacks
+@load protocols/http/detect-sqli
+
+# Uncomment this redef if you want to extract SMTP MIME entities for 
+# some file types.  The numbers given indicate how many bytes to extract for
+# the various mime types.
+redef SMTP::entity_excerpt_len += {
+#	["text/plain"] = 1024,
+#	["text/html"] = 1024,
+};

scripts/test-all-policy.bro

@@ -2,18 +2,17 @@
 # 
 # This is rarely makes sense, and is for testing only.
 # 
-# Note that we have unit test that makes sure that all policy files shipped are
+# Note that we have a unit test that makes sure that all policy files shipped are
 # actually loaded here. If we have files that are part of the distribution yet
 # can't be loaded here,  these must still be listed here with their load command
 # commented out.
 
 # The base/ scripts are all loaded by default and not included here.
 
-# @load test-all.bro
 # @load frameworks/communication/listen-clear.bro
 # @load frameworks/communication/listen-ssl.bro
-# @load frameworks/control/controllee
+# @load frameworks/control/controllee.bro
-# @load frameworks/control/controller
+# @load frameworks/control/controller.bro
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
 @load frameworks/software/version-changes.bro
@@ -27,6 +26,7 @@
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro
+# @load protocols/conn/scan.bro
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
 @load protocols/ftp/detect.bro
@@ -50,4 +50,4 @@
 @load tuning/defaults/remove-high-volume-notices.bro
 @load tuning/defaults/warnings.bro
 @load tuning/track-all-assets.bro
-@load site/local
+# @load hot.conn.bro

src/Expr.cc

@@ -231,7 +231,6 @@ bool Expr::DoUnserialize(UnserialInfo* info)
 NameExpr::NameExpr(ID* arg_id) : Expr(EXPR_NAME)
 	{
 	id = arg_id;
-	ReferenceID();
 	SetType(id->Type()->Ref());
 
 	EventHandler* h = event_registry->Lookup(id->Name());
@@ -244,29 +243,6 @@ NameExpr::~NameExpr()
 	Unref(id);
 	}
 
-void NameExpr::ReferenceID()
-	{
-	// ### This is a hack. We check whether one of the remote serializer's
-	// built-in functions is referenced. If so, we activate the serializer.
-	// A better solution would be to either (1) a generic mechanism in
-	// which have (internal) attributes associated with identifiers and
-	// as we see references to the identifiers, we do bookkeeping
-	// associated with their attribute (so in this case the attribute
-	// would be "flag that inter-Bro communication is being used"),
-	// or (2) after the parse is done, we'd query whether these
-	// particular identifiers were seen, rather than doing the test
-	// here for every NameExpr we create.
-	if ( id->Type()->Tag() == TYPE_FUNC )
-		{
-		const char* const* builtins = remote_serializer->GetBuiltins();
-		while( *builtins )
-			{
-			if ( streq(id->Name(), *builtins++) )
-				using_communication = true;
-			}
-		}
-	}
-
 Expr* NameExpr::Simplify(SimplifyType simp_type)
 	{
 	if ( simp_type != SIMPLIFY_LHS && id->IsConst() )
@@ -393,8 +369,6 @@ bool NameExpr::DoUnserialize(UnserialInfo* info)
 	if ( ! id )
 		return false;
 
-	ReferenceID();
-
 	return true;
 	}
 

src/Expr.h

@@ -217,7 +217,6 @@ protected:
 	friend class Expr;
 	NameExpr()	{ id = 0; }
 
-	void ReferenceID();
 	void ExprDescribe(ODesc* d) const;
 
 	DECLARE_SERIAL(NameExpr);

src/LogMgr.cc

@@ -433,6 +433,25 @@ LogMgr::Stream* LogMgr::FindStream(EnumVal* id)
 	return streams[idx];
 	}
 
+LogMgr::WriterInfo* LogMgr::FindWriter(LogWriter* writer)
+	{
+	for ( vector<Stream *>::iterator s = streams.begin(); s != streams.end(); ++s )
+		{
+		if ( ! *s )
+			continue;
+
+		for ( Stream::WriterMap::iterator i = (*s)->writers.begin(); i != (*s)->writers.end(); i++ )
+			{
+			WriterInfo* winfo = i->second;
+
+			if ( winfo->writer == writer )
+				return winfo;
+			}
+		}
+
+	return 0;
+	}
+
 void LogMgr::RemoveDisabledWriters(Stream* stream)
 	{
 	list<Stream::WriterPathPair> disabled;
@@ -1411,6 +1430,8 @@ void LogMgr::InstallRotationTimer(WriterInfo* winfo)
 	RecordVal* rc =
 		LookupRotationControl(winfo->type, winfo->writer->Path());
 
+	assert(rc);
+
 	int idx = rc->Type()->AsRecordType()->FieldOffset("interv");
 	double rotation_interval = rc->LookupWithDefault(idx)->AsInterval();
 
@@ -1448,34 +1469,63 @@ void LogMgr::Rotate(WriterInfo* winfo)
 	DBG_LOG(DBG_LOGGING, "Rotating %s at %.6f",
 		winfo->writer->Path().c_str(), network_time);
 
-	// Create the RotationInfo record.
+	// Build a temporary path for the writer to move the file to.
-	RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
+	struct tm tm;
-	info->Assign(0, winfo->type->Ref());
+	char buf[128];
-	info->Assign(1, new StringVal(winfo->writer->Path().c_str()));
+	const char* const date_fmt = "%y-%m-%d_%H.%M.%S";
-	info->Assign(2, new Val(winfo->open_time, TYPE_TIME));
+	time_t teatime = (time_t)winfo->open_time;
-	info->Assign(3, new Val(network_time, TYPE_TIME));
 
-	// Call the function building us the new path.
+	localtime_r(&teatime, &tm);
+	strftime(buf, sizeof(buf), date_fmt, &tm);
 
-	Func* rotation_path_func =
+	string tmp = string(fmt("%s-%s", winfo->writer->Path().c_str(), buf));
-		internal_func("Log::default_rotation_path_func");
+
+	// Trigger the rotation.
+	winfo->writer->Rotate(tmp, winfo->open_time, network_time, terminating);
+	}
+
+bool LogMgr::FinishedRotation(LogWriter* writer, string new_name, string old_name,
+		      double open, double close, bool terminating)
+	{
+	DBG_LOG(DBG_LOGGING, "Finished rotating %s at %.6f, new name %s",
+		writer->Path().c_str(), network_time, new_name.c_str());
+
+	WriterInfo* winfo = FindWriter(writer);
+	assert(winfo);
 
 	RecordVal* rc =
 		LookupRotationControl(winfo->type, winfo->writer->Path());
 
+	assert(rc);
+
+	// Create the RotationInfo record.
+	RecordVal* info = new RecordVal(BifType::Record::Log::RotationInfo);
+	info->Assign(0, winfo->type->Ref());
+	info->Assign(1, new StringVal(new_name.c_str()));
+	info->Assign(2, new StringVal(winfo->writer->Path().c_str()));
+	info->Assign(3, new Val(open, TYPE_TIME));
+	info->Assign(4, new Val(close, TYPE_TIME));
+	info->Assign(5, new Val(terminating, TYPE_BOOL));
+
 	int idx = rc->Type()->AsRecordType()->FieldOffset("postprocessor");
+	assert(idx >= 0);
 
-	string rotation_postprocessor =
+	Val* func = rc->Lookup(idx);
-		rc->LookupWithDefault(idx)->AsString()->CheckString();
+	if ( ! func )
-
+		{
-	val_list vl(1);
+		ID* id = global_scope()->Lookup("Log::__default_rotation_postprocessor");
-	vl.append(info);
+		assert(id);
-	Val* result = rotation_path_func->Call(&vl);
+		func = id->ID_Val();
-	string new_path = result->AsString()->CheckString();
-	Unref(result);
-
-	winfo->writer->Rotate(new_path, rotation_postprocessor,
-			      winfo->open_time, network_time, terminating);
 		}
 
+	assert(func);
+
+	// Call the postprocessor function.
+	val_list vl(1);
+	vl.append(info);
+	Val* v = func->AsFunc()->Call(&vl);
+	int result = v->AsBool();
+	Unref(v);
+	return result;
+	}
 

src/LogMgr.h

@@ -103,6 +103,10 @@ protected:
 
 	//// Functions safe to use by writers.
 
+	// Signals that a file has been rotated.
+	bool FinishedRotation(LogWriter* writer, string new_name, string old_name,
+			      double open, double close, bool terminating);
+
 	// Reports an error for the given writer.
 	void Error(LogWriter* writer, const char* msg);
 
@@ -127,6 +131,7 @@ private:
 	void Rotate(WriterInfo* info);
 	RecordVal* LookupRotationControl(EnumVal* writer, string path);
 	Filter* FindFilter(EnumVal* id, StringVal* filter);
+	WriterInfo* FindWriter(LogWriter* writer);
 
 	vector<Stream *> streams;	// Indexed by stream enum.
 };

src/LogWriter.cc

@@ -89,10 +89,10 @@ bool LogWriter::SetBuf(bool enabled)
 	return true;
 	}
 
-bool LogWriter::Rotate(string rotated_path, string postprocessor, double open,
+bool LogWriter::Rotate(string rotated_path, double open,
 		       double close, bool terminating)
 	{
-	if ( ! DoRotate(rotated_path, postprocessor, open, close, terminating) )
+	if ( ! DoRotate(rotated_path, open, close, terminating) )
 		{
 		disabled = true;
 		return false;
@@ -151,42 +151,8 @@ void LogWriter::DeleteVals(LogVal** vals)
 	log_mgr->DeleteVals(num_fields, vals);
 	}
 
-bool LogWriter::RunPostProcessor(string fname, string postprocessor,
+bool LogWriter::FinishedRotation(string new_name, string old_name, double open,
-				 string old_name, double open, double close,
+				 double close, bool terminating)
-				 bool terminating)
 	{
-	// This function operates in a way that is backwards-compatible with
+	return log_mgr->FinishedRotation(this, new_name, old_name, open, close, terminating);
-	// the old Bro log rotation scheme.
-
-	if ( ! postprocessor.size() )
-		return true;
-
-	const char* const fmt = "%y-%m-%d_%H.%M.%S";
-
-	struct tm tm1;
-	struct tm tm2;
-
-	time_t tt1 = (time_t)open;
-	time_t tt2 = (time_t)close;
-
-	localtime_r(&tt1, &tm1);
-	localtime_r(&tt2, &tm2);
-
-	char buf1[128];
-	char buf2[128];
-
-	strftime(buf1, sizeof(buf1), fmt, &tm1);
-	strftime(buf2, sizeof(buf2), fmt, &tm2);
-
-	string cmd = postprocessor;
-	cmd += " " + fname;
-	cmd += " " + old_name;
-	cmd += " " + string(buf1);
-	cmd += " " + string(buf2);
-	cmd += " " + string(terminating ? "1" : "0");
-	cmd += " &";
-
-	system(cmd.c_str());
-
-	return true;
 	}

src/LogWriter.h

@@ -60,8 +60,7 @@ public:
 
 	// Triggers rotation, if the writer supports that. (If not, it will
 	// be ignored).
-	bool Rotate(string rotated_path, string postprocessor, double open,
+	bool Rotate(string rotated_path, double open, double close, bool terminating);
-		    double close, bool terminating);
 
 	// Finishes writing to this logger regularly. Must not be called if
 	// an error has been indicated earlier. After calling this, no
@@ -77,7 +76,6 @@ public:
 	const LogField* const * Fields() const	{ return fields; }
 
 protected:
-
 	// Methods for writers to override. If any of these returs false, it
 	// will be assumed that a fatal error has occured that prevents the
 	// writer from further operation. It will then be disabled and
@@ -117,18 +115,17 @@ protected:
 	// current file and open a new one.  However, a writer may also
 	// trigger other apppropiate actions if semantics are similar.
 	// 
+	// Once rotation has finished, the implementation should call
+	// RotationDone() to signal the log manager that potential
+	// postprocessors can now run.
+	//
 	// "rotate_path" reflects the path to where the rotated output is to
 	// be moved, with specifics depending on the writer. It should
 	// generally be interpreted in a way consistent with that of "path"
 	// as passed into DoInit(). As an example, for file-based output, 
 	// "rotate_path" could be the original filename extended with a
 	// timestamp indicating the time of the rotation.
-
+	// 
-	// "postprocessor" is the name of a command to execute on the rotated
-	// file. If empty, no postprocessing should take place; if given but
-	// the writer doesn't support postprocessing, it can be ignored (but
-	// the method must still return true in that case).
-
 	// "open" and "close" are the network time's when the *current* file
 	// was opened and closed, respectively.
 	//
@@ -138,8 +135,8 @@ protected:
 	//
 	// A writer may ignore rotation requests if it doesn't fit with its
 	// semantics (but must still return true in that case).
-	virtual bool DoRotate(string rotated_path, string postprocessor,
+	virtual bool DoRotate(string rotated_path, double open, double close,
-			      double open, double close, bool terminating) = 0;
+			      bool terminating) = 0;
 
 	// Called once on termination. Not called when any of the other
 	// methods has previously signaled an error, i.e., executing this
@@ -157,11 +154,18 @@ protected:
 	// Reports an error to the user.
 	void Error(const char *msg);
 
-	// Runs a post-processor on the given file. Parameters correspond to
+	// Signals to the log manager that a file has been rotated.
-	// those of DoRotate(). 
+	//
-	bool RunPostProcessor(string fname, string postprocessor,
+	// new_name: The filename of the rotated file. old_name: The filename
-			      string old_name, double open, double close,
+	// of the origina file.
-			      bool terminating);
+	//
+	// open/close: The timestamps when the original file was opened and
+	// closed, respectively.
+	//
+	// terminating: True if rotation request occured due to the main Bro
+	// process shutting down.
+	bool FinishedRotation(string new_name, string old_name, double open,
+			      double close, bool terminating);
 
 private:
 	friend class LogMgr;

src/LogWriterAscii.cc

@@ -242,7 +242,7 @@ bool LogWriterAscii::DoWrite(int num_fields, const LogField* const * fields,
 	return true;
 	}
 
-bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double open,
+bool LogWriterAscii::DoRotate(string rotated_path, double open,
 			      double close, bool terminating)
 	{
 	if ( IsSpecial(Path()) )
@@ -254,10 +254,8 @@ bool LogWriterAscii::DoRotate(string rotated_path, string postprocessor, double
 	string nname = rotated_path + ".log";
 	rename(fname.c_str(), nname.c_str());
 
-	if ( postprocessor.size() &&
+	if ( ! FinishedRotation(nname, fname, open, close, terminating) )
-	     ! RunPostProcessor(nname, postprocessor, fname.c_str(),
+		Error(Fmt("error rotating %s to %s", fname.c_str(), nname.c_str()));
-				open, close, terminating) )
-		return false;
 
 	return DoInit(Path(), NumFields(), Fields());
 	}

src/LogWriterAscii.h

@@ -20,8 +20,8 @@ protected:
 	virtual bool DoWrite(int num_fields, const LogField* const * fields,
 			     LogVal** vals);
 	virtual bool DoSetBuf(bool enabled);
-	virtual bool DoRotate(string rotated_path, string postprocessr,
+	virtual bool DoRotate(string rotated_path, double open, double close,
-			      double open, double close, bool terminating);
+			      bool terminating);
 	virtual bool DoFlush();
 	virtual void DoFinish();
 

src/RSH.cc

@@ -15,7 +15,7 @@ Contents_Rsh_Analyzer::Contents_Rsh_Analyzer(Connection* conn, bool orig,
 						Rsh_Analyzer* arg_analyzer)
 : ContentLine_Analyzer(AnalyzerTag::Contents_Rsh, conn, orig)
 	{
-	num_bytes_to_scan = num_bytes_to_scan = 0;
+	num_bytes_to_scan = 0;
 	analyzer = arg_analyzer;
 
 	if ( orig )

src/RemoteSerializer.cc

@@ -3060,13 +3060,6 @@ bool RemoteSerializer::IsActive()
 	return false;
 	}
 
-
-const char* const* RemoteSerializer::GetBuiltins() const
-	{
-	static const char* builtins[] = { "connect", "listen", 0 };
-	return builtins;
-	}
-
 void RemoteSerializer::ReportError(const char* msg)
 	{
 	if ( current_peer && current_peer->phase != Peer::SETUP )

src/RemoteSerializer.h

@@ -128,10 +128,6 @@ public:
 	// Log some statistics.
 	void LogStats();
 
-	// Return a 0-terminated array of built-in functions which,
-	// when referenced, trigger the remote serializer's initialization.
-	const char* const* GetBuiltins() const;
-
 	// Tries to sent out all remaining data.
 	// FIXME: Do we still need this?
 	void Finish();

src/bro.bif

@@ -3636,3 +3636,23 @@ function piped_exec%(program: string, to_write: string%): bool
 
 	return new Val(true, TYPE_BOOL);
 	%}
+
+## Enables the communication system. Note that by default,
+## communication is off until explicitly enabled, and all other calls
+## to communication-related BiFs' will be ignored until done so.
+function enable_communication%(%): any
+	%{
+	if ( bro_start_network_time != 0.0 )
+		{
+		builtin_error("communication must be enabled in bro_init");
+		return 0;
+		}
+
+	if ( using_communication )
+		// Ignore duplicate calls.
+		return 0;
+
+	using_communication = 1;
+	remote_serializer->Init();
+	return 0;
+	%}

src/main.cc

@@ -880,9 +880,6 @@ int main(int argc, char** argv)
 		exit(0);
 		}
 
-	if ( using_communication )
-		remote_serializer->Init();
-
 	persistence_serializer->SetDir((const char *)state_dir->AsString()->CheckString());
 
 	// Print the ID.

testing/btest/Baseline/policy.frameworks.logging.rotate-custom/out

@@ -1,33 +1,33 @@
-1st test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0
+1st test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0
-1st test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_05.00.05 0
+1st test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0
-1st test-11-03-07_05.00.05.log test.log 11-03-07_05.00.05 11-03-07_06.00.05 0
+1st test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0
-1st test-11-03-07_06.00.05.log test.log 11-03-07_06.00.05 11-03-07_07.00.05 0
+1st test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0
-1st test-11-03-07_07.00.05.log test.log 11-03-07_07.00.05 11-03-07_08.00.05 0
+1st test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0
-1st test-11-03-07_08.00.05.log test.log 11-03-07_08.00.05 11-03-07_09.00.05 0
+1st test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0
-1st test-11-03-07_09.00.05.log test.log 11-03-07_09.00.05 11-03-07_10.00.05 0
+1st test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0
-1st test-11-03-07_10.00.05.log test.log 11-03-07_10.00.05 11-03-07_11.00.05 0
+1st test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0
-1st test-11-03-07_11.00.05.log test.log 11-03-07_11.00.05 11-03-07_12.00.05 0
+1st test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
-1st test-11-03-07_12.00.05.log test.log 11-03-07_12.00.05 11-03-07_12.59.55 1
+1st test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
-2nd test2-11-03-07_03.00.05.log test2.log 11-03-07_03.00.05 11-03-07_03.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_03.00.05.log, path=test2, open=1299466805.0, close=1299470395.0, terminating=F]
-2nd test2-11-03-07_03.59.55.log test2.log 11-03-07_03.59.55 11-03-07_04.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_03.59.55.log, path=test2, open=1299470395.0, close=1299470405.0, terminating=F]
-2nd test2-11-03-07_04.00.05.log test2.log 11-03-07_04.00.05 11-03-07_04.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_04.00.05.log, path=test2, open=1299470405.0, close=1299473995.0, terminating=F]
-2nd test2-11-03-07_04.59.55.log test2.log 11-03-07_04.59.55 11-03-07_05.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_04.59.55.log, path=test2, open=1299473995.0, close=1299474005.0, terminating=F]
-2nd test2-11-03-07_05.00.05.log test2.log 11-03-07_05.00.05 11-03-07_05.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_05.00.05.log, path=test2, open=1299474005.0, close=1299477595.0, terminating=F]
-2nd test2-11-03-07_05.59.55.log test2.log 11-03-07_05.59.55 11-03-07_06.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_05.59.55.log, path=test2, open=1299477595.0, close=1299477605.0, terminating=F]
-2nd test2-11-03-07_06.00.05.log test2.log 11-03-07_06.00.05 11-03-07_06.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_06.00.05.log, path=test2, open=1299477605.0, close=1299481195.0, terminating=F]
-2nd test2-11-03-07_06.59.55.log test2.log 11-03-07_06.59.55 11-03-07_07.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_06.59.55.log, path=test2, open=1299481195.0, close=1299481205.0, terminating=F]
-2nd test2-11-03-07_07.00.05.log test2.log 11-03-07_07.00.05 11-03-07_07.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_07.00.05.log, path=test2, open=1299481205.0, close=1299484795.0, terminating=F]
-2nd test2-11-03-07_07.59.55.log test2.log 11-03-07_07.59.55 11-03-07_08.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_07.59.55.log, path=test2, open=1299484795.0, close=1299484805.0, terminating=F]
-2nd test2-11-03-07_08.00.05.log test2.log 11-03-07_08.00.05 11-03-07_08.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_08.00.05.log, path=test2, open=1299484805.0, close=1299488395.0, terminating=F]
-2nd test2-11-03-07_08.59.55.log test2.log 11-03-07_08.59.55 11-03-07_09.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_08.59.55.log, path=test2, open=1299488395.0, close=1299488405.0, terminating=F]
-2nd test2-11-03-07_09.00.05.log test2.log 11-03-07_09.00.05 11-03-07_09.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_09.00.05.log, path=test2, open=1299488405.0, close=1299491995.0, terminating=F]
-2nd test2-11-03-07_09.59.55.log test2.log 11-03-07_09.59.55 11-03-07_10.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_09.59.55.log, path=test2, open=1299491995.0, close=1299492005.0, terminating=F]
-2nd test2-11-03-07_10.00.05.log test2.log 11-03-07_10.00.05 11-03-07_10.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_10.00.05.log, path=test2, open=1299492005.0, close=1299495595.0, terminating=F]
-2nd test2-11-03-07_10.59.55.log test2.log 11-03-07_10.59.55 11-03-07_11.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_10.59.55.log, path=test2, open=1299495595.0, close=1299495605.0, terminating=F]
-2nd test2-11-03-07_11.00.05.log test2.log 11-03-07_11.00.05 11-03-07_11.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_11.00.05.log, path=test2, open=1299495605.0, close=1299499195.0, terminating=F]
-2nd test2-11-03-07_11.59.55.log test2.log 11-03-07_11.59.55 11-03-07_12.00.05 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_11.59.55.log, path=test2, open=1299499195.0, close=1299499205.0, terminating=F]
-2nd test2-11-03-07_12.00.05.log test2.log 11-03-07_12.00.05 11-03-07_12.59.55 0
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_12.00.05.log, path=test2, open=1299499205.0, close=1299502795.0, terminating=F]
-2nd test2-11-03-07_12.59.55.log test2.log 11-03-07_12.59.55 11-03-07_12.59.55 1
+custom rotate, [writer=WRITER_ASCII, fname=test2-11-03-07_12.59.55.log, path=test2, open=1299502795.0, close=1299502795.0, terminating=T]
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
 1299470395.000000	10.0.0.2	20	10.0.0.3	0
@@ -49,16 +49,16 @@
 1299499195.000000	10.0.0.2	20	10.0.0.3	8
 1299499205.000000	10.0.0.1	20	10.0.0.2	1033
 1299502795.000000	10.0.0.2	20	10.0.0.3	9
-> test-11-03-07_03.00.05.log
+> test.2011-03-07-03-00-05.log
-> test-11-03-07_04.00.05.log
+> test.2011-03-07-04-00-05.log
-> test-11-03-07_05.00.05.log
+> test.2011-03-07-05-00-05.log
-> test-11-03-07_06.00.05.log
+> test.2011-03-07-06-00-05.log
-> test-11-03-07_07.00.05.log
+> test.2011-03-07-07-00-05.log
-> test-11-03-07_08.00.05.log
+> test.2011-03-07-08-00-05.log
-> test-11-03-07_09.00.05.log
+> test.2011-03-07-09-00-05.log
-> test-11-03-07_10.00.05.log
+> test.2011-03-07-10-00-05.log
-> test-11-03-07_11.00.05.log
+> test.2011-03-07-11-00-05.log
-> test-11-03-07_12.00.05.log
+> test.2011-03-07-12-00-05.log
 > test.log
 > test2-11-03-07_03.00.05.log
 > test2-11-03-07_03.59.55.log

testing/btest/Baseline/policy.frameworks.logging.rotate/out

@@ -1,50 +1,50 @@
-test-11-03-07_03.00.05.log test.log 11-03-07_03.00.05 11-03-07_04.00.05 0
+test.2011-03-07-03-00-05.log test 11-03-07_03.00.05 11-03-07_04.00.05 0
-test-11-03-07_04.00.05.log test.log 11-03-07_04.00.05 11-03-07_05.00.05 0
+test.2011-03-07-04-00-05.log test 11-03-07_04.00.05 11-03-07_05.00.05 0
-test-11-03-07_05.00.05.log test.log 11-03-07_05.00.05 11-03-07_06.00.05 0
+test.2011-03-07-05-00-05.log test 11-03-07_05.00.05 11-03-07_06.00.05 0
-test-11-03-07_06.00.05.log test.log 11-03-07_06.00.05 11-03-07_07.00.05 0
+test.2011-03-07-06-00-05.log test 11-03-07_06.00.05 11-03-07_07.00.05 0
-test-11-03-07_07.00.05.log test.log 11-03-07_07.00.05 11-03-07_08.00.05 0
+test.2011-03-07-07-00-05.log test 11-03-07_07.00.05 11-03-07_08.00.05 0
-test-11-03-07_08.00.05.log test.log 11-03-07_08.00.05 11-03-07_09.00.05 0
+test.2011-03-07-08-00-05.log test 11-03-07_08.00.05 11-03-07_09.00.05 0
-test-11-03-07_09.00.05.log test.log 11-03-07_09.00.05 11-03-07_10.00.05 0
+test.2011-03-07-09-00-05.log test 11-03-07_09.00.05 11-03-07_10.00.05 0
-test-11-03-07_10.00.05.log test.log 11-03-07_10.00.05 11-03-07_11.00.05 0
+test.2011-03-07-10-00-05.log test 11-03-07_10.00.05 11-03-07_11.00.05 0
-test-11-03-07_11.00.05.log test.log 11-03-07_11.00.05 11-03-07_12.00.05 0
+test.2011-03-07-11-00-05.log test 11-03-07_11.00.05 11-03-07_12.00.05 0
-test-11-03-07_12.00.05.log test.log 11-03-07_12.00.05 11-03-07_12.59.55 1
+test.2011-03-07-12-00-05.log test 11-03-07_12.00.05 11-03-07_12.59.55 1
-> test-11-03-07_03.00.05.log
+> test.2011-03-07-03-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299466805.000000	10.0.0.1	20	10.0.0.2	1024
 1299470395.000000	10.0.0.2	20	10.0.0.3	0
-> test-11-03-07_04.00.05.log
+> test.2011-03-07-04-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299470405.000000	10.0.0.1	20	10.0.0.2	1025
 1299473995.000000	10.0.0.2	20	10.0.0.3	1
-> test-11-03-07_05.00.05.log
+> test.2011-03-07-05-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299474005.000000	10.0.0.1	20	10.0.0.2	1026
 1299477595.000000	10.0.0.2	20	10.0.0.3	2
-> test-11-03-07_06.00.05.log
+> test.2011-03-07-06-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299477605.000000	10.0.0.1	20	10.0.0.2	1027
 1299481195.000000	10.0.0.2	20	10.0.0.3	3
-> test-11-03-07_07.00.05.log
+> test.2011-03-07-07-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299481205.000000	10.0.0.1	20	10.0.0.2	1028
 1299484795.000000	10.0.0.2	20	10.0.0.3	4
-> test-11-03-07_08.00.05.log
+> test.2011-03-07-08-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299484805.000000	10.0.0.1	20	10.0.0.2	1029
 1299488395.000000	10.0.0.2	20	10.0.0.3	5
-> test-11-03-07_09.00.05.log
+> test.2011-03-07-09-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299488405.000000	10.0.0.1	20	10.0.0.2	1030
 1299491995.000000	10.0.0.2	20	10.0.0.3	6
-> test-11-03-07_10.00.05.log
+> test.2011-03-07-10-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299492005.000000	10.0.0.1	20	10.0.0.2	1031
 1299495595.000000	10.0.0.2	20	10.0.0.3	7
-> test-11-03-07_11.00.05.log
+> test.2011-03-07-11-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299495605.000000	10.0.0.1	20	10.0.0.2	1032
 1299499195.000000	10.0.0.2	20	10.0.0.3	8
-> test-11-03-07_12.00.05.log
+> test.2011-03-07-12-00-05.log
 # t	id.orig_h	id.orig_p	id.resp_h	id.resp_p
 1299499205.000000	10.0.0.1	20	10.0.0.2	1033
 1299502795.000000	10.0.0.2	20	10.0.0.3	9

testing/btest/Baseline/policy.misc.bare-loaded-scripts/canonified_loaded_scripts.log

@@ -0,0 +1,13 @@
+# depth	name
+0	scripts/base/init-bare.bro
+1	build/src/const.bif.bro
+1	build/src/types.bif.bro
+1	build/src/strings.bif.bro
+1	build/src/bro.bif.bro
+1	build/src/reporter.bif.bro
+1	build/src/event.bif.bro
+1	scripts/base/frameworks/logging/__load__.bro
+2	scripts/base/frameworks/logging/./main.bro
+3	build/src/logging.bif.bro
+2	scripts/base/frameworks/logging/./writers/ascii.bro
+0	scripts/policy/misc/loaded-scripts.bro

testing/btest/Baseline/policy.protocols.smtp.basic/smtp.log

@@ -0,0 +1,2 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	helo	mailfrom	rcptto	date	from	to	reply_to	msg_id	in_reply_to	subject	x_originating_ip	first_received	second_received	last_reply	path	user_agent
+1254722768.219663	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	GP	<gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	Mon, 5 Oct 2009 11:36:07 +0530	"Gurpartap Singh" <gurpartap@patriots.in>	<raj_deol2002in@yahoo.co.in>	-	<000301ca4581$ef9e57f0$cedb07d0$@in>	-	SMTP	-	-	-	250 OK id=1Mugho-0003Dg-Un	74.53.140.153,10.10.1.4	Microsoft Office Outlook 12.0

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat

@@ -0,0 +1,13 @@
+Hello
+
+ 
+
+I send u smtp pcap file 
+
+Find the attachment
+
+ 
+
+GPS
+
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat

@@ -0,0 +1,264 @@
+Version 4.9.9.1
+* Many bug fixes
+* Improved editor
+
+Version 4.9.9.0
+* Support for latest Mingw compiler system builds
+* Bug fixes
+
+Version 4.9.8.9
+* New code tooltip display
+* Improved Indent/Unindent and Remove Comment
+* Improved automatic indent
+* Added support for the "interface" keyword
+* WebUpdate should now report installation problems from PackMan
+* New splash screen and association icons
+* Improved installer
+* Many bug fixes
+
+Version 4.9.8.7
+* Added support for GCC > 3.2
+* Debug variables are now resent during next debug session
+* Watched Variables not in correct context are now kept and updated when it is needed
+* Added new compiler/linker options: 20
+  - Strip executable
+  - Generate instructions for a specific machine (i386, i486, i586, i686, pentium, pentium-mmx, pentiumpro, pentium2, pentium3, pentium4, 20
+    k6, k6-2, k6-3, athlon, athlon-tbird, athlon-4, athlon-xp, athlon-mp, winchip-c6, winchip2, k8, c3 and c3-2)
+  - Enable use of processor specific built-in functions (mmmx, sse, sse2, pni, 3dnow)
+* "Default" button in Compiler Options is back
+* Error messages parsing improved
+* Bug fixes
+
+Version 4.9.8.5
+* Added the possibility to modify the value of a variable during debugging (right click on a watch variable and select "Modify value")
+* During Dev-C++ First Time COnfiguration window, users can now choose between using or not class browser and code completion features.
+* Many bug fixes
+
+Version 4.9.8.4
+* Added the possibility to specify an include directory for the code completion cache to be created at Dev-C++ first startup
+* Improved code completion cache
+* WebUpdate will now backup downloaded DevPaks in Dev-C++\Packages directory, and Dev-C++ executable in devcpp.exe.BACKUP
+* Big speed up in function parameters listing while editing
+* Bug fixes
+
+Version 4.9.8.3
+* On Dev-C++ first time configuration dialog, a code completion cache of all the standard 20
+  include files can now be generated.
+* Improved WebUpdate module
+* Many bug fixes
+
+Version 4.9.8.2
+* New debug feature for DLLs: attach to a running process
+* New project option: Use custom Makefile. 20
+* New WebUpdater module.
+* Allow user to specify an alternate configuration file in Environment Options 20
+  (still can be overriden by using "-c" command line parameter).
+* Lots of bug fixes.
+
+Version 4.9.8.1
+* When creating a DLL, the created static lib respects now the project-defined output directory
+
+Version 4.9.8.0
+* Changed position of compiler/linker parameters in Project Options.
+* Improved help file
+* Bug fixes
+
+Version 4.9.7.9
+* Resource errors are now reported in the Resource sheet
+* Many bug fixes
+
+Version 4.9.7.8
+* Made whole bottom report control floating instead of only debug output.
+* Many bug fixes
+
+Version 4.9.7.7
+* Printing settings are now saved
+* New environment options : "watch variable under mouse" and "Report watch errors"
+* Bug fixes
+
+Version 4.9.7.6
+* Debug variable browser
+* Added possibility to include in a Template the Project's directories (include, libs and ressources)
+* Changed tint of Class browser pictures colors to match the New Look style
+* Bug fixes
+
+Version 4.9.7.5
+* Bug fixes
+
+Version 4.9.7.4
+* When compiling with debugging symbols, an extra definition is passed to the
+  compiler: -D__DEBUG__
+* Each project creates a <project_name>_private.h file containing version
+  information definitions
+* When compiling the current file only, no dependency checks are performed
+* ~300% Speed-up in class parser
+* Added "External programs" in Tools/Environment Options (for units "Open with")
+* Added "Open with" in project units context menu
+* Added "Classes" toolbar
+* Fixed pre-compilation dependency checks to work correctly
+* Added new file menu entry: Save Project As
+* Bug-fix for double quotes in devcpp.cfg file read by vUpdate
+* Other bug fixes
+
+Version 4.9.7.3
+* When adding debugging symbols on request, remove "-s" option from linker
+* Compiling progress window
+* Environment options : "Show progress window" and "Auto-close progress window"
+* Bug fixes
+
+Version 4.9.7.2
+* Bug fixes
+
+Version 4.9.7.1
+* "Build priority" per-unit
+* "Include file in linking process" per-unit
+* New feature: compile current file only
+* Separated C++ compiler options from C compiler options in Makefile (see bug report #654744)
+* Separated C++ include dirs from C include dirs in Makefile (see bug report #654744)
+* Necessary UI changes in Project Options
+* Added display of project filename, project output and a summary of the project files in Project Options General tab.
+* Fixed the "compiler-dirs-with-spaces" bug that crept-in in 4.9.7.0
+* Multi-select files in project-view (when "double-click to open" is configured in Environment Settings)
+* Resource files are treated as ordinary files now
+* Updates in "Project Options/Files" code
+* MSVC import now creates the folders structure of the original VC project
+* Bug fixes
+
+Version 4.9.7.0
+* Allow customizing of per-unit compile command in projects
+* Added two new macros: <DATE> and <DATETIME>
+* Added support for macros in the "default source code" (Tools/Editor Options/Code)
+* Separated layout info from project file. It is now kept in a different file
+  (the same filename as the project's but with extension ".layout"). If you
+  have your project under CVS control, you ''ll know why this had to happen...
+* Compiler settings per-project
+* Compiler set per-project
+* Implemented new compiler settings framework
+* "Compile as C++" per-unit
+* "Include file in compilation process" per-unit
+* Project version info (creates the relevant VERSIONINFO struct in the private
+  resource)
+* Support XP Themes (creates the CommonControls 6.0 manifest file and includes
+  it in the private resource)
+* Added CVS "login" and "logout" commands
+* Project manager and debugging window (in Debug tab) can now be trasnformed into floating windows.
+* Added "Add Library" button in Project Options
+* Bug fixes
+
+Version 4.9.6.9
+* Implemented search in help files for the word at cursor (context sensitive help)
+* Implemented "compiler sets" infrastructure to switch between different compilers easily (e.g. gcc-2.95 and gcc-3.2)
+* Added "Files" tab in CVS form to allow selection of more than one file for
+  the requested CVS action
+  20
+Version 4.9.6.8
+* support for DLL application hosting, for debugging and executing DLLs under Dev-C++.
+* New class browser option: "Show inherited members"
+* Added support for the '::' member access operator in code-completion
+* Added *working* function arguments hint
+* Added bracket highlighting. When the caret is on a bracket, that bracket and
+  its counterpart are highlighted
+* Nested folders in project view
+
+Version 4.9.6.7
+* XP Theme support
+* Added CVS commands "Add" and "Remove"
+* Added configuration option for "Templates Directory" in "Environment Options"
+* Code-completion updates
+* Bug fixes
+
+Version 4.9.6.6
+* Editor colors are initialized properly on Dev-C++ first-run
+* Added doxygen-style comments in NewClass, NewMemberFunction and NewMemberVariable wizards
+* Added file's date/time stamp in File/Properties window
+* Current windows listing in Window menu
+* Bug fixes
+
+Version 4.9.6.5
+* CVS support
+* Window list (in Window menu)
+* bug fixes
+
+version 4.9.6.4
+* added ENTER key for opening file in project browser, DEL to delete from the project.
+* bug fixes
+
+version 4.9.6.3
+* Bug fixes
+
+version 4.9.6.2
+* Bug fixes
+
+version 4.9.6.1
+* New "Abort compilation" button
+* Bug fixes
+* Now checks for vRoach existance when sending a crash report
+
+Version 4.9.5.5
+* New option in Editor Options: Show editor hints. User can disable the hints
+  displayed in the editor when the mouse moves over a word. Since this was the
+  cause of many errors (although it should be fixed by now), we are giving the
+  user the option to disable this feature.
+* New option in Editor Options (code-completion): Use code-completion cache.
+  Well, it adds caching to code-completion. Depending on the cache size,
+  the program may take a bit longer to start-up, but provides very fast
+  code-completion and the user has all the commands (belonging to the files
+  he added in the cache) at his fingertips. If, for example, the user adds
+  "windows.h", he gets all the WinAPI! If he adds "wx/wx.h", he gets all of
+  wxWindows! You get the picture...
+* Removed "Only show classes from current file" option in class browser settings.
+  It used to be a checkbox, allowing only two states (on or off), but there is
+  a third relevant option now: "Project classes" so it didn't fit the purpose...
+  The user can define this in the class browser's context menu under "View mode".
+* Fixed the dreaded "Clock skew detected" compiler warning!
+* Fixed many class browser bugs, including some that had to do with class folders.
+
+Version 4.9.5.4
+* Under NT, 2000 and XP, user application data directory will be used to store config files (i.e : C:\Documents and Settings\Username\Local Settings\Application Data)
+
+Version 4.9.5.3
+* Added ExceptionsAnalyzer. If the devcpp.map file is in the devcpp.exe directory
+  then we even get a stack trace in the bug report!
+* Added new WebUpdate module (inactive temporarily).
+* Added new code for code-completion caching of files (disabled - work in progress).
+
+Version 4.9.5.2
+* Added new option in class-browser: Use colors
+  (available when right-clicking the class-browser
+  and selecting "View mode").
+* Dev-C++ now traps access violation of your programs (and of itself too ;)
+
+Version 4.9.5.1
+* Implemented the "File/Export/Project to HTML" function.
+* Added "Tip of the day" system.
+* When running a source file in explorer, don't spawn new instance.
+  Instead open the file in an already launched Dev-C++.
+* Class-parser speed-up (50% to 85% improvement timed!!!)
+* Many code-completion updates. Now takes into account context,
+  class inheritance and visibility (shows items only from files
+  #included directly or indirectly)!
+* Caching of result set of code-completion for speed-up.
+* New option "Execution/Parameters" (and "Debug/Parameters").
+
+Version 4.9.5.0 (5.0 beta 5):
+* CPU Window (still in development)
+* ToDo  list
+* Backtrace in debugging
+* Run to cursor
+* Folders in Project and Class Browser
+* Send custom commands to GDB
+* Makefile can now be customized.
+* Modified the behaviour of the -c param : 20
+  -c <config file directory>
+* Saving of custom syntax parameter group
+* Possibility of changing compilers and tools filename.
+* Many bug fixes
+
+
+Version 4.9.4.1 (5.0 beta 4.1):
+
+* back to gcc 2.95.3
+* Profiling support
+* new update/packages checker (vUpdate)
+* Lots of bugfixes
+

testing/btest/Baseline/policy.protocols.smtp.mime-extract/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	-	smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat	-

testing/btest/Baseline/policy.protocols.smtp.mime/smtp_entities.log

@@ -0,0 +1,4 @@
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	mid	filename	content_len	mime_type	md5	extraction_file	excerpt
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	79	FAKE_MIME	92bca2e6cdcde73647125da7dccbdd07	-	-
+1254722770.692743	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	-	1918	FAKE_MIME	-	-	-
+1254722770.692804	56gKBmhBBB6	10.10.1.4	1470	74.53.140.153	25	@50da4BEzauh	NEWS.txt	10823	FAKE_MIME	a968bb0f9f9d95835b2e74c845877e87	-	-

testing/btest/core/leaks.bro

@@ -2,4 +2,4 @@
 #
 # @TEST-REQUIRES: bro  --help 2>&1 | grep -q mem-leaks
 #
-# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace test-all 
+# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -r $TRACES/wikipedia.trace test-all-policy

testing/btest/istate/broccoli.bro

@@ -1,7 +1,7 @@
 # @TEST-REQUIRES: grep -vq '#define BROv6' $BUILD/config.h
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 #
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broping-record.bro
+# @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/test/broping-record.bro
 # @TEST-EXEC: btest-bg-run broccoli $BUILD/aux/broccoli/test/broping -r -c 3 127.0.0.1
 # @TEST-EXEC: btest-bg-wait -k 20
 # @TEST-EXEC: cat bro/ping.log | sed 's/one-way.*//g' >bro.log

testing/btest/istate/events-ssl.bro

@@ -1,14 +1,14 @@
 # 
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
+# @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run receiver bro ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
 # @TEST-EXEC: cmp sender/http.log receiver/http.log
 # 
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 bro -x sender/events.bst http/base   | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
+# @TEST-EXEC: bro -x sender/events.bst http/base   | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 bro -x receiver/events.bst http/base | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
+# @TEST-EXEC: bro -x receiver/events.bst http/base | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
 # @TEST-EXEC: cmp events.rec.log events.snd.log
 # 
 # We don't compare the transmitted event paramerters anymore. With the dynamic

testing/btest/istate/events.bro

@@ -1,14 +1,14 @@
 # 
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
+# @TEST-EXEC: btest-bg-run sender   bro -C -r $TRACES/web.trace --pseudo-realtime ../sender.bro
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run receiver bro ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 20
 # 
 # @TEST-EXEC: btest-diff sender/http.log
 # @TEST-EXEC: btest-diff receiver/http.log
 # @TEST-EXEC: cmp sender/http.log receiver/http.log
 # 
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
+# @TEST-EXEC: bro -x sender/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.snd.log
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
+# @TEST-EXEC: bro -x receiver/events.bst | sed 's/^Event \[[-0-9.]*\] //g' | grep '^http_' | grep -v http_stats | sed 's/(.*$//g'  >events.rec.log
 # @TEST-EXEC: cmp events.rec.log events.snd.log
 # 
 # We don't compare the transmitted event paramerters anymore. With the dynamic

testing/btest/istate/pybroccoli.py

@@ -2,7 +2,7 @@
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/src/libbroccoli.so || test -e $BUILD/aux/broccoli/src/libbroccoli.dylib
 # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/bindings/broccoli-python/_broccoli_intern.so
 #
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run bro    bro %INPUT $DIST/aux/broccoli/bindings/broccoli-python/tests/test.bro
+# @TEST-EXEC: btest-bg-run bro    bro %INPUT $DIST/aux/broccoli/bindings/broccoli-python/tests/test.bro
 # @TEST-EXEC: btest-bg-run python PYTHONPATH=$DIST/aux/broccoli/bindings/broccoli-python/:$BUILD/aux/broccoli/bindings/broccoli-python python $DIST/aux/broccoli/bindings/broccoli-python/tests/test.py
 # @TEST-EXEC: btest-bg-wait -k 20
 # @TEST-EXEC: btest-diff bro/.stdout

testing/btest/istate/sync.bro

@@ -1,6 +1,6 @@
 #
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run sender   bro %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender   bro %INPUT ../sender.bro
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run receiver bro %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro %INPUT ../receiver.bro
 # @TEST-EXEC: btest-bg-wait 20
 #
 # @TEST-EXEC: btest-diff sender/vars.log

testing/btest/policy/frameworks/cluster/start-it-up.bro

@@ -1,8 +1,8 @@
-# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 CLUSTER_NODE=proxy-2 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-2   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-2 bro %INPUT
-# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
-# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
 # @TEST-EXEC: btest-bg-wait -k 2
 # @TEST-EXEC: btest-diff manager-1/.stdout
 # @TEST-EXEC: btest-diff proxy-1/.stdout

testing/btest/policy/frameworks/control/configuration_update.bro

@@ -1,6 +1,6 @@
-# @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65531/tcp 
+# @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65531/tcp 
-# @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
+# @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT test-redef frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=configuration_update
-# @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown
+# @TEST-EXEC: btest-bg-run controller2 BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65531/tcp Control::cmd=shutdown
 # @TEST-EXEC: btest-bg-wait 1
 # @TEST-EXEC: btest-diff controllee/.stdout
 

testing/btest/policy/frameworks/control/id_value.bro

@@ -1,5 +1,5 @@
-# @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port_clear=65532/tcp 
+# @TEST-EXEC: btest-bg-run controllee  BROPATH=$BROPATH:.. bro %INPUT only-for-controllee frameworks/control/controllee Communication::listen_port_clear=65532/tcp 
-# @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var
+# @TEST-EXEC: btest-bg-run controller  BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65532/tcp Control::cmd=id_value Control::arg=test_var
 # @TEST-EXEC: btest-bg-wait -k 1
 # @TEST-EXEC: btest-diff controller/.stdout
 

testing/btest/policy/frameworks/control/shutdown.bro

@@ -1,5 +1,5 @@
-# @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65530/tcp 
+# @TEST-EXEC: btest-bg-run controllee BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controllee Communication::listen_port_clear=65530/tcp 
-# @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. ENABLE_COMMUNICATION=1 bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown
+# @TEST-EXEC: btest-bg-run controller BROPATH=$BROPATH:.. bro %INPUT frameworks/control/controller Control::host=127.0.0.1 Control::host_port=65530/tcp Control::cmd=shutdown
 # @TEST-EXEC: btest-bg-wait 1
 
 redef Communication::nodes = {

testing/btest/policy/frameworks/logging/remote-types.bro

@@ -1,6 +1,6 @@
 #
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: btest-bg-wait -k 1
 # @TEST-EXEC: btest-diff receiver/test.log
 # @TEST-EXEC: cmp receiver/test.log sender/test.log

testing/btest/policy/frameworks/logging/remote.bro

@@ -1,7 +1,7 @@
 #
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
+# @TEST-EXEC: btest-bg-run sender bro --pseudo-realtime %INPUT ../sender.bro
 # @TEST-EXEC: sleep 1
-# @TEST-EXEC: ENABLE_COMMUNICATION=1 btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
+# @TEST-EXEC: btest-bg-run receiver bro --pseudo-realtime %INPUT ../receiver.bro
 # @TEST-EXEC: sleep 1
 # @TEST-EXEC: btest-bg-wait -k 1
 # @TEST-EXEC: btest-diff sender/test.log

testing/btest/policy/frameworks/logging/rotate-custom.bro

@@ -1,5 +1,5 @@
 #
-# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT | egrep "test|test2" | sort >out
+# @TEST-EXEC: bro -b -r %DIR/rotation.trace %INPUT 2>&1 | egrep "test|test2" | sort >out
 # @TEST-EXEC: for i in `ls test*.log | sort`; do printf '> %s\n' $i; cat $i; done | sort | uniq >>out
 # @TEST-EXEC: btest-diff out
 
@@ -18,10 +18,16 @@ export {
 }
 
 redef Log::default_rotation_interval = 1hr;
-redef Log::default_rotation_postprocessor = "echo 1st";
+redef Log::default_rotation_postprocessor_cmd = "echo 1st";
+
+function custom_rotate(info: Log::RotationInfo) : bool
+{
+    print "custom rotate", info;
+    return T;
+}
 
 redef Log::rotation_control += {
-	[Log::WRITER_ASCII, "test2"] = [$interv=30mins, $postprocessor="echo 2nd"]
+	[Log::WRITER_ASCII, "test2"] = [$interv=30mins, $postprocessor=custom_rotate]
 };
 
 event bro_init()

testing/btest/policy/frameworks/logging/rotate.bro

@@ -1,6 +1,6 @@
 #
-# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT | grep "test" >out
+# @TEST-EXEC: bro -r %DIR/rotation.trace %INPUT 2>&1 | grep "test" >out
-# @TEST-EXEC: for i in test-*.log; do printf '> %s\n' $i; cat $i; done >>out
+# @TEST-EXEC: for i in test.*.log; do printf '> %s\n' $i; cat $i; done >>out
 # @TEST-EXEC: btest-diff out
 
 module Test;
@@ -18,7 +18,7 @@ export {
 }
 
 redef Log::default_rotation_interval = 1hr;
-redef Log::default_rotation_postprocessor = "echo";
+redef Log::default_rotation_postprocessor_cmd = "echo";
 
 event bro_init()
 {

testing/btest/policy/misc/bare-loaded-scripts.test

@@ -0,0 +1,12 @@
+# This test is meant to cover whether the set of scripts that get loaded by
+# default in bare mode matches a baseline of known defaults.
+#
+# As the output has absolute paths in it, we need to remove the common
+# prefix to make the test work everywhere. That's what the sed magic
+# below does. Don't ask. :-)
+
+# @TEST-EXEC: bro -b misc/loaded-scripts
+# @TEST-EXEC: test -e loaded_scripts.log
+# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
+# @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/policy/misc/check-bare-test-all-policy.bro

@@ -0,0 +1,7 @@
+# Makes sures test-all-policy.bro (which loads *all* other policy scripts)
+# compiles correctly even in bare mode.
+# 
+# @TEST-EXEC: bro -b %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load test-all-policy

testing/btest/policy/misc/check-test-all-policy.bro

@@ -0,0 +1,6 @@
+# Makes sures test-all-policy.bro (which loads *all* other policy scripts) compiles correctly.
+# 
+# @TEST-EXEC: bro %INPUT >output
+# @TEST-EXEC: btest-diff output
+
+@load test-all-policy

testing/btest/policy/misc/check-test-all.bro

@@ -1,6 +0,0 @@
-# Makes sures test-all.bro (which loads *all* other scripts) compiles correctly.
-#
-# @TEST-EXEC: bro %INPUT >output
-# @TEST-EXEC: btest-diff output
-
-@load test-all

testing/btest/policy/misc/init-default-coverage.bro

@@ -0,0 +1,11 @@
+# Makes sure that all base/* scripts are loaded by default via init-default.bro;
+# and that all scripts loaded there in there actually exist.
+
+@TEST-EXEC: test -d $DIST/scripts/base
+@TEST-EXEC: test -e $DIST/scripts/base/init-default.bro
+@TEST-EXEC: ( cd $DIST/scripts/base && find . -name '*.bro' ) | sort >"all scripts found"
+@TEST-EXEC: bro misc/loaded-scripts
+@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
+@TEST-EXEC: cat loaded_scripts.log.tmp | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+@TEST-EXEC: cat loaded_scripts.log.tmp | sed "s#`cat prefix`#./#g" | sort >init-default.bro
+@TEST-EXEC: diff -u "all scripts found" init-default.bro 1>&2

testing/btest/policy/misc/test-all-default-coverage.bro

@@ -0,0 +1,8 @@
+# Makes sure that all policy/* scripts are loaded in test-all-policy.bro; and that
+# all scripts loaded there actually exist.
+
+@TEST-EXEC: test -e $DIST/scripts/test-all-policy.bro
+@TEST-EXEC: test -d $DIST/scripts
+@TEST-EXEC: ( cd $DIST/scripts/policy && find . -name '*.bro' ) | sort >"all scripts found"
+@TEST-EXEC: cat $DIST/scripts/test-all-policy.bro | grep '@load' | sed 'sm^\( *# *\)\{0,\}@load *m./mg' | sort >test-all-policy.bro
+@TEST-EXEC: diff -u "all scripts found" test-all-policy.bro 1>&2

testing/btest/policy/misc/testing-coverage.bro

@@ -1,6 +0,0 @@
-# Makes sure that all policy scripts are loading in testing.bro; and that all
-# scripts loaded there actually exist.
-
-@TEST-EXEC: ( cd $DIST/policy && find . -name '*.bro' ) | sort >"all scripts found"
-@TEST-EXEC: cat $DIST/policy/test-all.bro | grep '@load' | sed 'sm^\( *# *\)\{0,\}@load *m./mg' | sort >test-all.bro
-@TEST-EXEC: diff -u "all scripts found" test-all.bro 1>&2

testing/btest/policy/protocols/smtp/basic.test

@@ -0,0 +1,4 @@
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp.log
+
+@load base/protocols/smtp

testing/btest/policy/protocols/smtp/mime-extract.test

@@ -0,0 +1,25 @@
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: btest-diff smtp-entity_10.10.1.4:1470-74.53.140.153:25_2.dat
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT SMTP::extraction_prefix="test"
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_1.dat
+# @TEST-EXEC: test -e test_10.10.1.4:1470-74.53.140.153:25_2.dat
+
+@load base/protocols/smtp
+
+redef SMTP::extract_file_types=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  	$pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}

testing/btest/policy/protocols/smtp/mime.test

@@ -0,0 +1,23 @@
+# Checks logging of mime types and md5 calculation.  Mime type in the log
+# is normalized to prevent sensitivity to libmagic version.
+
+# @TEST-REQUIRES: grep -q '#define HAVE_LIBMAGIC' $BUILD/config.h
+# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT
+# @TEST-EXEC: btest-diff smtp_entities.log
+
+@load base/protocols/smtp
+
+redef SMTP::generate_md5=/text\/plain/;
+
+event bro_init()
+	{
+	Log::remove_default_filter(SMTP::SMTP_ENTITIES);
+	Log::add_filter(SMTP::SMTP_ENTITIES, [$name="normalized-mime-types",
+									  $pred=function(rec: SMTP::EntityInfo): bool
+		{
+		if ( rec?$mime_type )
+			rec$mime_type = "FAKE_MIME";
+		return T;
+		}
+	]);
+	}