Merge branch 'master' of ssh://git.bro-ids.org/bro

CHANGES

@@ -1,4 +1,70 @@
 
+1.6-dev-1293 | 2011-09-22 19:44:37 -0700
+
+  * Smaller script tweaks. (Seth Hall)
+
+  * Duplicate notice suppression. (Seth Hall)
+
+    - Duplicate notices are discovered with the new Notice::Info
+      field $identifier.  It's a string that is left up to the
+      notice implementor to define which would indicate a
+      fundamentally duplicate notice.  The field is optional and
+      if it's not included it's not possible for notice
+      suppression to take place.
+
+    - Duplicate notices are suppressed by default for the interval
+      defined by the Notice::default_suppression_interval variable
+      (1 hour by default).
+
+    - A new notice action was defined ACTION_NO_SUPPRESS to prevent
+      suppression for a specific notice instance.  A convenience set
+      named not_suppressed_types was also created to not suppress
+      entire notice types.
+
+    - A new field was added to the PolicyItem type to modify the length
+      of time a notice should be suppressed if the predicate matches.
+      The field is named $suppress_for.  This name makes the code more
+      readable like this: $suppress_for = 1day
+
+    - New events were created to give visibility into the notice
+      framework's suppression activity.
+      - event Notice::begin_suppression(n: Notice::Info)
+      - event Notice::suppressed(n: Notice::Info)
+      - event Notice::end_suppression(n: Notice::Info)
+
+    - The suppression.bro script doesn't have a baseline because
+      it is causing a segfault in Bro.  This one test is the
+      reason that this is being integrated into a branch instead
+      of master. (Seth Hall)
+
+  * Fix crash on exit. Addresses #607. (Jon Siwek)
+
+  * Fix PktSrc setting next_timestamp even when no packet available.
+    (Jon Siwek)
+
+  * Fix lack of NUL-termination in to_upper/to_lower BIF's return val.
+    (Jon Siwek)
+
+  * Fixing unit tests and some minor bugs. (Jon Siwek)
+
+  * Fix broctl cluster log rotation. Addresses #619. (Jon Siwek)
+
+  * Added session ID to the SSL logging. (Seth Hall)
+
+  * Adding "install-aux" target + updating bro-aux submodule. (Jon
+    Siwek)
+
+  * Cleaning up INSTALL and README. (Jon Siwek)
+
+  * Remove $Id$ tags. (Jon Siwek)
+
+  * Remove policy.old directory. Addresses #511. (Jon Siwek)
+
+  * Small rework with ssl base script to reduce memory usage. (Seth
+    Hall)
+
+  * Updated the mozilla root certs. (Seth Hall)
+
 1.6-dev-1261 | 2011-09-15 17:13:55 -0700
 
   * Memory leak fixes. Addresses #574 (Jon Siwek)

VERSION

@@ -1 +1 @@
-1.6-dev-1261
+1.6-dev-1293

aux/binpac

@@ -1 +1 @@
-Subproject commit 796b6e7e1492021984bbc297b3fcc2952f20c778
+Subproject commit 0b14940f684ec5c52faae4908e8f04c182b099c1

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 2a5a709b5c85e74a88d88ef385ee0fb44ca02cd6
+Subproject commit a152760509e63e85360d5512aecf8ee772e9afe1

aux/broccoli

@@ -1 +1 @@
-Subproject commit adae807493dbb24ec8baf759e4589258ae57cc3e
+Subproject commit 9f2fbbc58921383d24c54af578c9ad012815147b

cmake

@@ -1 +1 @@
-Subproject commit 47fa45b726793aa15491d240cd5a84f4a4f7bd0a
+Subproject commit bbf129bd7bd33dfb5641ff0d9242f4b3ebba8e82

doc/scripts/DocSourcesList.cmake

@@ -102,7 +102,6 @@ rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/pf-ring-load-balancing.bro)
 rest_target(${psd} policy/misc/profiling.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
@@ -115,7 +114,7 @@ rest_target(${psd} policy/protocols/http/detect-MHR.bro)
 rest_target(${psd} policy/protocols/http/detect-intel.bro)
 rest_target(${psd} policy/protocols/http/detect-sqli.bro)
 rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/headers.bro)
+rest_target(${psd} policy/protocols/http/header-names.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)

scripts/base/frameworks/cluster/main.bro

@@ -23,10 +23,10 @@ export {
 	const manager_events = /Drop::.*/ &redef;
 	
 	## Events raised by the proxies and handled by the manager.
-	const proxy_events = /Notice::notice/ &redef;
+	const proxy_events = /EMPTY/ &redef;
 	
 	## Events raised by workers and handled by the manager.
-	const worker_events = /(Notice::notice|TimeMachine::command|Drop::.*)/ &redef;
+	const worker_events = /(TimeMachine::command|Drop::.*)/ &redef;
 	
 	## Events sent by the control host (i.e. BroControl) when dynamically 
 	## connecting to a running instance to update settings or request data.

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -8,8 +8,6 @@
 ##! This is where the cluster manager sets it's specific settings for other
 ##! frameworks and in the core.
 
-@load base/frameworks/notice
-
 @prefixes += cluster-manager
 
 # Load the script for local site configuration for the manager node.
@@ -23,10 +21,3 @@ redef Log::default_rotation_postprocessor_cmd = "archive-log";
 
 ## We're processing essentially *only* remote events.
 redef max_remote_events_processed = 10000;
-
-# Reraise remote notices locally.
-event Notice::notice(n: Notice::Info)
-	{
-	if ( is_remote_event() )
-		NOTICE(n);
-	}

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,5 +1,3 @@
-@load base/frameworks/notice
-
 @prefixes += cluster-worker
 
 # Load the script for local site configuration for the worker nodes.
@@ -15,12 +13,8 @@ redef Log::enable_remote_logging = T;
 redef Log::default_rotation_postprocessor_cmd = "delete-log";
 
 ## Record all packets into trace file.
-# TODO: should we really be setting this to T?
+##
+## Note that this only indicates that *if* we are recording packets, we want all
+## of them (rather than just those the core deems sufficiently important). Setting
+## this does not turn recording on. Use '-w <trace>' for that.
 redef record_all_packets = T;
-
-# Workers need to have a filter for the notice log which doesn't 
-# do remote logging since we forward the notice event directly.
-event bro_init()
-	{
-	Log::disable_stream(Notice::LOG);
-	}

scripts/base/frameworks/logging/main.bro

@@ -192,10 +192,7 @@ function default_path_func(id: ID, path: string, rec: any) : string
 				output = cat(output, "_", module_parts[3]);
 			if ( 4 in module_parts && module_parts[4] != "" )
 				output = cat(output, sub_bytes(module_parts[4],1,1), "_", sub_bytes(module_parts[4], 2, |module_parts[4]|));
-			# TODO: There seems to be some problem with the split function 
-			#       not putting \0 at the end of the string.  fmt will make
-			#       a better internal string.
-			return fmt("%s", to_lower(output));
+			return to_lower(output);
 			}
 		
 		# Example: Notice::POLICY_LOG -> "notice_policy"
@@ -205,10 +202,7 @@ function default_path_func(id: ID, path: string, rec: any) : string
 		return cat(to_lower(parts[1]),"_",to_lower(parts[2]));
 		}
 	else
-		{
-		# In case there is a logging stream in the global namespace.
 		return to_lower(id_str);
-		}
 	}
 
 # Run post-processor on file. If there isn't any postprocessor defined,
@@ -241,8 +235,7 @@ function create_stream(id: ID, stream: Stream) : bool
 
 function disable_stream(id: ID) : bool
 	{
-	if ( ! __disable_stream(id) )
-		return F;
+	return __disable_stream(id);
 	}
 
 function add_filter(id: ID, filter: Filter) : bool

scripts/base/frameworks/notice/__load__.bro

@@ -11,3 +11,10 @@
 # There shouldn't be any defaul toverhead from loading these since they 
 # *should* only do anything when notices have the ACTION_EMAIL action applied.
 @load ./extend-email/hostnames
+
+# The cluster framework must be loaded first.
+@load base/frameworks/cluster
+
+@if ( Cluster::is_enabled() )
+@load ./cluster
+@endif

scripts/base/frameworks/notice/cluster.bro

@@ -0,0 +1,43 @@
+##! Implements notice functionality across clusters.
+
+@load ./main
+@load base/frameworks/cluster
+
+module Notice;
+
+# Define the event used to transport notices on the cluster.
+global cluster_notice: event(n: Notice::Info);
+
+redef Cluster::manager_events += /Notice::begin_suppression/;
+redef Cluster::proxy_events += /Notice::cluster_notice/;
+redef Cluster::worker_events += /Notice::cluster_notice/;
+
+@if ( Cluster::local_node_type() != Cluster::MANAGER )
+event Notice::begin_suppression(n: Notice::Info)
+	{
+	suppressing[n$note, n$identifier] = n;
+	}
+
+event Notice::notice(n: Notice::Info)
+	{
+	# Send the locally generated notice on to the manager.
+	event Notice::cluster_notice(n);
+	}
+
+event bro_init() &priority=3
+	{
+	# Workers and proxies need to disable the notice streams because notice
+	# events are forwarded directly instead of being logged remotely.
+	Log::disable_stream(Notice::LOG);
+	Log::disable_stream(Notice::POLICY_LOG);
+	Log::disable_stream(Notice::ALARM_LOG);
+	}
+@endif
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+event Notice::cluster_notice(n: Notice::Info)
+	{
+	# Raise remotely received notices on the manager
+	NOTICE(n);
+	}
+@endif

scripts/base/frameworks/notice/main.bro

@@ -43,8 +43,16 @@ export {
 		## version of the alarm log is emailed in bulk to the address(es)
 		## configured in :bro:id:`Notice::mail_dest`.
 		ACTION_ALARM,
+		## Indicates that the notice should not be supressed by the normal
+		## duplicate notice suppression that the notice framework does.
+		ACTION_NO_SUPPRESS,
 	};
 	
+	## The notice framework is able to do automatic notice supression by 
+	## utilizing the $identifier field in :bro:type:`Info` records.
+	## Set this to "0secs" to completely disable automated notice suppression.
+	const default_suppression_interval = 1hrs &redef;
+	
 	type Info: record {
 		ts:             time           &log &optional;
 		uid:            string         &log &optional;
@@ -82,7 +90,6 @@ export {
 		
 		## These are policy items that returned T and applied their action
 		## to the notice.
-		## TODO: this can't take set() as a default. (bug)
 		policy_items:   set[count]     &log &optional;
 		
 		## By adding chunks of text into this element, other scripts can
@@ -90,6 +97,34 @@ export {
 		## is to extend the vector by handling the :bro:id:`Notice::notice`
 		## event and modifying the notice in place.
 		email_body_sections:  vector of string &default=vector();
+		
+		## This field is to be provided when a notice is generated for the
+		## purpose of deduplicating notices.  The identifier string should
+		## be unique for a single instance of the notice.  This field should be 
+		## filled out in almost all cases when generating notices to define 
+		## when a notice is conceptually a duplicate of a previous notice.
+		## 
+		## For example, an SSL certificate that is going to expire soon should 
+		## always have the same identifier no matter the client IP address 
+		## that connected and resulted in the certificate being exposed.  In
+		## this case, the resp_h, resp_p, and hash of the certificate would be
+		## used to create this value.  The hash of the cert is included
+		## because servers can return multiple certificates on the same port.
+		##
+		## Another example might be a host downloading a file which triggered
+		## a notice because the MD5 sum of the file it downloaded was known
+		## by some set of intelligence.  In that case, the orig_h (client)
+		## and MD5 sum would be used in this field to dedup because if the 
+		## same file is downloaded over and over again you really only want to
+		## know about it a single time.  This makes it possible to send those
+		## notices to email without worrying so much about sending thousands
+		## of emails.
+		identifier:          string         &optional;
+		
+		## This field indicates the length of time that this
+		## unique notice should be suppressed.  This field is automatically 
+		## filled out and should not be written to by any other script.
+		suppress_for:        interval       &log &optional;
 	};
 	
 	## Ignored notice types.
@@ -98,6 +133,8 @@ export {
 	const emailed_types: set[Notice::Type] = {} &redef;
 	## Alarmed notice types.
 	const alarmed_types: set[Notice::Type] = {} &redef;
+	## Types that should be suppressed for the default suppression interval.
+	const not_suppressed_types: set[Notice::Type] = {} &redef;
 	
 	## This is the record that defines the items that make up the notice policy.
 	type PolicyItem: record {
@@ -117,6 +154,9 @@ export {
 		## Indicates this item should terminate policy processing if the 
 		## predicate returns T.
 		halt:     bool                             &log &default=F;
+		## This defines the length of time that this particular notice should
+		## be supressed.
+		suppress_for: interval                     &log &optional;
 	};
 	
 	## This is the where the :bro:id:`Notice::policy` is defined.  All notice
@@ -124,6 +164,9 @@ export {
 	const policy: set[PolicyItem] = {
 		[$pred(n: Notice::Info) = { return (n$note in Notice::ignored_types); },
 		 $halt=T, $priority = 9],
+		[$pred(n: Notice::Info) = { return (n$note in Notice::not_suppressed_types); },
+		 $result = ACTION_NO_SUPPRESS,
+		 $priority = 9],
 		[$pred(n: Notice::Info) = { return (n$note in Notice::alarmed_types); },
 		 $priority = 8],
 		[$pred(n: Notice::Info) = { return (n$note in Notice::emailed_types); },
@@ -173,6 +216,13 @@ export {
 	## real time constraints.
 	const sync_functions: set[function(n: Notice::Info)] = set() &redef;
 	
+	## This event is generated when a notice begins to be suppressed.
+	global begin_suppression: event(n: Notice::Info);
+	## This event is generated on each occurence of an event being suppressed.
+	global suppressed: event(n: Notice::Info);
+	## This event is generated when a notice stops being suppressed.
+	global end_suppression: event(n: Notice::Info);
+	
 	## Call this function to send a notice in an email.  It is already used
 	## by default with the built in :bro:enum:`ACTION_EMAIL` and
 	## :bro:enum:`ACTION_PAGE` actions.
@@ -184,17 +234,41 @@ export {
 	## dest: recipient string to use for the mail
 	## Returns: a string of mail headers to which an email body can be appended
 	global email_headers: function(subject_desc: string, dest: string): string;
-
-	## This is an internally used function, please ignore it.  It's only used
-	## for filling out missing details of :bro:type:`Notice:Info` records
-	## before the synchronous and asynchronous event pathways have begun.
-	global apply_policy: function(n: Notice::Info);
 	
 	## This event can be handled to access the :bro:type:`Info`
 	## record as it is sent on to the logging framework.
 	global log_notice: event(rec: Info);
+	
+	## This is an internal wrapper for the global NOTICE function.  Please 
+	## disregard.
+	global internal_NOTICE: function(n: Notice::Info);
 }
 
+# This is used as a hack to implement per-item expiration intervals.
+function per_notice_suppression_interval(t: table[Notice::Type, string] of Notice::Info, idx: any): interval
+	{
+	local n: Notice::Type;
+	local s: string;
+	[n,s] = idx;
+	
+	local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
+	if ( suppress_time < 0secs )
+		suppress_time = 0secs;
+	
+	# If there is no more suppression time left, the notice needs to be sent
+	# to the end_suppression event.
+	if ( suppress_time == 0secs )
+		event Notice::end_suppression(t[n,s]);
+	
+	return suppress_time;
+	}
+
+# This is the internally maintained notice suppression table.  It's 
+# indexed on the Notice::Type and the $identifier field from the notice.
+global suppressing: table[Type, string] of Notice::Info = {} 
+		&create_expire=0secs
+		&expire_func=per_notice_suppression_interval;
+
 # This is an internal variable used to store the notice policy ordered by
 # priority.
 global ordered_policy: vector of PolicyItem = vector();
@@ -292,8 +366,32 @@ event notice(n: Notice::Info) &priority=-5
 		Log::write(Notice::LOG, n);
 	if ( ACTION_ALARM in n$actions )
 		Log::write(Notice::ALARM_LOG, n);
+	
+	# Normally suppress further notices like this one unless directed not to.
+	#  n$identifier *must* be specified for suppression to function at all.
+	if ( n?$identifier && 
+	     ACTION_NO_SUPPRESS !in n$actions &&
+	     [n$note, n$identifier] !in suppressing &&
+	     n$suppress_for != 0secs )
+		{
+		suppressing[n$note, n$identifier] = n;
+		event Notice::begin_suppression(n);
+		}
 	}
-
+	
+## This determines if a notice is being suppressed.  It is only used 
+## internally as part of the mechanics for the global NOTICE function.
+function is_being_suppressed(n: Notice::Info): bool
+	{
+	if ( n?$identifier && [n$note, n$identifier] in suppressing )
+		{
+		event Notice::suppressed(n);
+		return T;
+		}
+	else
+		return F;
+	}
+	
 # Executes a script with all of the notice fields put into the
 # new process' environment as "BRO_ARG_<field>" variables.
 function execute_with_notice(cmd: string, n: Notice::Info)
@@ -357,12 +455,23 @@ function apply_policy(n: Notice::Info)
 			add n$actions[ordered_policy[i]$result];
 			add n$policy_items[int_to_count(i)];
 			
+			# If the predicate matched and there was a suppression interval, 
+			# apply it to the notice now.
+			if ( ordered_policy[i]?$suppress_for )
+				n$suppress_for = ordered_policy[i]$suppress_for;
+			
 			# If the policy item wants to halt policy processing, do it now!
 			if ( ordered_policy[i]$halt )
 				break;
 			}
 		}
 	
+	# Apply the suppression time after applying the policy so that policy
+	# items can give custom suppression intervals.  If there is no 
+	# suppression interval given yet, the default is applied.
+	if ( ! n?$suppress_for )
+		n$suppress_for = default_suppression_interval;
+	
 	# Delete the connection record if it's there so we aren't sending that
 	# to remote machines.  It can cause problems due to the size of the 
 	# connection record.
@@ -402,19 +511,28 @@ event bro_init() &priority=10
 			}
 		}
 	}
+	
+function internal_NOTICE(n: Notice::Info)
+	{
+	# Suppress this notice if necessary.
+	if ( is_being_suppressed(n) )
+		return;
+
+	# Fill out fields that might be empty and do the policy processing.
+	apply_policy(n);
+
+	# Run the synchronous functions with the notice.
+	for ( func in sync_functions )
+		func(n);
+
+	# Generate the notice event with the notice.
+	event Notice::notice(n);
+	}
 
 module GLOBAL;
 
 ## This is the entry point in the global namespace for notice framework.
 function NOTICE(n: Notice::Info)
 	{
-	# Fill out fields that might be empty and do the policy processing.
-	Notice::apply_policy(n);
-
-	# Run the synchronous functions with the notice.
-	for ( func in Notice::sync_functions )
-		func(n);
-
-	# Generate the notice event with the notice.
-	event Notice::notice(n);
+	Notice::internal_NOTICE(n);
 	}

scripts/policy/protocols/http/header-names.bro

@@ -14,7 +14,7 @@ export {
 		
 		## The vector of HTTP header names sent by the server.  No header 
 		## values are included here, just the header names.
-		server_headers_names:  vector of string &log &optional;
+		server_header_names:  vector of string &log &optional;
 	};
 	
 	## A boolean value to determine if client header names are to be logged.

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -1,4 +1,5 @@
 
+@load base/protocols/ssh
 @load base/frameworks/metrics
 @load base/frameworks/notice
 @load base/frameworks/intel
@@ -72,4 +73,4 @@ event SSH::heuristic_failed_login(c: connection)
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
 		Metrics::add_data(FAILED_LOGIN, [$host=id$orig_h], 1);
-	}
+	}

scripts/test-all-policy.bro

@@ -25,6 +25,7 @@
 @load integration/barnyard2/types.bro
 @load misc/analysis-groups.bro
 @load misc/loaded-scripts.bro
+@load misc/profiling.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
 @load protocols/conn/known-services.bro
@@ -37,11 +38,16 @@
 @load protocols/http/detect-MHR.bro
 @load protocols/http/detect-sqli.bro
 @load protocols/http/detect-webapps.bro
+@load protocols/http/header-names.bro
 @load protocols/http/software.bro
 @load protocols/http/var-extraction-cookies.bro
 @load protocols/http/var-extraction-uri.bro
+@load protocols/smtp/blocklists.bro
 @load protocols/smtp/detect-suspicious-orig.bro
 @load protocols/smtp/software.bro
+@load protocols/ssh/detect-bruteforcing.bro
+@load protocols/ssh/geo-data.bro
+@load protocols/ssh/interesting-hostnames.bro
 @load protocols/ssh/software.bro
 @load protocols/ssl/known-certs.bro
 @load protocols/ssl/validate-certs.bro

src/Func.cc

@@ -334,7 +334,7 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 	     (flow != FLOW_RETURN /* we fell off the end */ ||
 	      ! result /* explicit return with no result */) &&
 	     ! f->HasDelayed() )
-		reporter->Warning("non-void function returns without a value:", id->Name());
+		reporter->Warning("non-void function returns without a value: %s", id->Name());
 
 	if ( result && g_trace_state.DoTrace() )
 		{

src/LogMgr.cc

@@ -1266,20 +1266,31 @@ LogWriter* LogMgr::CreateWriter(EnumVal* id, EnumVal* writer, string path,
 	winfo->postprocessor = 0;
 
 	// Search for a corresponding filter for the writer/path pair and use its
-	// rotation settings.
+	// rotation settings.  If no matching filter is found, fall back on
+	// looking up the logging framework's default rotation interval.
+	bool found_filter_match = false;
 	list<Filter*>::const_iterator it;
+
 	for ( it = stream->filters.begin(); it != stream->filters.end(); ++it )
 		{
 		Filter* f = *it;
 		if ( f->writer->AsEnum() == writer->AsEnum() &&
 		     f->path == winfo->writer->Path() )
 			{
+			found_filter_match = true;
 			winfo->interval = f->interval;
 			winfo->postprocessor = f->postprocessor;
 			break;
 			}
 		}
 
+	if ( ! found_filter_match )
+		{
+		ID* id = global_scope()->Lookup("Log::default_rotation_interval");
+		assert(id);
+		winfo->interval = id->ID_Val()->AsInterval();
+		}
+
 	InstallRotationTimer(winfo);
 
 	stream->writers.insert(

src/PktSrc.cc

@@ -21,6 +21,7 @@ PktSrc::PktSrc()
 	{
 	interface = readfile = 0;
 	data = last_data = 0;
+	memset(&hdr, 0, sizeof(hdr));
 	hdr_size = 0;
 	datalink = 0;
 	netmask = 0xffffff00;
@@ -75,7 +76,9 @@ int PktSrc::ExtractNextPacket()
 		}
 
 	data = last_data = pcap_next(pd, &hdr);
-	next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
+
+	if ( data )
+		next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
 
 	if ( pseudo_realtime )
 		current_wallclock = current_time(true);

src/Scope.cc

@@ -217,5 +217,5 @@ Scope* current_scope()
 
 Scope* global_scope()
 	{
-	return scopes[0];
+	return scopes.length() == 0 ? 0 : scopes[0];
 	}

src/Val.cc

@@ -636,14 +636,16 @@ MutableVal::~MutableVal()
 	{
 	for ( list<ID*>::iterator i = aliases.begin(); i != aliases.end(); ++i )
 		{
-		global_scope()->Remove((*i)->Name());
+		if ( global_scope() )
+			global_scope()->Remove((*i)->Name());
 		(*i)->ClearVal();	// just to make sure.
 		Unref((*i));
 		}
 
 	if ( id )
 		{
-		global_scope()->Remove(id->Name());
+		if ( global_scope() )
+			global_scope()->Remove(id->Name());
 		id->ClearVal(); // just to make sure.
 		Unref(id);
 		}

src/strings.bif

@@ -493,7 +493,7 @@ function to_lower%(str: string%): string
 			*ls++ = s[i];
 		}
 
-	return new StringVal(new BroString(1, byte_vec(lower_s), n));
+	return new StringVal(n, lower_s);
 	%}
 
 function to_upper%(str: string%): string
@@ -511,7 +511,7 @@ function to_upper%(str: string%): string
 			*us++ = s[i];
 		}
 
-	return new StringVal(new BroString(1, byte_vec(upper_s), n));
+	return new StringVal(n, upper_s);
 	%}
 
 function clean%(str: string%): string

testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log

@@ -1,5 +1,5 @@
 #separator \x09
 #path	notice
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	bool	string	string	string	double	double	addr	string	subnet
-1315167088.906913	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	4	-	-	-	-	-	-	1.2.3.4	-	-
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
+1316745737.870305	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 100/100	-	1.2.3.4	-	-	100	manager-1	Notice::ACTION_LOG	5	3600.000000	-	-	-	-	-	-	1.2.3.4	-	-

testing/btest/Baseline/scripts.base.frameworks.metrics.notice/notice.log

@@ -1,6 +1,6 @@
 #separator \x09
 #path	notice
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
-#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	bool	string	string	string	double	double	addr	string	subnet
-1315167098.061022	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	4	-	-	-	-	-	-	1.2.3.4	-	-
-1315167098.061022	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	4	-	-	-	-	-	-	6.5.4.3	-	-
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
+1316535463.805974	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=1.2.3.4) 3/2	-	1.2.3.4	-	-	3	bro	Notice::ACTION_LOG	5	3600.000000	-	-	-	-	-	-	1.2.3.4	-	-
+1316535463.805974	-	-	-	-	-	Test_Notice	Threshold crossed by metric_index(host=6.5.4.3) 2/2	-	6.5.4.3	-	-	2	bro	Notice::ACTION_LOG	5	3600.000000	-	-	-	-	-	-	6.5.4.3	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log

@@ -0,0 +1,5 @@
+#separator \x09
+#path	notice
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
+1316447937.245073	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-1	Notice::ACTION_LOG	5	3600.000000	-	-	-	-	-	-	-	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log

@@ -0,0 +1,5 @@
+#separator \x09
+#path	notice
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude	metric_index.host	metric_index.str	metric_index.network
+#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double	addr	string	subnet
+1316447918.818400	-	-	-	-	-	Test_Notice	test notice!	-	-	-	-	-	worker-2	Notice::ACTION_LOG	5	3600.000000	-	-	-	-	-	-	-	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.suppression-disable/notice.log

@@ -0,0 +1,6 @@
+#separator \x09
+#path	notice
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double
+1316442414.185287	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_NO_SUPPRESS,Notice::ACTION_LOG	2,5	3600.000000	-	-	-	-	-	-
+1316442414.185287	-	-	-	-	-	Test_Notice	another test	-	-	-	-	-	bro	Notice::ACTION_NO_SUPPRESS,Notice::ACTION_LOG	2,5	3600.000000	-	-	-	-	-	-

testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log

@@ -0,0 +1,5 @@
+#separator \x09
+#path	notice
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	note	msg	sub	src	dst	p	n	peer_descr	actions	policy_items	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+#types	time	string	addr	port	addr	port	enum	string	string	addr	addr	port	count	string	table	table	interval	bool	string	string	string	double	double
+1316442414.301956	-	-	-	-	-	Test_Notice	test	-	-	-	-	-	bro	Notice::ACTION_LOG	5	3600.000000	-	-	-	-	-	-

testing/btest/Baseline/scripts.policy.misc.bare-loaded-scripts/canonified_loaded_scripts.log

@@ -1,4 +1,7 @@
-# depth	name
+#separator \x09
+#path	loaded_scripts
+#fields	depth	name
+#types	count	string
 0	scripts/base/init-bare.bro
 1	build/src/base/const.bif.bro
 1	build/src/base/types.bif.bro

testing/btest/Baseline/scripts.policy.misc.default-loaded-scripts/canonified_loaded_scripts.log

@@ -1,4 +1,7 @@
-# depth	name
+#separator \x09
+#path	loaded_scripts
+#fields	depth	name
+#types	count	string
 0	scripts/base/init-bare.bro
 1	build/src/base/const.bif.bro
 1	build/src/base/types.bif.bro
@@ -29,6 +32,10 @@
 2	scripts/base/frameworks/notice/./actions/page.bro
 2	scripts/base/frameworks/notice/./actions/add-geodata.bro
 2	scripts/base/frameworks/notice/./extend-email/hostnames.bro
+2	scripts/base/frameworks/cluster/__load__.bro
+3	scripts/base/frameworks/cluster/./main.bro
+4	scripts/base/frameworks/control/__load__.bro
+5	scripts/base/frameworks/control/./main.bro
 1	scripts/base/frameworks/dpd/__load__.bro
 2	scripts/base/frameworks/dpd/./main.bro
 1	scripts/base/frameworks/signatures/__load__.bro
@@ -40,10 +47,6 @@
 2	scripts/base/frameworks/software/./main.bro
 1	scripts/base/frameworks/communication/__load__.bro
 2	scripts/base/frameworks/communication/./main.bro
-1	scripts/base/frameworks/control/__load__.bro
-2	scripts/base/frameworks/control/./main.bro
-1	scripts/base/frameworks/cluster/__load__.bro
-2	scripts/base/frameworks/cluster/./main.bro
 1	scripts/base/frameworks/metrics/__load__.bro
 2	scripts/base/frameworks/metrics/./main.bro
 2	scripts/base/frameworks/metrics/./non-cluster.bro

testing/btest/doc/blacklist-reminder.test

@@ -1,8 +0,0 @@
-# This test will fail if there are Bro scripts that have been temporarily
-# blacklisted from the documentation generation process for some reason
-# (e.g. they're a work-in-progress or otherwise fail to parse).  It's meant
-# to serve as a reminder that some future action may be needed to generate
-# documentation for the blacklisted scripts.
-#
-# @TEST-EXEC: $DIST/doc/scripts/genDocSourcesList.sh
-# @TEST-EXEC: btest-diff .stderr

testing/btest/scripts/base/frameworks/notice/cluster.bro

@@ -0,0 +1,29 @@
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 6
+# @TEST-EXEC: btest-diff manager-1/notice.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=27757/tcp, $workers=set("worker-1")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=27758/tcp, $manager="manager-1", $workers=set("worker-1")],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=27760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"],
+};
+@TEST-END-FILE
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+event delayed_notice()
+	{
+	if ( Cluster::node == "worker-1" )
+		NOTICE([$note=Test_Notice, $msg="test notice!"]);
+	}
+
+event bro_init()
+	{
+	schedule 1secs { delayed_notice() };
+	}

testing/btest/scripts/base/frameworks/notice/suppression-cluster.bro

@@ -0,0 +1,35 @@
+# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run proxy-1   BROPATH=$BROPATH:.. CLUSTER_NODE=proxy-1 bro %INPUT
+# @TEST-EXEC: sleep 1
+# @TEST-EXEC: btest-bg-run worker-1  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT
+# @TEST-EXEC: btest-bg-run worker-2  BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT
+# @TEST-EXEC: btest-bg-wait -k 5
+# @TEST-EXEC: btest-diff manager-1/notice.log
+
+@TEST-START-FILE cluster-layout.bro
+redef Cluster::nodes = {
+	["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=27757/tcp, $workers=set("worker-1", "worker-2")],
+	["proxy-1"] = [$node_type=Cluster::PROXY,     $ip=127.0.0.1, $p=27758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")],
+	["worker-1"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=27760/tcp, $manager="manager-1", $proxy="proxy-1"],
+	["worker-2"] = [$node_type=Cluster::WORKER,   $ip=127.0.0.1, $p=27761/tcp, $manager="manager-1", $proxy="proxy-1"],
+};
+@TEST-END-FILE
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+event delayed_notice()
+	{
+	NOTICE([$note=Test_Notice,
+	        $msg="test notice!",
+	        $identifier="this identifier is static"]);
+	}
+
+event bro_init() &priority=5
+	{
+	if ( Cluster::node == "worker-1" )
+		schedule 4secs { delayed_notice() };
+	if ( Cluster::node == "worker-2" )
+		schedule 1secs { delayed_notice() };
+	}

testing/btest/scripts/base/frameworks/notice/suppression-disable.bro

@@ -0,0 +1,16 @@
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+@load base/frameworks/notice
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+redef Notice::not_suppressed_types += { Test_Notice };
+
+event bro_init()
+	{
+	NOTICE([$note=Test_Notice, $msg="test", $identifier="static"]);
+	NOTICE([$note=Test_Notice, $msg="another test", $identifier="static"]);
+	}

testing/btest/scripts/base/frameworks/notice/suppression.bro

@@ -0,0 +1,23 @@
+# @TEST-EXEC: bro -b %INPUT
+# @TEST-EXEC: btest-diff notice.log
+
+@load base/frameworks/notice
+
+redef enum Notice::Type += {
+	Test_Notice,
+};
+
+# The second notice needs to be scheduled due to how the notice framework
+# uses the event queue.
+
+event second_notice()
+	{
+	NOTICE([$note=Test_Notice, $msg="another test", $identifier="static"]);
+	}
+
+event bro_init()
+	{
+	NOTICE([$note=Test_Notice, $msg="test", $identifier="static"]);
+	schedule 1msec { second_notice() };
+	}
+

testing/btest/scripts/base/init-default-coverage.bro

@@ -5,7 +5,7 @@
 #@TEST-EXEC: test -e $DIST/scripts/base/init-default.bro
 #@TEST-EXEC: ( cd $DIST/scripts/base && find . -name '*.bro' ) | sort >"all scripts found"
 #@TEST-EXEC: bro misc/loaded-scripts
-#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
+#@TEST-EXEC: cat loaded_scripts.log | egrep -v '/build/|/loaded-scripts.bro|#' | awk 'NR>1{print $2}' | sed 's#/./#/#g'  >loaded_scripts.log.tmp
 #@TEST-EXEC: cat loaded_scripts.log.tmp | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 #@TEST-EXEC: cat loaded_scripts.log.tmp | sed "s#`cat prefix`#./#g" | sort >init-default.bro
 #@TEST-EXEC: diff -u "all scripts found" init-default.bro 1>&2

testing/btest/scripts/policy/misc/bare-loaded-scripts.test

@@ -7,6 +7,6 @@
 
 # @TEST-EXEC: bro -b misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/scripts/policy/misc/default-loaded-scripts.test

@@ -7,6 +7,6 @@
 
 # @TEST-EXEC: bro misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
-# @TEST-EXEC: cat loaded_scripts.log | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
+# @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>1{print $2}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat prefix`##g" >canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log