Merge branch 'master' into fastpath

doc/scripts/CMakeLists.txt

@@ -119,7 +119,7 @@ macro(REST_TARGET srcDir broInput)
         ARGS -E remove_directory .state
         # generate the reST documentation using bro
         COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
-        ARGS --doc-scripts ${broInput} || (rm -rf .state *.log *.rst && exit 1)
+        ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
         # move generated doc into a new directory tree that
         # defines the final structure of documents
         COMMAND "${CMAKE_COMMAND}"

doc/scripts/DocSourcesList.cmake

@@ -16,13 +16,13 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)
 
-rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@@ -34,7 +34,9 @@ rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
+rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -64,11 +66,11 @@ rest_target(${psd} base/protocols/http/partial-content.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/mime/base.bro)
 rest_target(${psd} base/protocols/mime/file-extract.bro)
 rest_target(${psd} base/protocols/mime/file-hash.bro)
 rest_target(${psd} base/protocols/mime/file-ident.bro)
-rest_target(${psd} base/protocols/rpc/base.bro)
+rest_target(${psd} base/protocols/mime/main.bro)
+rest_target(${psd} base/protocols/rpc/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
@@ -99,8 +101,7 @@ rest_target(${psd} policy/frameworks/metrics/http-example.bro)
 rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/base.bro)
+rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/event.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)

doc/scripts/genDocSourcesList.sh

@@ -68,12 +68,12 @@ sourcedir=${thisdir}/../..
 
 echo "$statictext" > $outfile
 
-bifs=`( cd ${sourcedir}/build/src && find . -name \*\.bro | sort )`
+bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
 
 for file in $bifs
 do
-    f=${file:2}
+    f=${file:2}.bro
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src $f)" >> $outfile
+    echo "rest_target(\${CMAKE_BINARY_DIR}/src/base $f)" >> $outfile
 done
 
 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

scripts/base/frameworks/cluster/__load__.bro

@@ -1,7 +1,7 @@
 # Load the core cluster support.
 @load ./main
 
-@if ( Cluster::node != "" )
+@if ( Cluster::is_enabled() )
 
 # Give the node being started up it's peer name.
 redef peer_description = Cluster::node;
@@ -26,15 +26,15 @@ redef peer_description = Cluster::node;
 ## Set the port that this node is supposed to listen on.
 redef Communication::listen_port_clear = Cluster::nodes[Cluster::node]$p;
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::MANAGER )
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
 @load ./nodes/manager
 @endif
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::PROXY )
+@if ( Cluster::local_node_type() == Cluster::PROXY )
 @load ./nodes/proxy
 @endif
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::WORKER )
+@if ( Cluster::local_node_type() == Cluster::WORKER )
 @load ./nodes/worker
 @endif
 

scripts/base/frameworks/cluster/main.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/control
 
 module Cluster;
 
@@ -9,6 +10,7 @@ export {
 	} &log;
 	
 	type NodeType: enum {
+		NONE,
 		CONTROL,
 		MANAGER,
 		PROXY,
@@ -53,8 +55,8 @@ export {
 	
 	## This function can be called at any time to determine what type of
 	## cluster node the current Bro instance is going to be acting as.
-	## :bro:id:`is_enabled` should be called first to find out if this is
+	## If :bro:id:`Cluster::is_enabled` returns false, then
-	## actually going to be a cluster node.
+	## :bro:enum:`Cluster::NONE` is returned.
 	global local_node_type: function(): NodeType;
 	
 	## This gives the value for the number of workers currently connected to,
@@ -80,15 +82,15 @@ function is_enabled(): bool
 
 function local_node_type(): NodeType
 	{
-	return nodes[node]$node_type;
+	return is_enabled() ? nodes[node]$node_type : NONE;
 	}
 
-
 event remote_connection_handshake_done(p: event_peer)
 	{
 	if ( nodes[p$descr]$node_type == WORKER )
 		++worker_count;
 	}
+
 event remote_connection_closed(p: event_peer)
 	{
 	if ( nodes[p$descr]$node_type == WORKER )
@@ -100,8 +102,7 @@ event bro_init() &priority=5
 	# If a node is given, but it's an unknown name we need to fail.
 	if ( node != "" && node !in nodes )
 		{
-		local msg = "You didn't supply a valid node in the Cluster::nodes configuration.";
+		Reporter::error(fmt("'%s' is not a valid node in the Cluster::nodes configuration", node));
-		event reporter_error(current_time(), msg, "");
 		terminate();
 		}
 	

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -8,6 +8,8 @@
 ##! This is where the cluster manager sets it's specific settings for other
 ##! frameworks and in the core.
 
+@load base/frameworks/notice
+
 @prefixes += cluster-manager
 
 # Load the script for local site configuration for the manager node.

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/notice
 
 @prefixes += cluster-worker
 

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,5 +1,7 @@
 @load ./main
-@load base/frameworks/communication/main
+@load base/frameworks/communication
+
+@if ( Cluster::node in Cluster::nodes )
 
 module Cluster;
 
@@ -79,3 +81,5 @@ event bro_init() &priority=9
 			}
 		}
 	}
+
+@endif

scripts/base/frameworks/communication/main.bro

@@ -1,6 +1,8 @@
 ##! Connect to remote Bro or Broccoli instances to share state and/or transfer
 ##! events.
 
+@load base/frameworks/packet-filter
+
 module Communication;
 
 export {

scripts/base/frameworks/intel/main.bro

@@ -20,6 +20,8 @@
 #   canary
 #   friend
 
+@load base/frameworks/notice
+
 module Intel;
 
 export {

scripts/base/frameworks/logging/main.bro

@@ -159,7 +159,7 @@ export {
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
-@load logging.bif.bro # Needs Filter and Stream defined.
+@load base/logging.bif # Needs Filter and Stream defined.
 
 module Log;
 

scripts/base/frameworks/logging/postprocessors/scp.bro

@@ -0,0 +1,42 @@
+##! This script defines a postprocessing function that can be applied
+##! to a logging filter in order to automatically SCP (secure copy)
+##! a log stream (or a subset of it) to a remote host at configurable
+##! rotation time intervals.
+
+module Log;
+
+export {
+	## This postprocessor SCP's the rotated-log to all the remote hosts
+	## defined in :bro:id:`Log::scp_destinations` and then deletes
+	## the local copy of the rotated-log.  It's not active when
+	## reading from trace files.
+	global scp_postprocessor: function(info: Log::RotationInfo): bool;
+
+	## A container that describes the remote destination for the SCP command
+	## argument as ``user@host:path``.
+	type SCPDestination: record {
+		user: string;
+		host: string;
+		path: string;
+	};
+
+	## A table indexed by a particular log writer and filter path, that yields
+	## a set remote destinations.  The :bro:id:`Log::scp_postprocessor`
+	## function queries this table upon log rotation and performs a secure
+	## copy of the rotated-log to each destination in the set.
+	global scp_destinations: table[Writer, string] of set[SCPDestination];
+}
+
+function scp_postprocessor(info: Log::RotationInfo): bool
+	{
+	if ( reading_traces() || [info$writer, info$path] !in scp_destinations )
+		return T;
+
+	local command = "";
+	for ( d in scp_destinations[info$writer, info$path] )
+		command += fmt("scp %s %s@%s:%s;", info$fname, d$user, d$host, d$path);
+
+	command += fmt("/bin/rm %s", info$fname);
+	system(command);
+	return T;
+	}

scripts/base/frameworks/metrics/cluster.bro

@@ -8,6 +8,7 @@
 ##! to be an internal implementation detail.
 
 @load base/frameworks/cluster
+@load ./main
 
 module Metrics;
 

scripts/base/frameworks/metrics/non-cluster.bro

@@ -1,3 +1,4 @@
+@load ./main
 
 module Metrics;
 

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -4,6 +4,10 @@
 ##! probably a safe assumption to make in most cases.  If both addresses
 ##! are remote, it will use the $src address.
 
+@load ../main
+@load base/frameworks/notice
+@load base/utils/site
+
 module Notice;
 
 export {

scripts/base/frameworks/notice/actions/drop.bro

@@ -1,6 +1,8 @@
 ##! This script extends the built in notice code to implement the IP address
 ##! dropping functionality.
 
+@load ../main
+
 module Notice;
 
 export {

scripts/base/frameworks/notice/actions/email_admin.bro

@@ -1,3 +1,6 @@
+@load ../main
+@load base/utils/site
+
 module Notice;
 
 export {

scripts/base/frameworks/notice/actions/page.bro

@@ -1,3 +1,4 @@
+@load ../main
 
 module Notice;
 

scripts/base/frameworks/notice/extend-email/hostnames.bro

@@ -1,3 +1,4 @@
+@load ../main
 
 module Notice;
 

scripts/base/frameworks/notice/main.bro

@@ -41,7 +41,9 @@ export {
 		## Indicates that the notice should be sent to the email address(es) 
 		## configured in the :bro:id:`Notice::mail_dest` variable.
 		ACTION_EMAIL,
-		## Indicates that the notice should be alarmed.
+		## Indicates that the notice should be alarmed.  A readable ASCII
+		## version of the alarm log is emailed in bulk to the address(es)
+		## configured in :bro:id:`Notice::mail_dest`.
 		ACTION_ALARM,
 	};
 	
@@ -136,7 +138,8 @@ export {
 	
 	## Local system sendmail program.
 	const sendmail            = "/usr/sbin/sendmail" &redef;
-	## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action.
+	## Email address to send notices with the :bro:enum:`ACTION_EMAIL` action
+	## or to send bulk alarm logs on rotation with :bro:enum:`ACTION_ALARM`.
 	const mail_dest           = ""                   &redef;
 	
 	## Address that emails will be from.
@@ -146,6 +149,11 @@ export {
 	## Text string prefixed to the subject of all emails sent out.
 	const mail_subject_prefix = "[Bro]" &redef;
 
+	## A log postprocessing function that implements emailing the contents
+	## of a log upon rotation to any configured :bro:id:`Notice::mail_dest`.
+	## The rotated log is removed upon being sent.
+	global log_mailing_postprocessor: function(info: Log::RotationInfo): bool;
+
 	## This is the event that is called as the entry point to the 
 	## notice framework by the global :bro:id:`NOTICE` function.  By the time 
 	## this event is generated, default values have already been filled out in
@@ -172,6 +180,13 @@ export {
 	## :bro:enum:`ACTION_PAGE` actions.
 	global email_notice_to: function(n: Info, dest: string, extend: bool);
 
+	## Constructs mail headers to which an email body can be appended for
+	## sending with sendmail.
+	## subject_desc: a subject string to use for the mail
+	## dest: recipient string to use for the mail
+	## Returns: a string of mail headers to which an email body can be appended
+	global email_headers: function(subject_desc: string, dest: string): string;
+
 	## This is an internally used function, please ignore it.  It's only used
 	## for filling out missing details of :bro:type:`Notice:Info` records
 	## before the synchronous and asynchronous event pathways have begun.
@@ -186,21 +201,47 @@ export {
 # priority.
 global ordered_policy: vector of PolicyItem = vector();
 
+function log_mailing_postprocessor(info: Log::RotationInfo): bool
+	{
+	if ( ! reading_traces() && mail_dest != "" )
+		{
+		local headers = email_headers(fmt("Log Contents: %s", info$fname),
+		                              mail_dest);
+		local tmpfilename = fmt("%s.mailheaders.tmp", info$fname);
+		local tmpfile = open(tmpfilename);
+		write_file(tmpfile, headers);
+		close(tmpfile);
+		system(fmt("/bin/cat %s %s | %s -t -oi && /bin/rm %s %s",
+		       tmpfilename, info$fname, sendmail, tmpfilename, info$fname));
+		}
+	return T;
+	}
+
+# This extra export section here is just because this redefinition should
+# be documented as part of the "public API" of this script, but the redef
+# needs to occur after the postprocessor function implementation.
+export {
+	## By default, an ASCII version of the the alarm log is emailed daily to any
+	## configured :bro:id:`Notice::mail_dest` if not operating on trace files.
+	redef Log::rotation_control += {
+		[Log::WRITER_ASCII, "alarm-mail"] =
+		    [$interv=24hrs, $postprocessor=log_mailing_postprocessor]
+	};
+}
+
 event bro_init()
 	{
 	Log::create_stream(NOTICE_POLICY, [$columns=PolicyItem]);
 	Log::create_stream(Notice::NOTICE, [$columns=Info, $ev=log_notice]);
 	
 	Log::create_stream(ALARM, [$columns=Notice::Info]);
-	# Make sure that this log is output as text so that it can be packaged
+	# If Bro is configured for mailing notices, set up mailing for alarms.
-	# up and emailed later.
+	# Make sure that this alarm log is also output as text so that it can
-	Log::add_filter(ALARM, [$name="default", $writer=Log::WRITER_ASCII]);
+	# be packaged up and emailed later.
+	if ( ! reading_traces() && mail_dest != "" )
+		Log::add_filter(ALARM, [$name="alarm-mail", $path="alarm-mail",
+		                        $writer=Log::WRITER_ASCII]);
 	}
-	# TODO: need a way to call a Bro script level callback during file rotation.
-	#       we need more than a just $postprocessor.
-	#redef Log::rotation_control += {
-	#	[Log::WRITER_ASCII, "alarm"] = [$postprocessor="mail-alarms"];
-	#};
 
 # TODO: fix this.
 #function notice_tags(n: Notice::Info) : table[string] of string
@@ -220,20 +261,24 @@ event bro_init()
 #	return tgs;
 #	}
 
+function email_headers(subject_desc: string, dest: string): string
+	{
+	local header_text = string_cat(
+		"From: ", mail_from, "\n",
+		"Subject: ", mail_subject_prefix, " ", subject_desc, "\n",
+		"To: ", dest, "\n",
+		"User-Agent: Bro-IDS/", bro_version(), "\n");
+	if ( reply_to != "" )
+		header_text = string_cat(header_text, "Reply-To: ", reply_to, "\n");
+	return header_text;
+	}
+
 function email_notice_to(n: Notice::Info, dest: string, extend: bool)
 	{
 	if ( reading_traces() || dest == "" )
 		return;
 		
-	local email_text = string_cat(
+	local email_text = email_headers(fmt("%s", n$note), dest);
-		"From: ", mail_from, "\n",
-		"Subject: ", mail_subject_prefix, " ", fmt("%s", n$note), "\n",
-		"To: ", dest, "\n",
-		# TODO: BiF to get version (the resource_usage Bif seems like overkill).
-		"User-Agent: Bro-IDS/?.?.?\n"); 
-	
-	if ( reply_to != "" )
-		email_text = string_cat(email_text, "Reply-To: ", reply_to, "\n");
 	
 	# The notice emails always start off with the human readable message.
 	email_text = string_cat(email_text, "\n", n$msg, "\n");

scripts/base/frameworks/notice/weird.bro

@@ -1,3 +1,7 @@
+@load base/utils/conn-ids
+@load base/utils/site
+@load ./main
+
 module Weird;
 
 export {

scripts/base/frameworks/packet-filter/main.bro

@@ -4,6 +4,8 @@
 ##! open filter and all filters defined in Bro scripts with the
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
 
+@load base/frameworks/notice
+
 module PacketFilter;
 
 export {

scripts/base/frameworks/packet-filter/netstats.bro

@@ -1,5 +1,7 @@
 ##! This script reports on packet loss from the various packet sources.
 
+@load base/frameworks/notice
+
 module PacketFilter;
 
 export {

scripts/base/frameworks/signatures/main.bro

@@ -1,5 +1,7 @@
 ##! Script level signature support.
 
+@load base/frameworks/notice
+
 module Signatures;
 
 export {

scripts/base/frameworks/software/main.bro

@@ -4,6 +4,9 @@
 ##! that they analyze.  The entry point for providing new software detections
 ##! to this framework is through the :bro:id:`Software::found` function.
 
+@load base/utils/directions-and-hosts
+@load base/utils/numbers
+
 module Software;
 
 export {

scripts/base/protocols/conn/contents.bro

@@ -8,6 +8,8 @@
 ##! This script does not work well in a cluster context unless it has a 
 ##! remotely mounted disk to write the content files to.
 
+@load base/utils/files
+
 module Conn;
 
 export {

scripts/base/protocols/conn/main.bro

@@ -1,3 +1,4 @@
+@load base/utils/site
 
 module Conn;
 
@@ -12,7 +13,11 @@ export {
 		proto:        transport_proto &log;
 		service:      string          &log &optional;
 		duration:     interval        &log &optional;
+		## The number of payload bytes the originator sent. For TCP
+		## this is taken from sequence numbers and might be inaccurate 
+		## (e.g., due to large connections)
 		orig_bytes:   count           &log &optional;
+		## The number of payload bytes the responder sent. See ``orig_bytes``.
 		resp_bytes:   count           &log &optional;
 
 		## ==========   ===============================================
@@ -68,6 +73,17 @@ export {
 		## for instance. I.e., we just record that data went in that direction.
 		## This history is not meant to encode how much data that happened to be.
 		history:      string          &log &optional;
+		## Number of packets the originator sent.
+		## Only set if :bro:id:`use_conn_size_analyzer`=T
+		orig_pkts:     count      &log &optional;
+		## Number IP level bytes the originator sent (as seen on the wire,
+		## taken from IP total_length header field).
+		## Only set if :bro:id:`use_conn_size_analyzer`=T
+		orig_ip_bytes: count      &log &optional;
+		## Number of packets the responder sent. See ``orig_pkts``.
+		resp_pkts:     count      &log &optional;
+		## Number IP level bytes the responder sent. See ``orig_pkts``.
+		resp_ip_bytes: count      &log &optional;
 	};
 	
 	global log_conn: event(rec: Info);
@@ -143,31 +159,39 @@ function determine_service(c: connection): string
 	return to_lower(service);
 	}
 
+## Fill out the c$conn record for logging
 function set_conn(c: connection, eoc: bool)
 	{
 	if ( ! c?$conn )
 		{
-		local id = c$id;
 		local tmp: Info;
-		tmp$ts=c$start_time;
-		tmp$uid=c$uid;
-		tmp$id=id;
-		tmp$proto=get_port_transport_proto(id$resp_p);
-		if( |Site::local_nets| > 0 )
-			tmp$local_orig=Site::is_local_addr(id$orig_h);
 		c$conn = tmp;
 		}
 
+	c$conn$ts=c$start_time;
+	c$conn$uid=c$uid;
+	c$conn$id=c$id;
+	c$conn$proto=get_port_transport_proto(c$id$resp_p);
+	if( |Site::local_nets| > 0 )
+		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
+	
 	if ( eoc )
 		{
 		if ( c$duration > 0secs ) 
 			{
 			c$conn$duration=c$duration;
-			# TODO: these should optionally use Gregor's new
-			#       actual byte counting code if it's enabled.
 			c$conn$orig_bytes=c$orig$size;
 			c$conn$resp_bytes=c$resp$size;
 			}
+		if ( c$orig?$num_pkts )
+			{
+			# these are set if use_conn_size_analyzer=T
+			# we can have counts in here even without duration>0
+			c$conn$orig_pkts = c$orig$num_pkts;
+			c$conn$orig_ip_bytes = c$orig$num_bytes_ip;
+			c$conn$resp_pkts = c$resp$num_pkts;
+			c$conn$resp_ip_bytes = c$resp$num_bytes_ip;
+			}
 		local service = determine_service(c);
 		if ( service != "" ) 
 			c$conn$service=service;
@@ -178,11 +202,6 @@ function set_conn(c: connection, eoc: bool)
 		}
 	}
 
-event connection_established(c: connection) &priority=5
-	{
-	set_conn(c, F);
-	}
-	
 event content_gap(c: connection, is_orig: bool, seq: count, length: count) &priority=5
 	{
 	set_conn(c, F);
@@ -190,9 +209,13 @@ event content_gap(c: connection, is_orig: bool, seq: count, length: count) &prio
 	c$conn$missed_bytes = c$conn$missed_bytes + length;
 	}
 	
-event connection_state_remove(c: connection) &priority=-5
+event connection_state_remove(c: connection) &priority=5
 	{
 	set_conn(c, T);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
 	Log::write(CONN, c$conn);
 	}
 

scripts/base/protocols/dns/main.bro

@@ -1,3 +1,4 @@
+@load ./consts
 
 module DNS;
 

scripts/base/protocols/ftp/file-extract.bro

@@ -1,5 +1,8 @@
 ##! File extraction for FTP.
 
+@load ./main
+@load base/utils/files
+
 module FTP;
 
 export {

scripts/base/protocols/ftp/main.bro

@@ -7,6 +7,10 @@
 ##!
 ##! * Handle encrypted sessions correctly (get an example?)
 
+@load ./utils-commands
+@load base/utils/paths
+@load base/utils/numbers
+
 module FTP;
 
 export {

scripts/base/protocols/http/file-extract.bro

@@ -1,6 +1,10 @@
 ##! Extracts the items from HTTP traffic, one per file.  At this time only 
 ##! the message body from the server can be extracted with this script.
 
+@load ./main
+@load ./file-ident
+@load base/utils/files
+
 module HTTP;
 
 export {

scripts/base/protocols/http/file-hash.bro

@@ -1,5 +1,7 @@
 ##! Calculate hashes for HTTP body transfers.
 
+@load ./file-ident
+
 module HTTP;
 
 export {

scripts/base/protocols/http/file-ident.bro

@@ -1,6 +1,11 @@
 ##! This script is involved in the identification of file types in HTTP
 ##! response bodies.
 
+@load base/frameworks/signatures
+@load base/frameworks/notice
+@load ./main
+@load ./utils
+
 # Add the magic number signatures to the core signature set.
 redef signature_files += "base/protocols/http/file-ident.sig";
 # Ignore the signatures used to match files

scripts/base/protocols/http/main.bro

@@ -1,3 +1,5 @@
+@load base/utils/numbers
+@load base/utils/files
 
 module HTTP;
 

scripts/base/protocols/http/partial-content.bro

@@ -3,6 +3,10 @@
 ##!
 ##! This script doesn't work yet and isn't loaded by default.
 
+@load base/frameworks/notice
+@load ./main
+@load ./utils
+
 module HTTP;
 
 export {

scripts/base/protocols/http/utils.bro

@@ -1,5 +1,7 @@
 ##! Utilities specific for HTTP processing.
 
+@load ./main
+
 module HTTP;
 
 export {

scripts/base/protocols/irc/dcc-send.bro

@@ -8,6 +8,9 @@
 ##! Example line from IRC server indicating that the DCC SEND is about to start:
 ##!    PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
 
+@load ./main
+@load base/utils/files
+
 module IRC;
 
 export {

scripts/base/protocols/mime/__load__.bro

@@ -1,4 +1,4 @@
-@load protocols/mime/base
+@load ./main
-@load protocols/mime/file-ident
+@load ./file-ident
-@load protocols/mime/file-extract
+@load ./file-extract
-@load protocols/mime/file-hash
+@load ./file-hash

scripts/base/protocols/mime/file-hash.bro

@@ -1,4 +1,5 @@
 @load ./file-ident
+@load base/frameworks/notice
 
 module MIME;
 

scripts/base/protocols/mime/file-ident.bro

@@ -1,4 +1,4 @@
-@load ./base
+@load ./main
 
 module MIME;
 

scripts/base/protocols/rpc/main.bro

@@ -8,6 +8,8 @@
 # programs for which we don't have an analyzer. 
 #
 
+@load base/utils/conn-ids
+
 module RPC;
 
 export {

scripts/base/protocols/smtp/main.bro

@@ -1,3 +1,6 @@
+@load base/frameworks/notice
+@load base/utils/addrs
+@load base/utils/directions-and-hosts
 
 module SMTP;
 

scripts/base/protocols/ssh/main.bro

@@ -5,6 +5,12 @@
 ##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
 ##! is not attempted if the connection size analyzer isn't enabled.
 
+@load base/frameworks/notice
+@load base/utils/site
+@load base/utils/thresholds
+@load base/utils/conn-ids
+@load base/utils/directions-and-hosts
+
 module SSH;
 
 export {
@@ -94,6 +100,11 @@ function check_ssh_connection(c: connection, done: bool)
 	if ( c$ssh$done )
 		return;
 	
+	# Make sure conn_size_analyzer is active by checking 
+	# resp$num_bytes_ip
+	if ( !c$resp?$num_bytes_ip )
+		return;
+	
 	# If this is still a live connection and the byte count has not
 	# crossed the threshold, just return and let the resheduled check happen later.
 	if ( !done && c$resp$num_bytes_ip < authentication_data_size )

scripts/base/protocols/ssl/main.bro

@@ -1,3 +1,5 @@
+@load ./consts
+@load base/frameworks/notice
 
 module SSL;
 

scripts/base/protocols/ssl/mozilla-ca-list.bro

@@ -1,6 +1,6 @@
 # Don't edit!  This file is automatically generated.
 # Generated at: Wed Jun 29 07:52:38 -0400 2011
-
+@load base/protocols/ssl
 module SSL;
 redef root_certs += {
 	["GTE CyberTrust Global Root"] = "\x30\x82\x02\x5A\x30\x82\x01\xC3\x02\x02\x01\xA5\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x30\x75\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x47\x54\x45\x20\x43\x6F\x72\x70\x6F\x72\x61\x74\x69\x6F\x6E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x53\x6F\x6C\x75\x74\x69\x6F\x6E\x73\x2C\x20\x49\x6E\x63\x2E\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x1E\x17\x0D\x39\x38\x30\x38\x31\x33\x30\x30\x32\x39\x30\x30\x5A\x17\x0D\x31\x38\x30\x38\x31\x33\x32\x33\x35\x39\x30\x30\x5A\x30\x75\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x47\x54\x45\x20\x43\x6F\x72\x70\x6F\x72\x61\x74\x69\x6F\x6E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x53\x6F\x6C\x75\x74\x69\x6F\x6E\x73\x2C\x20\x49\x6E\x63\x2E\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\x95\x0F\xA0\xB6\xF0\x50\x9C\xE8\x7A\xC7\x88\xCD\xDD\x17\x0E\x2E\xB0\x94\xD0\x1B\x3D\x0E\xF6\x94\xC0\x8A\x94\xC7\x06\xC8\x90\x97\xC8\xB8\x64\x1A\x7A\x7E\x6C\x3C\x53\xE1\x37\x28\x73\x60\x7F\xB2\x97\x53\x07\x9F\x53\xF9\x6D\x58\x94\xD2\xAF\x8D\x6D\x88\x67\x80\xE6\xED\xB2\x95\xCF\x72\x31\xCA\xA5\x1C\x72\xBA\x5C\x02\xE7\x64\x42\xE7\xF9\xA9\x2C\xD6\x3A\x0D\xAC\x8D\x42\xAA\x24\x01\x39\xE6\x9C\x3F\x01\x85\x57\x0D\x58\x87\x45\xF8\xD3\x85\xAA\x93\x69\x26\x85\x70\x48\x80\x3F\x12\x15\xC7\x79\xB4\x1F\x05\x2F\x3B\x62\x99\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x03\x81\x81\x00\x6D\xEB\x1B\x09\xE9\x5E\xD9\x51\xDB\x67\x22\x61\xA4\x2A\x3C\x48\x77\xE3\xA0\x7C\xA6\xDE\x73\xA2\x14\x03\x85\x3D\xFB\xAB\x0E\x30\xC5\x83\x16\x33\x81\x13\x08\x9E\x7B\x34\x4E\xDF\x40\xC8\x74\xD7\xB9\x7D\xDC\xF4\x76\x55\x7D\x9B\x63\x54\x18\xE9\xF0\xEA\xF3\x5C\xB1\xD9\x8B\x42\x1E\xB9\xC0\x95\x4E\xBA\xFA\xD5\xE2\x7C\xF5\x68\x61\xBF\x8E\xEC\x05\x97\x5F\x5B\xB0\xD7\xA3\x85\x34\xC4\x24\xA7\x0D\x0F\x95\x93\xEF\xCB\x94\xD8\x9E\x1F\x9D\x5C\x85\x6D\xC7\xAA\xAE\x4F\x1F\x22\xB5\xCD\x95\xAD\xBA\xA7\xCC\xF9\xAB\x0B\x7A\x7F",

scripts/base/utils/directions-and-hosts.bro

@@ -1,3 +1,4 @@
+@load ./site
 
 type Direction: enum {
 	## The connection originator is not within the locally-monitored network,

scripts/policy/frameworks/control/controllee.bro

@@ -1,4 +1,4 @@
-
+@load base/frameworks/control
 # If an instance is a controllee, it implicitly needs to listen for remote
 # connections.
 @load frameworks/communication/listen-clear

scripts/policy/frameworks/control/controller.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/control
 @load base/frameworks/communication
 
 module Control;

scripts/policy/frameworks/dpd/detect-protocols.bro

@@ -1,5 +1,9 @@
 ##! Finds connections with protocols on non-standard ports with DPD.
 
+@load base/frameworks/notice
+@load base/utils/site
+@load base/utils/conn-ids
+
 module ProtocolDetector;
 
 export {

scripts/policy/frameworks/dpd/packet-segment-logging.bro

@@ -4,6 +4,8 @@
 ##! A caveat to logging packet data is that in some cases, the packet may
 ##! not be the packet that actually caused the protocol violation.
 
+@load base/frameworks/dpd
+
 module DPD;
 
 export {

scripts/policy/frameworks/metrics/conn-example.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/metrics
+@load base/utils/site
 
 redef enum Metrics::ID += { 
 	CONNS_ORIGINATED, 

scripts/policy/frameworks/metrics/http-example.bro

@@ -1,3 +1,6 @@
+@load base/frameworks/metrics
+@load base/protocols/http
+@load base/utils/site
 
 redef enum Metrics::ID += {
 	HTTP_REQUESTS_BY_STATUS_CODE,
@@ -20,7 +23,7 @@ event bro_init()
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if ( rec?$host )
-		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host]);
+		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host], 1);
 	if ( rec?$status_code )
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)]);
+		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], 1);
 	}

scripts/policy/frameworks/metrics/ssl-example.bro

@@ -1,4 +1,5 @@
-
+@load base/frameworks/metrics
+@load base/protocols/ssl
 
 redef enum Metrics::ID += {
 	SSL_SERVERNAME,

scripts/policy/frameworks/software/version-changes.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice
+@load base/frameworks/software
 
 module Software;
 

scripts/policy/frameworks/software/vulnerable.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice
+@load base/frameworks/software
 
 module Software;
 

scripts/policy/integration/barnyard2/__load__.bro

@@ -1,3 +1,2 @@
-@load integration/barnyard2/types
+@load ./types
-@load integration/barnyard2/event
+@load ./main
-@load integration/barnyard2/base

scripts/policy/integration/barnyard2/event.bro

@@ -1,3 +0,0 @@
-## This is the event that Barnyard2 instances will send if they're 
-## configured with the bro_alert output plugin.
-global barnyard_alert: event(id: Barnyard2::PacketID, alert: Barnyard2::AlertData, msg: string, data: string);

scripts/policy/integration/barnyard2/main.bro

@@ -2,7 +2,7 @@
 ##! Barnyard2 and logs them.  In the future it will do more correlation
 ##! and derive new notices from the alerts.
 
-@load integration/barnyard2/types
+@load ./types
 
 module Barnyard2;
 

scripts/policy/integration/barnyard2/types.bro

@@ -22,4 +22,11 @@ export {
 		dst_ip: addr;
 		dst_p: port;
 	} &log;
+
+	## This is the event that Barnyard2 instances will send if they're 
+	## configured with the bro_alert output plugin.
+	global barnyard_alert: event(id: Barnyard2::PacketID,
+	                             alert: Barnyard2::AlertData,
+	                             msg: string,
+	                             data: string);
 }

scripts/policy/protocols/conn/known-hosts.bro

@@ -3,6 +3,8 @@
 ##! output provides an easy way to determine a count of the IP addresses in
 ##! use on a network per day.
 
+@load base/utils/directions-and-hosts
+
 module KnownHosts;
 
 export {

scripts/policy/protocols/conn/known-services.bro

@@ -3,6 +3,8 @@
 ##! completed a TCP handshake with another host.  If a protocol is detected
 ##! during the session, the protocol will also be logged.
 
+@load base/utils/directions-and-hosts
+
 module KnownServices;
 
 redef enum Log::ID += { KNOWN_SERVICES };

scripts/policy/protocols/conn/scan.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/notice
+@load base/frameworks/notice/main
 @load port-name
 
 module Scan;

scripts/policy/protocols/dns/auth-addl.bro

@@ -1,3 +1,4 @@
+@load base/protocols/dns/main
 
 redef dns_skip_all_auth = F;
 redef dns_skip_all_addl = F;

scripts/policy/protocols/dns/detect-external-names.bro

@@ -8,6 +8,9 @@
 ##!     to be within a local zone.  :bro:id:`local_zones` variable **must**
 ##!     be set appropriately for this detection.
 
+@load base/frameworks/notice/main
+@load base/utils/site
+
 module DNS;
 
 export {

scripts/policy/protocols/ftp/detect.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice/main
+@load base/protocols/ftp/main
 
 module FTP;
 

scripts/policy/protocols/ftp/software.bro

@@ -6,6 +6,8 @@
 ##! * Detect client software with password given for anonymous users
 ##!   (e.g. cyberduck@example.net)
 
+@load base/frameworks/software/main
+
 module FTP;
 
 export {

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,9 +1,14 @@
 ##! This script takes MD5 sums of files transferred over HTTP and checks them with
 ##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
-##! documentation for the base/protocols/http/file-hash.bro script to see how to 
+##! documentation for the :doc:base/protocols/http/file-hash.bro script to see how to 
 ##! configure which transfers will have hashes calculated.
 
+@load base/frameworks/notice/main
+@load base/protocols/http/main
+@load base/protocols/http/utils
+@load base/protocols/http/file-hash
+
 export {
 	redef enum Notice::Type += { 
 		## If the MD5 sum of a file transferred over HTTP 

scripts/policy/protocols/http/detect-intel.bro

@@ -1,5 +1,9 @@
 ##! Intelligence based HTTP detections.
 
+@load base/protocols/http/main
+@load base/protocols/http/utils
+@load base/frameworks/intel/main
+
 module HTTP;
 
 event log_http(rec: Info)

scripts/policy/protocols/http/detect-sqli.bro

@@ -1,5 +1,9 @@
 ##! SQL injection detection in HTTP.
 
+@load base/frameworks/notice/main
+@load base/frameworks/metrics/main
+@load base/protocols/http/main
+
 module HTTP;
 
 export {

scripts/policy/protocols/http/detect-webapps.bro

@@ -1,3 +1,7 @@
+@load base/frameworks/signatures/main
+@load base/frameworks/software/main
+@load base/protocols/http/main
+@load base/protocols/http/utils
 
 module HTTP;
 

scripts/policy/protocols/http/headers.bro

@@ -1,5 +1,7 @@
 ##! Extract and include the header keys used for each request in the log.
 
+@load base/protocols/http/main
+
 module HTTP;
 
 export {

scripts/policy/protocols/http/software.bro

@@ -1,5 +1,7 @@
 ##! Software identification and extraction for HTTP traffic.
 
+@load base/frameworks/software/main
+
 module HTTP;
 
 export {

scripts/policy/protocols/http/var-extraction-cookies.bro

@@ -1,5 +1,8 @@
 ##! This script extracts and logs variables from cookies sent by clients
 
+@load base/protocols/http/main
+@load base/protocols/http/utils
+
 module HTTP;
 
 redef record Info += {

scripts/policy/protocols/http/var-extraction-uri.bro

@@ -1,5 +1,8 @@
 ##! This script extracts and logs variables from the requested URI
 
+@load base/protocols/http/main
+@load base/protocols/http/utils
+
 module HTTP;
 
 redef record Info += {

scripts/policy/protocols/smtp/detect-suspicious-orig.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice/main
+@load base/protocols/smtp/main
 
 module SMTP;
 

scripts/policy/protocols/smtp/software.bro

@@ -7,6 +7,9 @@
 ##! * Find some heuristic to determine if email was sent through 
 ##!   a MS Exhange webmail interface as opposed to a desktop client.
 
+@load base/frameworks/software/main
+@load base/protocols/smtp/main
+
 module SMTP;
 
 export {

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -1,4 +1,8 @@
 
+@load base/frameworks/metrics
+@load base/frameworks/notice
+@load base/frameworks/intel
+
 module SSH;
 
 export {

scripts/policy/protocols/ssh/geo-data.bro

@@ -1,6 +1,9 @@
 ##! This implements all of the additional information and geodata detections 
 ##! for SSH analysis.
 
+@load base/frameworks/notice/main
+@load base/protocols/ssh/main
+
 module SSH;
 
 export {

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/notice/main
 
 module SSH;
 

scripts/policy/protocols/ssh/software.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/software/main
 
 module SSH;
 

scripts/policy/protocols/ssl/known-certs.bro

@@ -1,3 +1,4 @@
+@load base/utils/directions-and-hosts
 
 module KnownCerts;
 

scripts/policy/protocols/ssl/validate-certs.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice/main
+@load base/protocols/ssl/main
 
 module SSL;
 

scripts/policy/tuning/defaults/remove-high-volume-notices.bro

@@ -1,6 +1,9 @@
 ##! This strives to tune out high volume and less useful data 
 ##! from the notice log.
 
+@load base/frameworks/notice
+@load base/frameworks/notice/weird
+
 # Remove these notices from logging since they can be too noisy.
 redef Notice::ignored_types += {
 	Weird::Content_Gap,

scripts/policy/tuning/defaults/warnings.bro

@@ -2,6 +2,8 @@
 ##! good to set in most cases or other things that could be done to achieve 
 ##! better detection.
 
+@load base/utils/site
+
 event bro_init() &priority=-10
 	{
 	if ( |Site::local_nets| == 0 )

scripts/policy/tuning/track-all-assets.bro

@@ -1,4 +1,4 @@
-
+@load base/frameworks/software
 @load protocols/conn/known-hosts
 @load protocols/conn/known-services
 @load protocols/ssl/known-certs

scripts/site/local-manager.bro

@@ -1,5 +1,7 @@
 ##! Local site policy loaded only by the manager in a cluster.
 
+@load base/frameworks/notice
+
 # If you are running a cluster you should define your Notice::policy here 
 # so that notice processing occurs on the manager.
 redef Notice::policy += {

scripts/site/local.bro

@@ -62,6 +62,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 # Uncomment this redef if you want to extract SMTP MIME entities for 
 # some file types.  The numbers given indicate how many bytes to extract for
 # the various mime types.
+@load base/protocols/smtp/entities-excerpt
 redef SMTP::entity_excerpt_len += {
 #	["text/plain"] = 1024,
 #	["text/html"] = 1024,

scripts/test-all-policy.bro

@@ -15,11 +15,13 @@
 # @load frameworks/control/controller.bro
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
+@load frameworks/metrics/conn-example.bro
+@load frameworks/metrics/http-example.bro
+@load frameworks/metrics/ssl-example.bro
 @load frameworks/software/version-changes.bro
 @load frameworks/software/vulnerable.bro
 @load integration/barnyard2/__load__.bro
-@load integration/barnyard2/base.bro
+@load integration/barnyard2/main.bro
-@load integration/barnyard2/event.bro
 @load integration/barnyard2/types.bro
 @load misc/analysis-groups.bro
 @load misc/loaded-scripts.bro

src/BroDoc.cc

@@ -60,7 +60,7 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 	if ( ! reST_file )
 		fprintf(stderr, "Failed to open %s\n", reST_filename.c_str());
 
-#ifdef DEBUG
+#ifdef DOCDEBUG
 	fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str());
 	fprintf(stdout, "\trelative dir: %s\n", rel.c_str());
 	fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str());

src/CMakeLists.txt

@@ -107,20 +107,28 @@ macro(BIF_TARGET bifInput)
     add_custom_command(OUTPUT ${bifOutputs}
                        COMMAND bifcl
                        ARGS ${CMAKE_CURRENT_SOURCE_DIR}/${bifInput} || (rm -f ${bifOutputs} && exit 1)
+                       # in order be able to run bro from the build directory,
+                       # the generated bro script needs to be inside a
+                       # a directory tree named the same way it will be
+                       # referenced from an @load
+                       COMMAND "${CMAKE_COMMAND}"
+                       ARGS -E copy ${bifInput}.bro base/${bifInput}.bro
+                       COMMAND "${CMAKE_COMMAND}"
+                       ARGS -E remove -f ${bifInput}.bro
                        DEPENDS ${bifInput}
                        DEPENDS bifcl
                        COMMENT "[BIFCL] Processing ${bifInput}"
     )
     list(APPEND ALL_BIF_OUTPUTS ${bifOutputs})
     list(APPEND INSTALL_BIF_OUTPUTS
-         ${CMAKE_CURRENT_BINARY_DIR}/${bifInput}.bro)
+         ${CMAKE_CURRENT_BINARY_DIR}/base/${bifInput}.bro)
 endmacro(BIF_TARGET)
 
 # returns a list of output files that bifcl will produce
 # for given input file in ${outputFileVar}
 macro(GET_BIF_OUTPUT_FILES inputFile outputFileVar)
     set(${outputFileVar}
-        ${inputFile}.bro
+        base/${inputFile}.bro
         ${inputFile}.func_def
         ${inputFile}.func_h
         ${inputFile}.func_init
@@ -424,7 +432,7 @@ set(brolibs
 target_link_libraries(bro ${brolibs})
 
 install(TARGETS bro DESTINATION bin)
-install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH})
+install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)
 
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)

src/SSL-binpac.cc

@@ -7,6 +7,7 @@ SSL_Analyzer_binpac::SSL_Analyzer_binpac(Connection* c)
 : TCP_ApplicationAnalyzer(AnalyzerTag::SSL, c)
 	{
 	interp = new binpac::SSL::SSL_Conn(this);
+	had_gap = false;
 	}
 
 SSL_Analyzer_binpac::~SSL_Analyzer_binpac()
@@ -36,12 +37,24 @@ void SSL_Analyzer_binpac::DeliverStream(int len, const u_char* data, bool orig)
 
 	if ( TCP()->IsPartial() )
 		return;
+	if ( had_gap )
+		// XXX: If only one side had a content gap, we could still try to
+		// deliver data to the other side if the script layer can handle this.
+		return; 
 
+	try
+		{
 		interp->NewData(orig, data, data + len);
 		}
+	catch ( binpac::Exception const &e )
+		{
+		ProtocolViolation(fmt("Binpac exception: %s", e.c_msg()));
+		}
+	}
 
 void SSL_Analyzer_binpac::Undelivered(int seq, int len, bool orig)
 	{
 	TCP_ApplicationAnalyzer::Undelivered(seq, len, orig);
+	had_gap = true;
 	interp->NewGap(orig, len);
 	}

src/SSL-binpac.h

@@ -30,6 +30,7 @@ public:
 
 protected:
 	binpac::SSL::SSL_Conn* interp;
+	bool had_gap;
 
 };
 

src/bro.bif

@@ -3607,3 +3607,9 @@ function enable_communication%(%): any
 	remote_serializer->Init();
 	return 0;
 	%}
+
+## Returns the Bro version string
+function bro_version%(%): string
+	%{
+	return new StringVal(bro_version());
+	%}

src/main.cc

@@ -932,9 +932,8 @@ int main(int argc, char** argv)
 
 	if ( dead_handlers->length() > 0 && check_for_unused_event_handlers )
 		{
-		reporter->Warning("event handlers never invoked:");
 		for ( int i = 0; i < dead_handlers->length(); ++i )
-			reporter->Warning("\t", (*dead_handlers)[i]);
+			reporter->Warning("event handler never invoked: %s", (*dead_handlers)[i]);
 		}
 
 	delete dead_handlers;

testing/btest/Baseline/core.check-unused-event-handlers/.stderr

@@ -0,0 +1 @@
+warning in <params>, line 1: event handler never invoked: this_is_never_used

testing/btest/Baseline/core.print-bpf-filters-ipv4/conn.log

@@ -1,2 +1,2 @@
-# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
-1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.73330307006836	98	9417	SF	-	0	ShADdFaf
+1128727435.450898	UWkUyAuUGXf	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.73330307006836	98	9417	SF	-	0	ShADdFaf	12	710	10	9945

testing/btest/Baseline/core.vlan-mpls/conn.log

@@ -1,4 +1,4 @@
-# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history
+# ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes
-952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.10255992412567	25	0	SH	-	0	-
+952109346.874907	UWkUyAuUGXf	10.1.2.1	11001	10.34.0.1	23	tcp	-	2.10255992412567	25	0	SH	-	0	-	11	280	0	0
-1128727435.450898	56gKBmhBBB6	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.73330307006836	98	9417	SF	-	0	ShADdFaf
+1128727435.450898	56gKBmhBBB6	141.42.64.125	56730	125.190.109.199	80	tcp	http	1.73330307006836	98	9417	SF	-	0	ShADdFaf	12	710	10	9945
-1278600802.069419	50da4BEzauh	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.00415205955505371	9	3429	SF	-	0	ShADadfF
+1278600802.069419	50da4BEzauh	10.20.80.1	50343	10.0.0.15	80	tcp	-	0.00415205955505371	9	3429	SF	-	0	ShADadfF	7	361	7	3801