Merge remote-tracking branch 'origin/topic/jsiwek/reorg-followup'

doc/scripts/CMakeLists.txt

@@ -119,7 +119,7 @@ macro(REST_TARGET srcDir broInput)
         ARGS -E remove_directory .state
         # generate the reST documentation using bro
         COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
-        ARGS --doc-scripts ${broInput} || (rm -rf .state *.log *.rst && exit 1)
+        ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
         # move generated doc into a new directory tree that
         # defines the final structure of documents
         COMMAND "${CMAKE_COMMAND}"

doc/scripts/DocSourcesList.cmake

@@ -16,13 +16,13 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)
 
-rest_target(${CMAKE_BINARY_DIR}/src bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src/base types.bif.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@@ -34,7 +34,9 @@ rest_target(${psd} base/frameworks/dpd/main.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
+rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -64,11 +66,11 @@ rest_target(${psd} base/protocols/http/partial-content.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/mime/base.bro)
 rest_target(${psd} base/protocols/mime/file-extract.bro)
 rest_target(${psd} base/protocols/mime/file-hash.bro)
 rest_target(${psd} base/protocols/mime/file-ident.bro)
-rest_target(${psd} base/protocols/rpc/base.bro)
+rest_target(${psd} base/protocols/mime/main.bro)
+rest_target(${psd} base/protocols/rpc/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
@@ -99,8 +101,7 @@ rest_target(${psd} policy/frameworks/metrics/http-example.bro)
 rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/base.bro)
+rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/event.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)

doc/scripts/genDocSourcesList.sh

@@ -68,12 +68,12 @@ sourcedir=${thisdir}/../..
 
 echo "$statictext" > $outfile
 
-bifs=`( cd ${sourcedir}/build/src && find . -name \*\.bro | sort )`
+bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
 
 for file in $bifs
 do
-    f=${file:2}
+    f=${file:2}.bro
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src $f)" >> $outfile
+    echo "rest_target(\${CMAKE_BINARY_DIR}/src/base $f)" >> $outfile
 done
 
 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

scripts/base/frameworks/cluster/__load__.bro

@@ -1,7 +1,7 @@
 # Load the core cluster support.
 @load ./main
 
-@if ( Cluster::node != "" )
+@if ( Cluster::is_enabled() )
 
 # Give the node being started up it's peer name.
 redef peer_description = Cluster::node;
@@ -26,15 +26,15 @@ redef peer_description = Cluster::node;
 ## Set the port that this node is supposed to listen on.
 redef Communication::listen_port_clear = Cluster::nodes[Cluster::node]$p;
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::MANAGER )
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
 @load ./nodes/manager
 @endif
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::PROXY )
+@if ( Cluster::local_node_type() == Cluster::PROXY )
 @load ./nodes/proxy
 @endif
 
-@if ( Cluster::nodes[Cluster::node]$node_type == Cluster::WORKER )
+@if ( Cluster::local_node_type() == Cluster::WORKER )
 @load ./nodes/worker
 @endif
 

scripts/base/frameworks/cluster/main.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/control
 
 module Cluster;
 
@@ -9,6 +10,7 @@ export {
 	} &log;
 	
 	type NodeType: enum {
+		NONE,
 		CONTROL,
 		MANAGER,
 		PROXY,
@@ -53,8 +55,8 @@ export {
 	
 	## This function can be called at any time to determine what type of
 	## cluster node the current Bro instance is going to be acting as.
-	## :bro:id:`is_enabled` should be called first to find out if this is
+	## If :bro:id:`Cluster::is_enabled` returns false, then
-	## actually going to be a cluster node.
+	## :bro:enum:`Cluster::NONE` is returned.
 	global local_node_type: function(): NodeType;
 	
 	## This gives the value for the number of workers currently connected to,
@@ -80,15 +82,15 @@ function is_enabled(): bool
 
 function local_node_type(): NodeType
 	{
-	return nodes[node]$node_type;
+	return is_enabled() ? nodes[node]$node_type : NONE;
 	}
 
-
 event remote_connection_handshake_done(p: event_peer)
 	{
 	if ( nodes[p$descr]$node_type == WORKER )
 		++worker_count;
 	}
+
 event remote_connection_closed(p: event_peer)
 	{
 	if ( nodes[p$descr]$node_type == WORKER )
@@ -100,8 +102,7 @@ event bro_init() &priority=5
 	# If a node is given, but it's an unknown name we need to fail.
 	if ( node != "" && node !in nodes )
 		{
-		local msg = "You didn't supply a valid node in the Cluster::nodes configuration.";
+		Reporter::error(fmt("'%s' is not a valid node in the Cluster::nodes configuration", node));
-		event reporter_error(current_time(), msg, "");
 		terminate();
 		}
 	

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -8,6 +8,8 @@
 ##! This is where the cluster manager sets it's specific settings for other
 ##! frameworks and in the core.
 
+@load base/frameworks/notice
+
 @prefixes += cluster-manager
 
 # Load the script for local site configuration for the manager node.

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/notice
 
 @prefixes += cluster-worker
 

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,5 +1,7 @@
 @load ./main
-@load base/frameworks/communication/main
+@load base/frameworks/communication
+
+@if ( Cluster::node in Cluster::nodes )
 
 module Cluster;
 
@@ -79,3 +81,5 @@ event bro_init() &priority=9
 			}
 		}
 	}
+
+@endif

scripts/base/frameworks/communication/main.bro

@@ -1,6 +1,8 @@
 ##! Connect to remote Bro or Broccoli instances to share state and/or transfer
 ##! events.
 
+@load base/frameworks/packet-filter
+
 module Communication;
 
 export {

scripts/base/frameworks/intel/main.bro

@@ -20,6 +20,8 @@
 #   canary
 #   friend
 
+@load base/frameworks/notice
+
 module Intel;
 
 export {

scripts/base/frameworks/logging/main.bro

@@ -159,7 +159,7 @@ export {
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
-@load logging.bif.bro # Needs Filter and Stream defined.
+@load base/logging.bif # Needs Filter and Stream defined.
 
 module Log;
 

scripts/base/frameworks/metrics/cluster.bro

@@ -8,6 +8,7 @@
 ##! to be an internal implementation detail.
 
 @load base/frameworks/cluster
+@load ./main
 
 module Metrics;
 

scripts/base/frameworks/metrics/non-cluster.bro

@@ -1,3 +1,4 @@
+@load ./main
 
 module Metrics;
 

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -4,6 +4,10 @@
 ##! probably a safe assumption to make in most cases.  If both addresses
 ##! are remote, it will use the $src address.
 
+@load ../main
+@load base/frameworks/notice
+@load base/utils/site
+
 module Notice;
 
 export {

scripts/base/frameworks/notice/actions/drop.bro

@@ -1,6 +1,8 @@
 ##! This script extends the built in notice code to implement the IP address
 ##! dropping functionality.
 
+@load ../main
+
 module Notice;
 
 export {

scripts/base/frameworks/notice/actions/email_admin.bro

@@ -1,3 +1,6 @@
+@load ../main
+@load base/utils/site
+
 module Notice;
 
 export {

scripts/base/frameworks/notice/actions/page.bro

@@ -1,3 +1,4 @@
+@load ../main
 
 module Notice;
 

scripts/base/frameworks/notice/extend-email/hostnames.bro

@@ -1,3 +1,4 @@
+@load ../main
 
 module Notice;
 

scripts/base/frameworks/notice/weird.bro

@@ -1,3 +1,7 @@
+@load base/utils/conn-ids
+@load base/utils/site
+@load ./main
+
 module Weird;
 
 export {

scripts/base/frameworks/packet-filter/main.bro

@@ -4,6 +4,8 @@
 ##! open filter and all filters defined in Bro scripts with the
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
 
+@load base/frameworks/notice
+
 module PacketFilter;
 
 export {

scripts/base/frameworks/packet-filter/netstats.bro

@@ -1,5 +1,7 @@
 ##! This script reports on packet loss from the various packet sources.
 
+@load base/frameworks/notice
+
 module PacketFilter;
 
 export {

scripts/base/frameworks/signatures/main.bro

@@ -1,5 +1,7 @@
 ##! Script level signature support.
 
+@load base/frameworks/notice
+
 module Signatures;
 
 export {

scripts/base/frameworks/software/main.bro

@@ -4,6 +4,9 @@
 ##! that they analyze.  The entry point for providing new software detections
 ##! to this framework is through the :bro:id:`Software::found` function.
 
+@load base/utils/directions-and-hosts
+@load base/utils/numbers
+
 module Software;
 
 export {

scripts/base/init-bare.bro

@@ -1,5 +1,5 @@
-@load const.bif.bro
+@load base/const.bif
-@load types.bif.bro
+@load base/types.bif
 
 # Type declarations
 type string_array: table[count] of string;
@@ -301,9 +301,9 @@ type entropy_test_result: record {
 };
 
 # Prototypes of Bro built-in functions.
-@load strings.bif.bro
+@load base/strings.bif
-@load bro.bif.bro
+@load base/bro.bif
-@load reporter.bif.bro
+@load base/reporter.bif
 
 global log_file_name: function(tag: string): string &redef;
 global open_log_file: function(tag: string): file &redef;
@@ -1290,7 +1290,7 @@ type bittorrent_benc_dir: table[string] of bittorrent_benc_value;
 ## The header table type used by the bittorrenttracker analyzer.
 type bt_tracker_headers: table[string] of string;
 
-@load event.bif.bro
+@load base/event.bif
 
 # The filter the user has set via the -f command line options, or
 # empty if none.

scripts/base/protocols/conn/contents.bro

@@ -8,6 +8,8 @@
 ##! This script does not work well in a cluster context unless it has a 
 ##! remotely mounted disk to write the content files to.
 
+@load base/utils/files
+
 module Conn;
 
 export {

scripts/base/protocols/conn/main.bro

@@ -1,3 +1,4 @@
+@load base/utils/site
 
 module Conn;
 

scripts/base/protocols/dns/main.bro

@@ -1,3 +1,4 @@
+@load ./consts
 
 module DNS;
 

scripts/base/protocols/ftp/file-extract.bro

@@ -1,5 +1,8 @@
 ##! File extraction for FTP.
 
+@load ./main
+@load base/utils/files
+
 module FTP;
 
 export {

scripts/base/protocols/ftp/main.bro

@@ -7,6 +7,10 @@
 ##!
 ##! * Handle encrypted sessions correctly (get an example?)
 
+@load ./utils-commands
+@load base/utils/paths
+@load base/utils/numbers
+
 module FTP;
 
 export {

scripts/base/protocols/http/file-extract.bro

@@ -1,6 +1,10 @@
 ##! Extracts the items from HTTP traffic, one per file.  At this time only 
 ##! the message body from the server can be extracted with this script.
 
+@load ./main
+@load ./file-ident
+@load base/utils/files
+
 module HTTP;
 
 export {

scripts/base/protocols/http/file-hash.bro

@@ -1,5 +1,7 @@
 ##! Calculate hashes for HTTP body transfers.
 
+@load ./file-ident
+
 module HTTP;
 
 export {

scripts/base/protocols/http/file-ident.bro

@@ -1,6 +1,11 @@
 ##! This script is involved in the identification of file types in HTTP
 ##! response bodies.
 
+@load base/frameworks/signatures
+@load base/frameworks/notice
+@load ./main
+@load ./utils
+
 # Add the magic number signatures to the core signature set.
 redef signature_files += "base/protocols/http/file-ident.sig";
 # Ignore the signatures used to match files

scripts/base/protocols/http/main.bro

@@ -1,3 +1,5 @@
+@load base/utils/numbers
+@load base/utils/files
 
 module HTTP;
 

scripts/base/protocols/http/partial-content.bro

@@ -3,6 +3,10 @@
 ##!
 ##! This script doesn't work yet and isn't loaded by default.
 
+@load base/frameworks/notice
+@load ./main
+@load ./utils
+
 module HTTP;
 
 export {

scripts/base/protocols/http/utils.bro

@@ -1,5 +1,7 @@
 ##! Utilities specific for HTTP processing.
 
+@load ./main
+
 module HTTP;
 
 export {

scripts/base/protocols/irc/dcc-send.bro

@@ -8,6 +8,9 @@
 ##! Example line from IRC server indicating that the DCC SEND is about to start:
 ##!    PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
 
+@load ./main
+@load base/utils/files
+
 module IRC;
 
 export {

scripts/base/protocols/mime/__load__.bro

@@ -1,4 +1,4 @@
-@load protocols/mime/base
+@load ./main
-@load protocols/mime/file-ident
+@load ./file-ident
-@load protocols/mime/file-extract
+@load ./file-extract
-@load protocols/mime/file-hash
+@load ./file-hash

scripts/base/protocols/mime/file-hash.bro

@@ -1,4 +1,5 @@
 @load ./file-ident
+@load base/frameworks/notice
 
 module MIME;
 

scripts/base/protocols/mime/file-ident.bro

@@ -1,4 +1,4 @@
-@load ./base
+@load ./main
 
 module MIME;
 

scripts/base/protocols/rpc/main.bro

@@ -8,6 +8,8 @@
 # programs for which we don't have an analyzer. 
 #
 
+@load base/utils/conn-ids
+
 module RPC;
 
 export {

scripts/base/protocols/smtp/main.bro

@@ -1,3 +1,6 @@
+@load base/frameworks/notice
+@load base/utils/addrs
+@load base/utils/directions-and-hosts
 
 module SMTP;
 

scripts/base/protocols/ssh/main.bro

@@ -5,6 +5,12 @@
 ##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
 ##! is not attempted if the connection size analyzer isn't enabled.
 
+@load base/frameworks/notice
+@load base/utils/site
+@load base/utils/thresholds
+@load base/utils/conn-ids
+@load base/utils/directions-and-hosts
+
 module SSH;
 
 export {

scripts/base/protocols/ssl/main.bro

@@ -1,3 +1,5 @@
+@load ./consts
+@load base/frameworks/notice
 
 module SSL;
 

scripts/base/protocols/ssl/mozilla-ca-list.bro

@@ -1,6 +1,6 @@
 # Don't edit!  This file is automatically generated.
 # Generated at: Wed Jun 29 07:52:38 -0400 2011
-
+@load base/protocols/ssl
 module SSL;
 redef root_certs += {
 	["GTE CyberTrust Global Root"] = "\x30\x82\x02\x5A\x30\x82\x01\xC3\x02\x02\x01\xA5\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x30\x75\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x47\x54\x45\x20\x43\x6F\x72\x70\x6F\x72\x61\x74\x69\x6F\x6E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x53\x6F\x6C\x75\x74\x69\x6F\x6E\x73\x2C\x20\x49\x6E\x63\x2E\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x1E\x17\x0D\x39\x38\x30\x38\x31\x33\x30\x30\x32\x39\x30\x30\x5A\x17\x0D\x31\x38\x30\x38\x31\x33\x32\x33\x35\x39\x30\x30\x5A\x30\x75\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x47\x54\x45\x20\x43\x6F\x72\x70\x6F\x72\x61\x74\x69\x6F\x6E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x53\x6F\x6C\x75\x74\x69\x6F\x6E\x73\x2C\x20\x49\x6E\x63\x2E\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\x95\x0F\xA0\xB6\xF0\x50\x9C\xE8\x7A\xC7\x88\xCD\xDD\x17\x0E\x2E\xB0\x94\xD0\x1B\x3D\x0E\xF6\x94\xC0\x8A\x94\xC7\x06\xC8\x90\x97\xC8\xB8\x64\x1A\x7A\x7E\x6C\x3C\x53\xE1\x37\x28\x73\x60\x7F\xB2\x97\x53\x07\x9F\x53\xF9\x6D\x58\x94\xD2\xAF\x8D\x6D\x88\x67\x80\xE6\xED\xB2\x95\xCF\x72\x31\xCA\xA5\x1C\x72\xBA\x5C\x02\xE7\x64\x42\xE7\xF9\xA9\x2C\xD6\x3A\x0D\xAC\x8D\x42\xAA\x24\x01\x39\xE6\x9C\x3F\x01\x85\x57\x0D\x58\x87\x45\xF8\xD3\x85\xAA\x93\x69\x26\x85\x70\x48\x80\x3F\x12\x15\xC7\x79\xB4\x1F\x05\x2F\x3B\x62\x99\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x03\x81\x81\x00\x6D\xEB\x1B\x09\xE9\x5E\xD9\x51\xDB\x67\x22\x61\xA4\x2A\x3C\x48\x77\xE3\xA0\x7C\xA6\xDE\x73\xA2\x14\x03\x85\x3D\xFB\xAB\x0E\x30\xC5\x83\x16\x33\x81\x13\x08\x9E\x7B\x34\x4E\xDF\x40\xC8\x74\xD7\xB9\x7D\xDC\xF4\x76\x55\x7D\x9B\x63\x54\x18\xE9\xF0\xEA\xF3\x5C\xB1\xD9\x8B\x42\x1E\xB9\xC0\x95\x4E\xBA\xFA\xD5\xE2\x7C\xF5\x68\x61\xBF\x8E\xEC\x05\x97\x5F\x5B\xB0\xD7\xA3\x85\x34\xC4\x24\xA7\x0D\x0F\x95\x93\xEF\xCB\x94\xD8\x9E\x1F\x9D\x5C\x85\x6D\xC7\xAA\xAE\x4F\x1F\x22\xB5\xCD\x95\xAD\xBA\xA7\xCC\xF9\xAB\x0B\x7A\x7F",

scripts/base/utils/directions-and-hosts.bro

@@ -1,3 +1,4 @@
+@load ./site
 
 type Direction: enum {
 	## The connection originator is not within the locally-monitored network,

scripts/policy/frameworks/control/controllee.bro

@@ -1,4 +1,4 @@
-
+@load base/frameworks/control
 # If an instance is a controllee, it implicitly needs to listen for remote
 # connections.
 @load frameworks/communication/listen-clear

scripts/policy/frameworks/control/controller.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/control
 @load base/frameworks/communication
 
 module Control;

scripts/policy/frameworks/dpd/detect-protocols.bro

@@ -1,5 +1,9 @@
 ##! Finds connections with protocols on non-standard ports with DPD.
 
+@load base/frameworks/notice
+@load base/utils/site
+@load base/utils/conn-ids
+
 module ProtocolDetector;
 
 export {

scripts/policy/frameworks/dpd/packet-segment-logging.bro

@@ -4,6 +4,8 @@
 ##! A caveat to logging packet data is that in some cases, the packet may
 ##! not be the packet that actually caused the protocol violation.
 
+@load base/frameworks/dpd
+
 module DPD;
 
 export {

scripts/policy/frameworks/metrics/conn-example.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/metrics
+@load base/utils/site
 
 redef enum Metrics::ID += { 
 	CONNS_ORIGINATED, 

scripts/policy/frameworks/metrics/http-example.bro

@@ -1,3 +1,6 @@
+@load base/frameworks/metrics
+@load base/protocols/http
+@load base/utils/site
 
 redef enum Metrics::ID += {
 	HTTP_REQUESTS_BY_STATUS_CODE,
@@ -20,7 +23,7 @@ event bro_init()
 event HTTP::log_http(rec: HTTP::Info)
 	{
 	if ( rec?$host )
-		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host]);
+		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host], 1);
 	if ( rec?$status_code )
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)]);
+		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], 1);
 	}

scripts/policy/frameworks/metrics/ssl-example.bro

@@ -1,4 +1,5 @@
-
+@load base/frameworks/metrics
+@load base/protocols/ssl
 
 redef enum Metrics::ID += {
 	SSL_SERVERNAME,

scripts/policy/frameworks/software/version-changes.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice
+@load base/frameworks/software
 
 module Software;
 

scripts/policy/frameworks/software/vulnerable.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice
+@load base/frameworks/software
 
 module Software;
 

scripts/policy/integration/barnyard2/__load__.bro

@@ -1,3 +1,2 @@
-@load integration/barnyard2/types
+@load ./types
-@load integration/barnyard2/event
+@load ./main
-@load integration/barnyard2/base

scripts/policy/integration/barnyard2/event.bro

@@ -1,3 +0,0 @@
-## This is the event that Barnyard2 instances will send if they're 
-## configured with the bro_alert output plugin.
-global barnyard_alert: event(id: Barnyard2::PacketID, alert: Barnyard2::AlertData, msg: string, data: string);

scripts/policy/integration/barnyard2/main.bro

@@ -2,7 +2,7 @@
 ##! Barnyard2 and logs them.  In the future it will do more correlation
 ##! and derive new notices from the alerts.
 
-@load integration/barnyard2/types
+@load ./types
 
 module Barnyard2;
 

scripts/policy/integration/barnyard2/types.bro

@@ -22,4 +22,11 @@ export {
 		dst_ip: addr;
 		dst_p: port;
 	} &log;
+
+	## This is the event that Barnyard2 instances will send if they're 
+	## configured with the bro_alert output plugin.
+	global barnyard_alert: event(id: Barnyard2::PacketID,
+	                             alert: Barnyard2::AlertData,
+	                             msg: string,
+	                             data: string);
 }

scripts/policy/protocols/conn/known-hosts.bro

@@ -3,6 +3,8 @@
 ##! output provides an easy way to determine a count of the IP addresses in
 ##! use on a network per day.
 
+@load base/utils/directions-and-hosts
+
 module KnownHosts;
 
 export {

scripts/policy/protocols/conn/known-services.bro

@@ -3,6 +3,8 @@
 ##! completed a TCP handshake with another host.  If a protocol is detected
 ##! during the session, the protocol will also be logged.
 
+@load base/utils/directions-and-hosts
+
 module KnownServices;
 
 redef enum Log::ID += { KNOWN_SERVICES };

scripts/policy/protocols/conn/scan.bro

@@ -1,4 +1,4 @@
-@load base/frameworks/notice
+@load base/frameworks/notice/main
 @load port-name
 
 module Scan;

scripts/policy/protocols/dns/auth-addl.bro

@@ -1,3 +1,4 @@
+@load base/protocols/dns/main
 
 redef dns_skip_all_auth = F;
 redef dns_skip_all_addl = F;

scripts/policy/protocols/dns/detect-external-names.bro

@@ -8,6 +8,9 @@
 ##!     to be within a local zone.  :bro:id:`local_zones` variable **must**
 ##!     be set appropriately for this detection.
 
+@load base/frameworks/notice/main
+@load base/utils/site
+
 module DNS;
 
 export {

scripts/policy/protocols/ftp/detect.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice/main
+@load base/protocols/ftp/main
 
 module FTP;
 

scripts/policy/protocols/ftp/software.bro

@@ -6,6 +6,8 @@
 ##! * Detect client software with password given for anonymous users
 ##!   (e.g. cyberduck@example.net)
 
+@load base/frameworks/software/main
+
 module FTP;
 
 export {

scripts/policy/protocols/http/detect-MHR.bro

@@ -4,6 +4,11 @@
 ##! documentation for the :doc:base/protocols/http/file-hash.bro script to see how to 
 ##! configure which transfers will have hashes calculated.
 
+@load base/frameworks/notice/main
+@load base/protocols/http/main
+@load base/protocols/http/utils
+@load base/protocols/http/file-hash
+
 export {
 	redef enum Notice::Type += { 
 		## If the MD5 sum of a file transferred over HTTP 

scripts/policy/protocols/http/detect-intel.bro

@@ -1,5 +1,9 @@
 ##! Intelligence based HTTP detections.
 
+@load base/protocols/http/main
+@load base/protocols/http/utils
+@load base/frameworks/intel/main
+
 module HTTP;
 
 event log_http(rec: Info)

scripts/policy/protocols/http/detect-sqli.bro

@@ -1,5 +1,9 @@
 ##! SQL injection detection in HTTP.
 
+@load base/frameworks/notice/main
+@load base/frameworks/metrics/main
+@load base/protocols/http/main
+
 module HTTP;
 
 export {

scripts/policy/protocols/http/detect-webapps.bro

@@ -1,3 +1,7 @@
+@load base/frameworks/signatures/main
+@load base/frameworks/software/main
+@load base/protocols/http/main
+@load base/protocols/http/utils
 
 module HTTP;
 

scripts/policy/protocols/http/headers.bro

@@ -1,5 +1,7 @@
 ##! Extract and include the header keys used for each request in the log.
 
+@load base/protocols/http/main
+
 module HTTP;
 
 export {

scripts/policy/protocols/http/software.bro

@@ -1,5 +1,7 @@
 ##! Software identification and extraction for HTTP traffic.
 
+@load base/frameworks/software/main
+
 module HTTP;
 
 export {

scripts/policy/protocols/http/var-extraction-cookies.bro

@@ -1,5 +1,8 @@
 ##! This script extracts and logs variables from cookies sent by clients
 
+@load base/protocols/http/main
+@load base/protocols/http/utils
+
 module HTTP;
 
 redef record Info += {

scripts/policy/protocols/http/var-extraction-uri.bro

@@ -1,5 +1,8 @@
 ##! This script extracts and logs variables from the requested URI
 
+@load base/protocols/http/main
+@load base/protocols/http/utils
+
 module HTTP;
 
 redef record Info += {

scripts/policy/protocols/smtp/detect-suspicious-orig.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice/main
+@load base/protocols/smtp/main
 
 module SMTP;
 

scripts/policy/protocols/smtp/software.bro

@@ -7,6 +7,9 @@
 ##! * Find some heuristic to determine if email was sent through 
 ##!   a MS Exhange webmail interface as opposed to a desktop client.
 
+@load base/frameworks/software/main
+@load base/protocols/smtp/main
+
 module SMTP;
 
 export {

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -1,4 +1,8 @@
 
+@load base/frameworks/metrics
+@load base/frameworks/notice
+@load base/frameworks/intel
+
 module SSH;
 
 export {

scripts/policy/protocols/ssh/geo-data.bro

@@ -1,6 +1,9 @@
 ##! This implements all of the additional information and geodata detections 
 ##! for SSH analysis.
 
+@load base/frameworks/notice/main
+@load base/protocols/ssh/main
+
 module SSH;
 
 export {

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/notice/main
 
 module SSH;
 

scripts/policy/protocols/ssh/software.bro

@@ -1,3 +1,4 @@
+@load base/frameworks/software/main
 
 module SSH;
 

scripts/policy/protocols/ssl/known-certs.bro

@@ -1,3 +1,4 @@
+@load base/utils/directions-and-hosts
 
 module KnownCerts;
 

scripts/policy/protocols/ssl/validate-certs.bro

@@ -1,3 +1,5 @@
+@load base/frameworks/notice/main
+@load base/protocols/ssl/main
 
 module SSL;
 

scripts/policy/tuning/defaults/remove-high-volume-notices.bro

@@ -1,6 +1,9 @@
 ##! This strives to tune out high volume and less useful data 
 ##! from the notice log.
 
+@load base/frameworks/notice
+@load base/frameworks/notice/weird
+
 # Remove these notices from logging since they can be too noisy.
 redef Notice::ignored_types += {
 	Weird::Content_Gap,

scripts/policy/tuning/defaults/warnings.bro

@@ -2,6 +2,8 @@
 ##! good to set in most cases or other things that could be done to achieve 
 ##! better detection.
 
+@load base/utils/site
+
 event bro_init() &priority=-10
 	{
 	if ( |Site::local_nets| == 0 )

scripts/policy/tuning/track-all-assets.bro

@@ -1,4 +1,4 @@
-
+@load base/frameworks/software
 @load protocols/conn/known-hosts
 @load protocols/conn/known-services
 @load protocols/ssl/known-certs

scripts/site/local-manager.bro

@@ -1,5 +1,7 @@
 ##! Local site policy loaded only by the manager in a cluster.
 
+@load base/frameworks/notice
+
 # If you are running a cluster you should define your Notice::policy here 
 # so that notice processing occurs on the manager.
 redef Notice::policy += {

scripts/site/local.bro

@@ -62,6 +62,7 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 # Uncomment this redef if you want to extract SMTP MIME entities for 
 # some file types.  The numbers given indicate how many bytes to extract for
 # the various mime types.
+@load base/protocols/smtp/entities-excerpt
 redef SMTP::entity_excerpt_len += {
 #	["text/plain"] = 1024,
 #	["text/html"] = 1024,

scripts/test-all-policy.bro

@@ -15,11 +15,13 @@
 # @load frameworks/control/controller.bro
 @load frameworks/dpd/detect-protocols.bro
 @load frameworks/dpd/packet-segment-logging.bro
+@load frameworks/metrics/conn-example.bro
+@load frameworks/metrics/http-example.bro
+@load frameworks/metrics/ssl-example.bro
 @load frameworks/software/version-changes.bro
 @load frameworks/software/vulnerable.bro
 @load integration/barnyard2/__load__.bro
-@load integration/barnyard2/base.bro
+@load integration/barnyard2/main.bro
-@load integration/barnyard2/event.bro
 @load integration/barnyard2/types.bro
 @load misc/analysis-groups.bro
 @load misc/loaded-scripts.bro

src/BroDoc.cc

@@ -60,7 +60,7 @@ BroDoc::BroDoc(const std::string& rel, const std::string& abs)
 	if ( ! reST_file )
 		fprintf(stderr, "Failed to open %s\n", reST_filename.c_str());
 
-#ifdef DEBUG
+#ifdef DOCDEBUG
 	fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str());
 	fprintf(stdout, "\trelative dir: %s\n", rel.c_str());
 	fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str());

src/CMakeLists.txt

@@ -107,20 +107,28 @@ macro(BIF_TARGET bifInput)
     add_custom_command(OUTPUT ${bifOutputs}
                        COMMAND bifcl
                        ARGS ${CMAKE_CURRENT_SOURCE_DIR}/${bifInput} || (rm -f ${bifOutputs} && exit 1)
+                       # in order be able to run bro from the build directory,
+                       # the generated bro script needs to be inside a
+                       # a directory tree named the same way it will be
+                       # referenced from an @load
+                       COMMAND "${CMAKE_COMMAND}"
+                       ARGS -E copy ${bifInput}.bro base/${bifInput}.bro
+                       COMMAND "${CMAKE_COMMAND}"
+                       ARGS -E remove -f ${bifInput}.bro
                        DEPENDS ${bifInput}
                        DEPENDS bifcl
                        COMMENT "[BIFCL] Processing ${bifInput}"
     )
     list(APPEND ALL_BIF_OUTPUTS ${bifOutputs})
     list(APPEND INSTALL_BIF_OUTPUTS
-         ${CMAKE_CURRENT_BINARY_DIR}/${bifInput}.bro)
+         ${CMAKE_CURRENT_BINARY_DIR}/base/${bifInput}.bro)
 endmacro(BIF_TARGET)
 
 # returns a list of output files that bifcl will produce
 # for given input file in ${outputFileVar}
 macro(GET_BIF_OUTPUT_FILES inputFile outputFileVar)
     set(${outputFileVar}
-        ${inputFile}.bro
+        base/${inputFile}.bro
         ${inputFile}.func_def
         ${inputFile}.func_h
         ${inputFile}.func_init
@@ -424,7 +432,7 @@ set(brolibs
 target_link_libraries(bro ${brolibs})
 
 install(TARGETS bro DESTINATION bin)
-install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH})
+install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)
 
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)

src/main.cc

@@ -932,9 +932,8 @@ int main(int argc, char** argv)
 
 	if ( dead_handlers->length() > 0 && check_for_unused_event_handlers )
 		{
-		reporter->Warning("event handlers never invoked:");
 		for ( int i = 0; i < dead_handlers->length(); ++i )
-			reporter->Warning("\t", (*dead_handlers)[i]);
+			reporter->Warning("event handler never invoked: %s", (*dead_handlers)[i]);
 		}
 
 	delete dead_handlers;

testing/btest/Baseline/core.check-unused-event-handlers/.stderr

@@ -0,0 +1 @@
+warning in <params>, line 1: event handler never invoked: this_is_never_used