Large reorganization.

- Scripts now use the full path for @load to remove the subpaths
  from the shipped BROPATH.
- Some script sets have been reorganized to make optional loads
  more obvious.

policy/all.bro

@@ -18,6 +18,4 @@
 @load frameworks/reporter
 @load frameworks/cluster
 
-@load detectors/http-MHR
-
 @load tuning/defaults

policy/bro.init

@@ -1344,7 +1344,7 @@ const sig_max_group_size = 50 &redef;
 const enable_syslog = F &redef;
 
 # This is transmitted to peers receiving our events.
-const peer_description = "" &redef;
+const peer_description = "bro" &redef;
 
 # If true, broadcast events/state received from one peer to other peers.
 # NOTE: These options are only temporary. They will disappear when we get a

policy/frameworks/communication/base/__load__.bro

@@ -0,0 +1 @@
+@load frameworks/communication/base/main

policy/frameworks/communication/base/main.bro

@@ -1,7 +1,7 @@
 ##! Connect to remote Bro or Broccoli instances to share state and/or transfer
 ##! events.
 
-@load packet-filter
+@load frameworks/packet-filter
 
 module Communication;
 

policy/frameworks/communication/listen-clear.bro

@@ -1,6 +1,6 @@
 ##! Listen for other Bro instances to make unencrypted connections.
 
-@load communication/base
+@load frameworks/communication/base
 
 module Communication;
 

policy/frameworks/communication/listen-ssl.bro

@@ -1,6 +1,6 @@
 ##! Listen for other Bro instances and encrypt the connection with SSL.
 
-@load communication/base
+@load frameworks/communication/base
 
 module Communication;
 

policy/frameworks/dpd/__load__.bro

@@ -1,2 +1 @@
-@load dpd/base
-@load dpd/packet-segment-logging
+@load frameworks/dpd/base

policy/frameworks/dpd/base/__load__.bro

@@ -0,0 +1 @@
+@load frameworks/dpd/base/main

policy/frameworks/dpd/base/main.bro

@@ -1,13 +1,12 @@
 ##! Activates port-independent protocol detection and selectively disables
 ##! analyzers if protocol violations occur.
 
-@load functions
-@load signatures
+@load frameworks/signatures
 
 module DPD;
 
 ## Add the DPD signatures to the signature framework.
-redef signature_files += "frameworks/dpd/dpd.sig";
+redef signature_files += "frameworks/dpd/base/dpd.sig";
 
 export {
 	redef enum Log::ID += { DPD };

policy/frameworks/dpd/detect-protocols.bro

@@ -0,0 +1,242 @@
+##! Finds connections with protocols on non-standard ports with DPD.
+
+@load frameworks/notice
+
+module ProtocolDetector;
+
+export {
+	redef enum Notice += {
+		Off_Port_Protocol_Found, # raised for each connection found
+	};
+
+	# Table of (protocol, resp_h, resp_p) tuples known to be uninteresting
+	# in the given direction.  For all other protocols detected on
+	# non-standard ports, we raise a ProtocolFound notice.  (More specific
+	# filtering can then be done via notice_filters.)
+	#
+	# Use 0.0.0.0 for to wildcard-match any resp_h.
+
+	type dir: enum { NONE, INCOMING, OUTGOING, BOTH };
+
+	const valids: table[count, addr, port] of dir = {
+		# A couple of ports commonly used for benign HTTP servers.
+
+		# For now we want to see everything.
+
+		# [ANALYZER_HTTP, 0.0.0.0, 81/tcp] = OUTGOING,
+		# [ANALYZER_HTTP, 0.0.0.0, 82/tcp] = OUTGOING,
+		# [ANALYZER_HTTP, 0.0.0.0, 83/tcp] = OUTGOING,
+		# [ANALYZER_HTTP, 0.0.0.0, 88/tcp] = OUTGOING,
+		# [ANALYZER_HTTP, 0.0.0.0, 8001/tcp] = OUTGOING,
+		# [ANALYZER_HTTP, 0.0.0.0, 8090/tcp] = OUTGOING,
+		# [ANALYZER_HTTP, 0.0.0.0, 8081/tcp] = OUTGOING,
+		#
+		# [ANALYZER_HTTP, 0.0.0.0, 6346/tcp] = BOTH, # Gnutella
+		# [ANALYZER_HTTP, 0.0.0.0, 6347/tcp] = BOTH, # Gnutella
+		# [ANALYZER_HTTP, 0.0.0.0, 6348/tcp] = BOTH, # Gnutella
+	} &redef;
+
+	# Set of analyzers for which we suppress ServerFound notices
+	# (but not ProtocolFound).  Along with avoiding clutter in the
+	# log files, this also saves memory because for these we don't
+	# need to remember which servers we already have reported, which
+	# for some can be a lot.
+	const suppress_servers: set [count] = {
+		# ANALYZER_HTTP
+	} &redef;
+
+	# We consider a connection to use a protocol X if the analyzer for X
+	# is still active (i) after an interval of minimum_duration, or (ii)
+	# after a payload volume of minimum_volume, or (iii) at the end of the
+	# connection.
+	const minimum_duration = 30 secs &redef;
+	const minimum_volume = 4e3 &redef;	# bytes
+
+	# How often to check the size of the connection.
+	const check_interval = 5 secs;
+
+	# Entry point for other analyzers to report that they recognized
+	# a certain (sub-)protocol.
+	global found_protocol: function(c: connection, analyzer: count,
+					protocol: string);
+
+	# Table keeping reported (server, port, analyzer) tuples (and their
+	# reported sub-protocols).
+	global servers: table[addr, port, string] of set[string]
+				&read_expire = 14 days;
+}
+
+# Table that tracks currently active dynamic analyzers per connection.
+global conns: table[conn_id] of set[count];
+
+# Table of reports by other analyzers about the protocol used in a connection.
+global protocols: table[conn_id] of set[string];
+
+type protocol : record {
+	a: string;	# analyzer name
+	sub: string;	# "sub-protocols" reported by other sources
+};
+
+function get_protocol(c: connection, a: count) : protocol
+	{
+	local str = "";
+	if ( c$id in protocols )
+		{
+		for ( p in protocols[c$id] )
+			str = |str| > 0 ? fmt("%s/%s", str, p) : p;
+		}
+
+	return [$a=analyzer_name(a), $sub=str];
+	}
+
+function fmt_protocol(p: protocol) : string
+	{
+	return p$sub != "" ? fmt("%s (via %s)", p$sub, p$a) : p$a;
+	}
+
+function do_notice(c: connection, a: count, d: dir)
+	{
+	if ( d == BOTH )
+		return;
+
+	if ( d == INCOMING && is_local_addr(c$id$resp_h) )
+		return;
+
+	if ( d == OUTGOING && ! is_local_addr(c$id$resp_h) )
+		return;
+
+	local p = get_protocol(c, a);
+	local s = fmt_protocol(p);
+
+	NOTICE([$note=ProtocolFound,
+		$msg=fmt("%s %s on port %s", id_string(c$id), s, c$id$resp_p),
+		$sub=s, $conn=c, $n=a]);
+
+	# We report multiple ServerFound's per host if we find a new
+	# sub-protocol.
+	local known = [c$id$resp_h, c$id$resp_p, p$a] in servers;
+
+	local newsub = F;
+	if ( known )
+		newsub = (p$sub != "" &&
+			  p$sub !in servers[c$id$resp_h, c$id$resp_p, p$a]);
+
+	if ( (! known || newsub) && a !in suppress_servers )
+		{
+		NOTICE([$note=ServerFound,
+			$msg=fmt("%s: %s server on port %s%s", c$id$resp_h, s,
+				c$id$resp_p, (known ? " (update)" : "")),
+			$p=c$id$resp_p, $sub=s, $conn=c, $src=c$id$resp_h, $n=a]);
+
+		if ( ! known )
+			servers[c$id$resp_h, c$id$resp_p, p$a] = set();
+
+		add servers[c$id$resp_h, c$id$resp_p, p$a][p$sub];
+		}
+	}
+
+function report_protocols(c: connection)
+	{
+	# We only report the connection if both sides have transferred data.
+	if ( c$resp$size == 0 || c$orig$size == 0 )
+		{
+		delete conns[c$id];
+		delete protocols[c$id];
+		return;
+		}
+
+	local analyzers = conns[c$id];
+
+	for ( a in analyzers )
+		{
+		if ( [a, c$id$resp_h, c$id$resp_p] in valids )
+			do_notice(c, a, valids[a, c$id$resp_h, c$id$resp_p]);
+
+		else if ( [a, 0.0.0.0, c$id$resp_p] in valids )
+			do_notice(c, a, valids[a, 0.0.0.0, c$id$resp_p]);
+		else
+			do_notice(c, a, NONE);
+
+		append_addl(c, analyzer_name(a));
+		}
+
+	delete conns[c$id];
+	delete protocols[c$id];
+	}
+
+event ProtocolDetector::check_connection(c: connection)
+	{
+	if ( c$id !in conns )
+		return;
+
+	local duration = network_time() - c$start_time;
+	local size = c$resp$size + c$orig$size;
+
+	if ( duration >= minimum_duration || size >= minimum_volume )
+		report_protocols(c);
+	else
+		{
+		local delay = min_interval(minimum_duration - duration,
+					   check_interval);
+		schedule delay { ProtocolDetector::check_connection(c) };
+		}
+	}
+
+event connection_state_remove(c: connection)
+	{
+	if ( c$id !in conns )
+		{
+		delete protocols[c$id];
+		return;
+		}
+
+	# Reports all analyzers that have remained to the end.
+	report_protocols(c);
+	}
+
+event protocol_confirmation(c: connection, atype: count, aid: count)
+	{
+	# Don't report anything running on a well-known port.
+	if ( atype in dpd_config && c$id$resp_p in dpd_config[atype]$ports )
+		return;
+
+	if ( c$id in conns )
+		{
+		local analyzers = conns[c$id];
+		add analyzers[atype];
+		}
+	else
+		{
+		conns[c$id] = set(atype);
+
+		local delay = min_interval(minimum_duration, check_interval);
+		schedule delay { ProtocolDetector::check_connection(c) };
+		}
+	}
+
+# event connection_analyzer_disabled(c: connection, analyzer: count)
+# 	{
+# 	if ( c$id !in conns )
+# 		return;
+# 
+# 	delete conns[c$id][analyzer];
+# 	}
+
+function append_proto_addl(c: connection)
+	{
+	for ( a in conns[c$id] )
+		append_addl(c, fmt_protocol(get_protocol(c, a)));
+	}
+
+function found_protocol(c: connection, analyzer: count, protocol: string)
+	{
+	# Don't report anything running on a well-known port.
+	if ( analyzer in dpd_config &&
+	     c$id$resp_p in dpd_config[analyzer]$ports )
+		return;
+
+	if ( c$id !in protocols )
+		protocols[c$id] = set();
+
+	add protocols[c$id][protocol];
+	}

policy/frameworks/dpd/packet-segment-logging.bro

@@ -4,8 +4,6 @@
 ##! A caveat to logging packet data is that in some cases, the packet may
 ##! not be the packet that actually caused the protocol violation.
 
-@load dpd/base
-
 module DPD;
 
 export {

policy/frameworks/intel/__load__.bro

@@ -1 +1 @@
-@load intel/base
+@load frameworks/intel/base

policy/frameworks/intel/base.bro

@@ -20,7 +20,7 @@
 #   canary
 #   friend
 
-@load notice
+@load frameworks/notice
 
 module Intel;
 

policy/frameworks/logging/__load__.bro

@@ -1,3 +1,3 @@
-@load logging/base
+@load frameworks/logging/base
 
-@load logging/plugins/ascii
+@load frameworks/logging/plugins/ascii

policy/frameworks/metrics/__load__.bro

@@ -1,4 +1 @@
-@load metrics/base
-
-@load metrics/http-example
-@load metrics/conn-example
+@load frameworks/metrics/base

policy/frameworks/metrics/base/__load__.bro

@@ -0,0 +1 @@
+@load frameworks/metrics/base/main

policy/frameworks/metrics/conn-example.bro

@@ -1,4 +1,4 @@
-@load metrics/base
+@load frameworks/metrics
 
 redef enum Metrics::ID += { 
 	CONNS_ORIGINATED, 

policy/frameworks/metrics/http-example.bro

@@ -1,5 +1,5 @@
-@load metrics/base
-@load http
+@load frameworks/metrics
+@load protocols/http
 
 redef enum Metrics::ID += {
 	HTTP_REQUESTS_BY_STATUS_CODE,

policy/frameworks/notice/__load__.bro

@@ -1,3 +1,2 @@
 @load frameworks/notice/base
 @load frameworks/notice/weird
-#@load notice/action-filters

policy/frameworks/packet-filter/__load__.bro

@@ -1,2 +1,2 @@
-@load packet-filter/base
-@load packet-filter/netstats
+@load frameworks/packet-filter/base
+@load frameworks/packet-filter/netstats

policy/frameworks/packet-filter/base.bro

@@ -4,7 +4,7 @@
 ##! open filter and all filters defined in Bro scripts with the
 ##! :bro:id:`capture_filters` and :bro:id:`restrict_filters` variables.
 
-@load notice
+@load frameworks/notice
 
 module PacketFilter;
 

policy/frameworks/packet-filter/netstats.bro

@@ -1,6 +1,6 @@
 ##! This script reports on packet loss from the various packet sources.
 
-@load notice
+@load frameworks/notice
 
 module PacketFilter;
 

policy/frameworks/reporter/__load__.bro

@@ -1 +1 @@
-@load frameworks/reporter/main
+@load frameworks/reporter/base

policy/frameworks/signatures/__load__.bro

@@ -1,3 +1,3 @@
-@load signatures/base
+@load frameworks/signatures/base
 
-redef signature_files += "signatures/detect-windows-shells.sig";
+redef signature_files += "frameworks/signatures/detect-windows-shells.sig";

policy/frameworks/signatures/base.bro

@@ -1,6 +1,6 @@
 ##! Script level signature support script.
 
-@load notice
+@load frameworks/notice
 
 module Signatures;
 

policy/frameworks/software/__load__.bro

@@ -1,2 +1 @@
-@load software/base
-@load software/vulnerable
+@load frameworks/software/base

policy/frameworks/software/base/__load__.bro

@@ -0,0 +1 @@
+@load frameworks/software/base/main

policy/frameworks/software/base/main.bro

@@ -4,8 +4,7 @@
 ##! that they analyze.  The entry point for providing new software detections
 ##! to this framework is through the :bro:id:`Software::found` function.
 
-@load functions
-@load notice
+@load frameworks/notice
 @load utils/directions-and-hosts
 @load utils/numbers
 

policy/frameworks/software/vulnerable.bro

@@ -1,5 +1,5 @@
-@load software/base
-@load notice
+@load frameworks/software
+@load frameworks/notice
 
 module Software;
 

policy/frameworks/time-machine/notice.bro

@@ -0,0 +1,3 @@
+# If we asked the Time Machine to capture, the filename prefix.
+# TODO: implement this as a timemachine/notice.bro script?
+#captured: string &optional;

policy/functions.bro

@@ -1,3 +1,3 @@
 @load site
-@load dpd
+@load frameworks/dpd
 

policy/protocols/conn/__load__.bro

@@ -1,5 +1,2 @@
-@load conn/base
-@load conn/known-hosts
-@load conn/known-services
-@load conn/contents
-@load conn/inactivity
+@load protocols/conn/base
+

policy/protocols/conn/base/__load__.bro

@@ -0,0 +1,7 @@
+@load protocols/conn/base/main
+@load protocols/conn/base/known-hosts
+@load protocols/conn/base/known-services
+@load protocols/conn/base/contents
+@load protocols/conn/base/inactivity
+
+

policy/protocols/dns/__load__.bro

@@ -1,3 +1 @@
-@load dns/consts
-@load dns/base
-@load dns/detect
+@load protocols/dns/base

policy/protocols/dns/auth-addl.bro

@@ -1,4 +1,4 @@
-@load dns/base
+@load protocols/dns/base
 
 # TODO: remove these when the options are removed from the core analyzers.
 redef dns_skip_all_auth = F;

policy/protocols/dns/base/__load__.bro

@@ -0,0 +1,4 @@
+@load protocols/dns/base/consts
+@load protocols/dns/base/main
+@load protocols/dns/base/detect
+

policy/protocols/dns/base/detect.bro

@@ -8,8 +8,7 @@
 ##!     to be within a local zone.  :bro:id:`local_zones` variable **must**
 ##!     be set appropriately for this detection.
 
-@load dns/base
-@load notice
+@load frameworks/notice
 
 module DNS;
 

policy/protocols/dns/base/main.bro

@@ -1,5 +1,4 @@
-@load functions
-@load dns/consts
+@load protocols/dns/base/consts
 
 module DNS;
 

policy/protocols/ftp/__load__.bro

@@ -1,4 +1,5 @@
-@load ftp/base
-@load ftp/detect
-@load ftp/software
-@load ftp/file-extract
+@load protocols/ftp/utils-commands
+@load protocols/ftp/base
+@load protocols/ftp/detect
+@load protocols/ftp/software
+@load protocols/ftp/file-extract

policy/protocols/ftp/base.bro

@@ -7,9 +7,6 @@
 ##!
 ##! * Handle encrypted sessions correctly (get an example?)
 
-@load functions
-@load ftp/utils-commands
-
 @load utils/paths
 @load utils/numbers
 

policy/protocols/ftp/detect.bro

@@ -1,5 +1,5 @@
-@load ftp/base
-@load notice
+@load protocols/ftp
+@load frameworks/notice
 
 module FTP;
 

policy/protocols/ftp/file-extract.bro

@@ -1,6 +1,6 @@
 ##! File extraction for FTP.
 
-@load ftp/base
+@load protocols/ftp
 @load utils/conn_ids
 @load utils/files
 

policy/protocols/ftp/software.bro

@@ -6,8 +6,8 @@
 ##! * Detect client software with password given for anonymous users
 ##!   (e.g. cyberduck@example.net)
 
-@load ftp/base
-@load software
+@load protocols/ftp
+@load frameworks/software
 
 module FTP;
 

policy/protocols/http/__load__.bro

@@ -1,14 +1,10 @@
 ##! This script is the wrapper script for HTTP analysis.
 ##! :Author: Seth Hall <seth@icir.org> - Inspired by the work of many others.
 
-@load http/base
-@load http/detect-sqli
-@load http/detect-intel
-@load http/file-ident
-@load http/file-hash
-@load http/file-extract
-@load http/software
-@load http/headers
+@load protocols/http/base/main
+
+#@load protocols/http/detect-MHR
+#@load protocols/http/headers
 
 # Disabling web app detection for now.  It's too intense and will probably
 # be moved out of the core http protocol support later.

policy/protocols/http/base/__load__.bro

@@ -0,0 +1,7 @@
+@load protocols/http/base/main
+@load protocols/http/base/detect-sqli
+@load protocols/http/base/detect-intel
+@load protocols/http/base/file-ident
+@load protocols/http/base/file-hash
+@load protocols/http/base/file-extract
+@load protocols/http/base/software

policy/protocols/http/base/file-ident.bro

@@ -7,7 +7,7 @@
 @load notice
 @load signatures
 
-redef signature_files += "http/file-ident.sig";
+redef signature_files += "protocols/http/file-ident.sig";
 # Ignore the signatures used to match files
 redef Signatures::ignored_ids += /^matchfile-/;
 

policy/protocols/http/partial-content.bro

@@ -0,0 +1,94 @@
+##! This script makes it possible for the HTTP analysis scripts to analyze
+##! the apparent normal case of "206 Partial Content" responses.
+
+@load notice
+
+module HTTP;
+
+export {
+	redef enum Notice::Type += {
+		Partial_Content_Out_Of_Order,
+	};
+	
+	type Range: record {
+		from: count;
+		to:   count;
+	} &log;
+
+	redef record Info += {
+		current_range:   count           &default=0;
+		request_ranges:  vector of Range &optional;
+		response_range:  Range           &optional;
+	};
+	
+	## Index is client IP address, server IP address, and URL being requested.  The
+	## URL is tracked as part of the index in case multiple partial content segmented
+	## files are being transferred simultaneously between the server and client.
+	global partial_content_files: table[addr, addr, string] of Info &read_expire=5mins &redef;
+}
+
+event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=2
+	{
+	local parts: table[count] of string;
+	if ( is_orig && name == "RANGE" )
+		{
+		# Example --> Range: bytes=1-1,2336-4951
+		parts = split(value, /[=]/);
+		if ( 2 in parts )
+			{
+			local ranges = split(parts[2], /,/);
+			for ( i in ranges )
+				{
+				if ( ! c$http?$request_ranges )
+					c$http$request_ranges = vector();
+				parts = split(ranges[i], /-/);
+				local r: Range = [$from=extract_count(parts[1]), $to=extract_count(parts[2])];
+				print r;
+				c$http$request_ranges[|c$http$request_ranges|] = r;
+				}
+			}
+		}
+	else if ( ! is_orig && name == "CONTENT-RANGE" )
+		{
+		# Example --> Content-Range: bytes 2336-4951/489528
+		parts = split(value, /[0-9]*/);
+		
+		c$http$response_range = [$from=extract_count(parts[2]), $to=extract_count(parts[4])];
+		
+		}
+	}
+	
+event http_reply(c: connection, version: string, code: count, reason: string) &priority=5
+	{
+	if ( code != 206 || ! c$http?$request_ranges )
+		return;
+
+	local url = build_url(c$http);
+	if ( [c$id$orig_h, c$id$resp_h, url] !in partial_content_files )
+		{
+		partial_content_files[c$id$orig_h, c$id$resp_h, url] = copy(c$http);
+		}
+	}
+
+event http_entity_data(c: connection, is_orig: bool, length: count, data: string)
+	{
+	if ( is_orig || c$http$status_code != 206 || ! c$http?$request_ranges )
+		return;
+	
+	local url = build_url(c$http);
+	local http = partial_content_files[c$id$orig_h, c$id$resp_h, url];
+	local range = http$request_ranges[http$current_range];
+	
+	print http$current_range;
+	if ( http$current_range == 0 &&
+	     c$http$response_range$from == 0 )
+		{
+		print "correct file beginning!";
+		}
+	}
+
+event http_end_entity(c: connection, is_orig: bool)
+	{
+	print "end entity";
+	++c$http$current_range;
+	}

policy/protocols/irc/__load__.bro

@@ -1,2 +1,2 @@
-@load irc/base
-@load irc/dcc-send
+@load protocols/irc/base
+@load protocols/irc/dcc-send

policy/protocols/irc/dcc-send.bro

@@ -8,7 +8,7 @@
 ##! Example line from IRC server indicating that the DCC SEND is about to start:
 ##!    PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
 
-@load irc/base
+@load protocols/irc
 
 module IRC;
 

policy/protocols/mime/__load__.bro

@@ -1,4 +1,4 @@
-@load mime/base
-@load mime/file-ident
-@load mime/file-extract
-@load mime/file-hash
+@load protocols/mime/base
+@load protocols/mime/file-ident
+@load protocols/mime/file-extract
+@load protocols/mime/file-hash

policy/protocols/mime/file-extract.bro

@@ -1,4 +1,4 @@
-@load mime/file-ident
+@load protocols/mime/file-ident
 @load utils/files
 
 module MIME;

policy/protocols/mime/file-hash.bro

@@ -1,4 +1,4 @@
-@load mime/file-ident
+@load protocols/mime/file-ident
 
 module MIME;
 

policy/protocols/mime/file-ident.bro

@@ -1,4 +1,4 @@
-@load mime/base
+@load protocols/mime/base
 
 module MIME;
 

policy/protocols/smtp/__load__.bro

@@ -1,2 +1,4 @@
-@load smtp/base
-@load smtp/software
+@load protocols/smtp/base
+
+## This should be optional
+@load protocols/smtp/detect-suspicious-orig

policy/protocols/smtp/base/__load__.bro

@@ -0,0 +1,2 @@
+@load protocols/smtp/base/main
+@load protocols/smtp/base/software

policy/protocols/smtp/base/main.bro

@@ -1,8 +1,4 @@
-@load functions
-@load notice
-@load software
-
-@load smtp/detect
+@load frameworks/notice
 @load utils/addrs
 
 module SMTP;
@@ -12,11 +8,9 @@ export {
 
 	redef enum Notice::Type += { 
 		## Indicates that the server sent a reply mentioning an SMTP block list.
-		SMTP_BL_Error_Message, 
+		BL_Error_Message, 
 		## Indicates the client's address is seen in the block list error message.
-		SMTP_BL_Blocked_Host, 
-		## When mail seems to originate from a suspicious location.
-		SMTP_Suspicious_Origination,
+		BL_Blocked_Host, 
 	};
 
 	type Info: record {
@@ -70,6 +64,25 @@ export {
 	##    NO_HOSTS - never capture the path.
 	const mail_path_capture = ALL_HOSTS &redef;
 	
+	# This matches content in SMTP error messages that indicate some
+	# block list doesn't like the connection/mail.
+	const bl_error_messages = 
+	    /spamhaus\.org\//
+	  | /sophos\.com\/security\//
+	  | /spamcop\.net\/bl/
+	  | /cbl\.abuseat\.org\// 
+	  | /sorbs\.net\// 
+	  | /bsn\.borderware\.com\//
+	  | /mail-abuse\.com\//
+	  | /b\.barracudacentral\.com\//
+	  | /psbl\.surriel\.com\// 
+	  | /antispam\.imp\.ch\// 
+	  | /dyndns\.com\/.*spam/
+	  | /rbl\.knology\.net\//
+	  | /intercept\.datapacket\.net\//
+	  | /uceprotect\.net\//
+	  | /hostkarma\.junkemailfilter\.com\// &redef;
+	
 	global log_smtp: event(rec: Info);
 	
 	## Configure the default ports for SMTP analysis.
@@ -195,7 +208,7 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 		# Raise a notice when an SMTP error about a block list is discovered.
 		if ( bl_error_messages in msg )
 			{
-			local note = SMTP_BL_Error_Message;
+			local note = BL_Error_Message;
 			local message = fmt("%s received an error message mentioning an SMTP block list", c$id$orig_h);
 
 			# Determine if the originator's IP address is in the message.
@@ -203,7 +216,7 @@ event smtp_reply(c: connection, is_orig: bool, code: count, cmd: string,
 			local text_ip = "";
 			if ( |ips| > 0 && to_addr(ips[0]) == c$id$orig_h )
 				{
-				note = SMTP_BL_Blocked_Host;
+				note = BL_Blocked_Host;
 				message = fmt("%s is on an SMTP block list", c$id$orig_h);
 				}
 			

policy/protocols/smtp/base/software.bro

@@ -6,8 +6,7 @@
 ##! * Find some heuristic to determine if email was sent through 
 ##!   a MS Exhange webmail interface as opposed to a desktop client.
 
-@load smtp/base
-@load software
+@load frameworks/software
 
 module SMTP;
 
@@ -15,7 +14,7 @@ export {
 	redef enum Software::Type += {
 		MAIL_CLIENT,
 		MAIL_SERVER,
-		WEBMAIL
+		WEBMAIL_SERVER
 	};
 	
 	redef record Info += {
@@ -63,7 +62,7 @@ event log_smtp(rec: Info)
 		local client_ip = rec$path[|rec$path|-1];
 		if ( rec$is_webmail )
 			{
-			s_type = WEBMAIL;
+			s_type = WEBMAIL_SERVER;
 			# If the earliest received header indicates that the connection
 			# was via HTTP, then that likely means the actual mail software 
 			# is installed on the second address in the path.

policy/protocols/smtp/detect-suspicious-orig.bro

@@ -0,0 +1,49 @@
+module SMTP;
+
+@load frameworks/notice
+@load protocols/smtp/base
+
+export {
+	redef enum Notice::Type += {
+		Suspicious_Origination
+	};
+
+	## Places where it's suspicious for mail to originate from represented as
+	## all-capital, two character country codes (e.x. US).  It requires 
+	## libGeoIP support built in.
+	const suspicious_origination_countries: set[string] = {} &redef;
+	const suspicious_origination_networks: set[subnet] = {} &redef;
+
+}
+
+event log_smtp(rec: Info)
+	{
+	local ip: addr;
+	local loc: geo_location;
+	if ( rec?$x_originating_ip )
+		{
+		ip = rec$x_originating_ip;
+		loc = lookup_location(ip);
+		
+		if ( loc$country_code in suspicious_origination_countries ||
+			 ip in suspicious_origination_networks )
+			{
+			NOTICE([$note=Suspicious_Origination,
+			        $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
+			        $id=rec$id]);
+			}
+		}
+	if ( rec?$path )
+		{
+		ip = rec$path[|rec$path|-1];
+		loc = lookup_location(ip);
+
+		if ( loc$country_code in suspicious_origination_countries ||
+			 ip in suspicious_origination_networks )
+			{
+			NOTICE([$note=Suspicious_Origination,
+			        $msg=fmt("Based up Received headers, email originated from %s (%s).", loc$country_code, ip),
+			        $id=rec$id]);
+			}
+		}
+	}

policy/protocols/smtp/detect.bro

@@ -1,60 +0,0 @@
-module SMTP;
-
-export {
-	## Places where it's suspicious for mail to originate from represented as
-	## all-capital, two character country codes (e.x. US).  It requires 
-	## libGeoIP support built in.
-	const suspicious_origination_countries: set[string] = {} &redef;
-	const suspicious_origination_networks: set[subnet] = {} &redef;
-
-	# This matches content in SMTP error messages that indicate some
-	# block list doesn't like the connection/mail.
-	const bl_error_messages = 
-	    /spamhaus\.org\//
-	  | /sophos\.com\/security\//
-	  | /spamcop\.net\/bl/
-	  | /cbl\.abuseat\.org\// 
-	  | /sorbs\.net\// 
-	  | /bsn\.borderware\.com\//
-	  | /mail-abuse\.com\//
-	  | /b\.barracudacentral\.com\//
-	  | /psbl\.surriel\.com\// 
-	  | /antispam\.imp\.ch\// 
-	  | /dyndns\.com\/.*spam/
-	  | /rbl\.knology\.net\//
-	  | /intercept\.datapacket\.net\//
-	  | /uceprotect\.net\//
-	  | /hostkarma\.junkemailfilter\.com\// &redef;
-}
-
-
-#if ( c$smtp?$x_originating_ip )
-#	{
-#	ip = session$log$x_originating_ip;
-#	loc = lookup_location(ip);
-#	
-#	if ( loc$country_code in suspicious_origination_countries ||
-#		 ip in suspicious_origination_networks )
-#		{
-#		NOTICE([$note=SMTP_Suspicious_Origination,
-#			    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
-#			    $sub=fmt("Subject: %s", session$log$subject),
-#			    $conn=c]);
-#		}
-#	if ( session$log?$received_from_originating_ip &&
-#	     session$log$received_from_originating_ip != session$log$x_originating_ip )
-#		{
-#		ip = session$log$received_from_originating_ip;
-#		loc = lookup_location(ip);
-#
-#		if ( loc$country_code in suspicious_origination_countries ||
-#			 ip in suspicious_origination_networks )
-#			{
-#			NOTICE([$note=SMTP_Suspicious_Origination,
-#				    $msg=fmt("An email originated from %s (%s).", loc$country_code, ip),
-#				    $sub=fmt("Subject: %s", session$log$subject),
-#					$conn=c]);
-#			}
-#		}
-#	}
-#

policy/protocols/ssh/__load__.bro

@@ -1,2 +1,2 @@
-@load ssh/base
-@load ssh/software
+@load protocols/ssh/base
+@load protocols/ssh/software

policy/protocols/ssh/base.bro

@@ -1,4 +1,4 @@
-@load notice
+@load frameworks/notice
 @load utils/thresholds
 
 module SSH;

policy/protocols/ssh/software.bro

@@ -1,5 +1,5 @@
-@load ssh/base
-@load software
+@load protocols/ssh
+@load frameworks/software
 
 module SSH;
 

policy/protocols/ssl/__load__.bro

@@ -1,4 +1,4 @@
-@load ssl/consts
-@load ssl/base
-@load ssl/mozilla-ca-list
-@load ssl/known-certs
+@load protocols/ssl/consts
+@load protocols/ssl/base
+@load protocols/ssl/mozilla-ca-list
+@load protocols/ssl/known-certs

policy/protocols/ssl/base.bro

@@ -1,4 +1,4 @@
-@load notice
+@load frameworks/notice
 
 module SSL;
 

policy/protocols/ssl/mozilla-ca-list.bro

@@ -1,6 +1,6 @@
 # Don't edit!  This file is automatically generated.
 # Generated at: Wed Jun 29 07:52:38 -0400 2011
-@load ssl
+@load protocols/ssl
 module SSL;
 redef root_certs += {
 	["GTE CyberTrust Global Root"] = "\x30\x82\x02\x5A\x30\x82\x01\xC3\x02\x02\x01\xA5\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x30\x75\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x47\x54\x45\x20\x43\x6F\x72\x70\x6F\x72\x61\x74\x69\x6F\x6E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x53\x6F\x6C\x75\x74\x69\x6F\x6E\x73\x2C\x20\x49\x6E\x63\x2E\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x1E\x17\x0D\x39\x38\x30\x38\x31\x33\x30\x30\x32\x39\x30\x30\x5A\x17\x0D\x31\x38\x30\x38\x31\x33\x32\x33\x35\x39\x30\x30\x5A\x30\x75\x31\x0B\x30\x09\x06\x03\x55\x04\x06\x13\x02\x55\x53\x31\x18\x30\x16\x06\x03\x55\x04\x0A\x13\x0F\x47\x54\x45\x20\x43\x6F\x72\x70\x6F\x72\x61\x74\x69\x6F\x6E\x31\x27\x30\x25\x06\x03\x55\x04\x0B\x13\x1E\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x53\x6F\x6C\x75\x74\x69\x6F\x6E\x73\x2C\x20\x49\x6E\x63\x2E\x31\x23\x30\x21\x06\x03\x55\x04\x03\x13\x1A\x47\x54\x45\x20\x43\x79\x62\x65\x72\x54\x72\x75\x73\x74\x20\x47\x6C\x6F\x62\x61\x6C\x20\x52\x6F\x6F\x74\x30\x81\x9F\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x01\x05\x00\x03\x81\x8D\x00\x30\x81\x89\x02\x81\x81\x00\x95\x0F\xA0\xB6\xF0\x50\x9C\xE8\x7A\xC7\x88\xCD\xDD\x17\x0E\x2E\xB0\x94\xD0\x1B\x3D\x0E\xF6\x94\xC0\x8A\x94\xC7\x06\xC8\x90\x97\xC8\xB8\x64\x1A\x7A\x7E\x6C\x3C\x53\xE1\x37\x28\x73\x60\x7F\xB2\x97\x53\x07\x9F\x53\xF9\x6D\x58\x94\xD2\xAF\x8D\x6D\x88\x67\x80\xE6\xED\xB2\x95\xCF\x72\x31\xCA\xA5\x1C\x72\xBA\x5C\x02\xE7\x64\x42\xE7\xF9\xA9\x2C\xD6\x3A\x0D\xAC\x8D\x42\xAA\x24\x01\x39\xE6\x9C\x3F\x01\x85\x57\x0D\x58\x87\x45\xF8\xD3\x85\xAA\x93\x69\x26\x85\x70\x48\x80\x3F\x12\x15\xC7\x79\xB4\x1F\x05\x2F\x3B\x62\x99\x02\x03\x01\x00\x01\x30\x0D\x06\x09\x2A\x86\x48\x86\xF7\x0D\x01\x01\x04\x05\x00\x03\x81\x81\x00\x6D\xEB\x1B\x09\xE9\x5E\xD9\x51\xDB\x67\x22\x61\xA4\x2A\x3C\x48\x77\xE3\xA0\x7C\xA6\xDE\x73\xA2\x14\x03\x85\x3D\xFB\xAB\x0E\x30\xC5\x83\x16\x33\x81\x13\x08\x9E\x7B\x34\x4E\xDF\x40\xC8\x74\xD7\xB9\x7D\xDC\xF4\x76\x55\x7D\x9B\x63\x54\x18\xE9\xF0\xEA\xF3\x5C\xB1\xD9\x8B\x42\x1E\xB9\xC0\x95\x4E\xBA\xFA\xD5\xE2\x7C\xF5\x68\x61\xBF\x8E\xEC\x05\x97\x5F\x5B\xB0\xD7\xA3\x85\x34\xC4\x24\xA7\x0D\x0F\x95\x93\xEF\xCB\x94\xD8\x9E\x1F\x9D\x5C\x85\x6D\xC7\xAA\xAE\x4F\x1F\x22\xB5\xCD\x95\xAD\xBA\xA7\xCC\xF9\xAB\x0B\x7A\x7F",

policy/protocols/syslog/__load__.bro

@@ -1 +1,2 @@
-@load syslog/base
+@load protocols/syslog/consts
+@load protocols/syslog/base

policy/protocols/syslog/base.bro

@@ -1,7 +1,5 @@
 ##! Core script support for logging syslog messages.
 
-@load syslog/consts 
-
 module Syslog;
 
 export {

policy/tuning/defaults/remove-high-volume-notices.bro

@@ -1,10 +1,7 @@
 ##! This strives to tune out high volume and less useful data 
 ##! from the notice log.
 
-@load notice
-
-# Load the policy scripts where the notices are defined.
-@load frameworks/notice/weird
+@load frameworks/notice
 
 # Remove these notices from logging since they can be too noisy.
 redef Notice::ignored_types += {