Merge branch 'topic/script-reference' of ssh://git.bro-ids.org/bro into topic/script-reference

Conflicts:
	scripts/base/frameworks/notice/actions/pp-alarms.bro
	scripts/base/frameworks/notice/main.bro
	src/bro.bif
	src/const.bif
	src/event.bif
	src/strings.bif
	src/types.bif

scripts/base/frameworks/control/main.bro

@@ -1,43 +1,30 @@
-##! This is a utility script that sends the current values of all &redef'able
+##! The control framework provides the foundation for providing "commands"
-##! consts to a remote Bro then sends the
+##! that can be taken remotely at runtime to modify a running Bro instance
-##! :bro:id:`Control::configuration_update` event and terminates processing.
+##! or collect information from the running instance.
-##!
-##! Intended to be used from the command line like this when starting a controller::
-##!
-##!     bro <scripts> frameworks/control/controller Control::host=<host_addr> Control::port=<host_port> Control::cmd=<command> [Control::arg=<arg>]
-##!
-##! A controllee only needs to load the controllee script in addition
-##! to the specific analysis scripts desired.  It may also need a node
-##! configured as a controller node in the communications nodes configuration::
-##!
-##!     bro <scripts> frameworks/control/controllee
-##!
-##! To use the framework as a controllee, it only needs to be loaded and
-##! the controlled node need to accept all events in the "Control::" namespace
-##! from the host where the control actions will be performed from along with
-##! using the "control" class.
 
 module Control;
 
 export {
-	## This is the address of the host that will be controlled.
+	## The address of the host that will be controlled.
 	const host = 0.0.0.0 &redef;
 
-	## This is the port of the host that will be controlled.
+	## The port of the host that will be controlled.
 	const host_port = 0/tcp &redef;
 
-	## This is the command that is being done.  It's typically set on the
+	## The command that is being done.  It's typically set on the
-	## command line and influences whether this instance starts up as a
+	## command line.
-	## controller or controllee.
 	const cmd = "" &redef;
 
 	## This can be used by commands that take an argument.
 	const arg = "" &redef;
 
+	## Events that need to be handled by controllers.
 	const controller_events = /Control::.*_request/ &redef;
+	
+	## Events that need to be handled by controllees.
 	const controllee_events = /Control::.*_response/ &redef;
 
-	## These are the commands that can be given on the command line for
+	## The commands that can currently be given on the command line for
 	## remote control.
 	const commands: set[string] = {
 		"id_value",
@@ -45,11 +32,10 @@ export {
 		"net_stats",
 		"configuration_update",
 		"shutdown",
-	};
+	} &redef;
 
 	## Variable IDs that are to be ignored by the update process.
-	const ignore_ids: set[string] = {
+	const ignore_ids: set[string] = { };
-	};
 
 	## Event for requesting the value of an ID (a variable).
 	global id_value_request: event(id: string);

scripts/base/frameworks/intel/main.bro

@@ -11,7 +11,7 @@
 #   user_name
 #   file_name
 #   file_md5
-#   x509_cert - DER encoded, not PEM (ascii armored)
+#   x509_md5
 
 # Example tags: 
 #   infrastructure
@@ -25,6 +25,7 @@
 module Intel;
 
 export {
+	## The intel logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
 	redef enum Notice::Type += {
@@ -33,72 +34,117 @@ export {
 		Detection,
 	};
 	
+	## Record type used for logging information from the intelligence framework.
+	## Primarily for problems or oddities with inserting and querying data.  
+	## This is important since the content of the intelligence framework can 
+	## change quite dramatically during runtime and problems may be introduced 
+	## into the data.
 	type Info: record {
+		## The current network time.
 		ts:      time   &log;
+		## Represents the severity of the message. 
 		## This value should be one of: "info", "warn", "error"
 		level:   string &log;
+		## The message.
 		message: string &log;
 	};
 	
+	## Record to represent metadata associated with a single piece of
+	## intelligence.
 	type MetaData: record {
+		## A description for the data.
 		desc:        string      &optional;
+		## A URL where more information may be found about the intelligence.
 		url:         string      &optional;
+		## The time at which the data was first declared to be intelligence.
 		first_seen:  time        &optional;
+		## When this data was most recent inserted into the framework.
 		latest_seen: time        &optional;
+		## Arbitrary text tags for the data.
 		tags:        set[string];
 	};
 	
+	## Record to represent a singular piece of intelligence.
 	type Item: record {
+		## If the data is an IP address, this hold the address.
 		ip:          addr        &optional;
+		## If the data is textual, this holds the text.
 		str:         string      &optional;
+		## If the data is numeric, this holds the number.
 		num:         int         &optional;
+		## The subtype of the data for when either the $str or $num fields are
+		## given.  If one of those fields are given, this field must be present.
 		subtype:     string      &optional;
 		
+		## The next five fields are temporary until a better model for 
+		## attaching metadata to an intelligence item is created.
 		desc:        string      &optional;
 		url:         string      &optional;
 		first_seen:  time        &optional;
 		latest_seen: time        &optional;
 		tags:        set[string];
 		
-		## These single string tags are throw away until pybroccoli supports sets
+		## These single string tags are throw away until pybroccoli supports sets.
 		tag1: string &optional;
 		tag2: string &optional;
 		tag3: string &optional;
 	};
 	
+	## Record model used for constructing queries against the intelligence 
+	## framework.
 	type QueryItem: record {
-		ip:          addr        &optional;
+		## If an IP address is being queried for, this field should be given.
-		str:         string      &optional;
+		ip:        addr        &optional;
-		num:         int         &optional;
+		## If a string is being queried for, this field should be given.
-		subtype:     string      &optional;
+		str:       string      &optional;
+		## If numeric data is being queried for, this field should be given.
+		num:       int         &optional;
+		## If either a string or number is being queried for, this field should
+		## indicate the subtype of the data.
+		subtype:   string      &optional;
 		
-		or_tags:     set[string] &optional;
+		## A set of tags where if a single metadata record attached to an item
-		and_tags:    set[string] &optional; 
+		## has any one of the tags defined in this field, it will match.
+		or_tags:   set[string] &optional;
+		## A set of tags where a single metadata record attached to an item 
+		## must have all of the tags defined in this field.
+		and_tags:  set[string] &optional; 
 		
 		## The predicate can be given when searching for a match.  It will
 		## be tested against every :bro:type:`Intel::MetaData` item associated
 		## with the data being matched on.  If it returns T a single time, the 
-		## matcher will consider that the item has matched.
+		## matcher will consider that the item has matched.  This field can
-		pred:    function(meta: Intel::MetaData): bool &optional;
+		## be used for constructing arbitrarily complex queries that may not
+		## be possible with the $or_tags or $and_tags fields.
+		pred:      function(meta: Intel::MetaData): bool &optional;
 	};
 	
-	
+	## Function to insert data into the intelligence framework.
+	## 
+	## item: The data item.
+	##
+	## Returns: T if the data was successfully inserted into the framework,
+	##          otherwise it returns F.
 	global insert: function(item: Item): bool;
+	
+	## A wrapper for the :bro:id:`Intel::insert` function.  This is primarily
+	## used as the external API for inserting data into the intelligence 
+	## using Broccoli.
 	global insert_event: event(item: Item);
+	
+	## Function for matching data within the intelligence framework.
 	global matcher: function(item: QueryItem): bool;
-
-	type MetaDataStore: table[count] of MetaData;
-	type DataStore: record {
-		ip_data:     table[addr] of MetaDataStore;
-		## The first string is the actual value and the second string is the subtype.
-		string_data: table[string, string] of MetaDataStore;
-		int_data:    table[int, string] of MetaDataStore;
-	};
-	global data_store: DataStore;
-
-
 }
 
+type MetaDataStore: table[count] of MetaData;
+type DataStore: record {
+	ip_data:     table[addr] of MetaDataStore;
+	# The first string is the actual value and the second string is the subtype.
+	string_data: table[string, string] of MetaDataStore;
+	int_data:    table[int, string] of MetaDataStore;
+};
+global data_store: DataStore;
+
 event bro_init()
 	{
 	Log::create_stream(Intel::LOG, [$columns=Info]);

scripts/base/frameworks/metrics/cluster.bro

@@ -13,11 +13,11 @@
 module Metrics;
 
 export {
-	## This value allows a user to decide how large of result groups the 
+	## Allows a user to decide how large of result groups the 
-	## workers should transmit values.
+	## workers should transmit values for cluster metric aggregation.
 	const cluster_send_in_groups_of = 50 &redef;
 	
-	## This is the percent of the full threshold value that needs to be met 
+	## The percent of the full threshold value that needs to be met 
 	## on a single worker for that worker to send the value to its manager in
 	## order for it to request a global view for that value.  There is no
 	## requirement that the manager requests a global view for the index
@@ -25,11 +25,11 @@ export {
 	## recently.
 	const cluster_request_global_view_percent = 0.1 &redef;
 	
-	## This event is sent by the manager in a cluster to initiate the 
+	## Event sent by the manager in a cluster to initiate the 
 	## collection of metrics values for a filter.
 	global cluster_filter_request: event(uid: string, id: ID, filter_name: string);
 
-	## This event is sent by nodes that are collecting metrics after receiving
+	## Event sent by nodes that are collecting metrics after receiving
 	## a request for the metric filter from the manager.
 	global cluster_filter_response: event(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool);
 

scripts/base/frameworks/metrics/main.bro

@@ -1,13 +1,16 @@
-##! This is the implementation of the metrics framework.
+##! The metrics framework provides a way to count and measure data.  
 
 @load base/frameworks/notice
 
 module Metrics;
 
 export {
+	## The metrics logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
+	## Identifiers for metrics to collect.
 	type ID: enum {
+		## Blank placeholder value.
 		NOTHING,
 	};
 	
@@ -15,10 +18,13 @@ export {
 	## current value to the logging stream.
 	const default_break_interval = 15mins &redef;
 	
-	## This is the interval for how often notices will happen after they have
+	## This is the interval for how often threshold based notices will happen 
-	## already fired.
+	## after they have already fired.
 	const renotice_interval = 1hr &redef;
 	
+	## Represents a thing which is having metrics collected for it.  An instance
+	## of this record type and a :bro:type:`Metrics::ID` together represent a 
+	## single measurement.
 	type Index: record {
 		## Host is the value to which this metric applies.
 		host:         addr &optional;
@@ -37,17 +43,30 @@ export {
 		network:      subnet &optional;
 	} &log;
 	
+	## The record type that is used for logging metrics.
 	type Info: record {
+		## Timestamp at which the metric was "broken".
 		ts:           time   &log;
+		## What measurement the metric represents.
 		metric_id:    ID     &log;
+		## The name of the filter being logged.  :bro:type:`Metrics::ID` values
+		## can have multiple filters which represent different perspectives on
+		## the data so this is necessary to understand the value.
 		filter_name:  string &log;
+		## What the metric value applies to.
 		index:        Index  &log;
+		## The simple numeric value of the metric.
 		value:        count  &log;
 	};
 	
-	# TODO: configure a metrics filter logging stream to log the current
+    # TODO: configure a metrics filter logging stream to log the current
 	#       metrics configuration in case someone is looking through
 	#       old logs and the configuration has changed since then.
+	
+	## Filters define how the data from a metric is aggregated and handled.  
+	## Filters can be used to set how often the measurements are cut or "broken"
+	## and logged or how the data within them is aggregated.  It's also 
+	## possible to disable logging and use filters for thresholding.
 	type Filter: record {
 		## The :bro:type:`Metrics::ID` that this filter applies to.
 		id:                ID                      &optional;
@@ -62,7 +81,7 @@ export {
 		aggregation_mask:  count                   &optional;
 		## This is essentially a mapping table between addresses and subnets.
 		aggregation_table: table[subnet] of subnet &optional;
-		## The interval at which the metric should be "broken" and written
+		## The interval at which this filter should be "broken" and written
 		## to the logging stream.  The counters are also reset to zero at 
 		## this time so any threshold based detection needs to be set to a 
 		## number that should be expected to happen within this period.
@@ -79,25 +98,51 @@ export {
 		notice_threshold:  count                   &optional;
 		## A series of thresholds at which to generate notices.
 		notice_thresholds: vector of count         &optional;
-		## How often this notice should be raised for this metric index.  It 
+		## How often this notice should be raised for this filter.  It 
 		## will be generated everytime it crosses a threshold, but if the 
 		## $break_interval is set to 5mins and this is set to 1hr the notice
 		## only be generated once per hour even if something crosses the
 		## threshold in every break interval.
 		notice_freq:       interval                &optional;
 	};
-
-	type MetricTable: table[Index] of count &default=0;
 	
+	## Function to associate a metric filter with a metric ID.
+	## 
+	## id: The metric ID that the filter should be associated with.
+	##
+	## filter: The record representing the filter configuration.
 	global add_filter: function(id: ID, filter: Filter);
+	
+	## Add data into a :bro:type:`Metrics::ID`.  This should be called when
+	## a script has measured some point value and is ready to increment the
+	## counters.
+	##
+	## id: The metric ID that the data represents.
+	##
+	## index: The metric index that the value is to be added to.
+	##
+	## increment: How much to increment the counter by.
 	global add_data: function(id: ID, index: Index, increment: count);
+	
+	## Helper function to represent a :bro:type:`Metrics::Index` value as 
+	## a simple string
+	## 
+	## index: The metric index that is to be converted into a string.
+	##
+	## Returns: A string reprentation of the metric index.
 	global index2str: function(index: Index): string;
 	
-	# This is the event that is used to "finish" metrics and adapt the metrics
+	## Event that is used to "finish" metrics and adapt the metrics
-	# framework for clustered or non-clustered usage.
+	## framework for clustered or non-clustered usage.
+	##
+	## ..note: This is primarily intended for internal use.
 	global log_it: event(filter: Filter);
 	
+	## Event to access metrics records as they are passed to the logging framework.
 	global log_metrics: event(rec: Info);
+	
+	## Type to store a table of metrics values.  Interal use only!
+	type MetricTable: table[Index] of count &default=0;
 }
 
 redef record Notice::Info += {

scripts/base/frameworks/software/main.bro

@@ -1,5 +1,5 @@
 ##! This script provides the framework for software version detection and
-##! parsing, but doesn't actually do any detection on it's own.  It relys on
+##! parsing but doesn't actually do any detection on it's own.  It relys on
 ##! other protocol specific scripts to parse out software from the protocols
 ##! that they analyze.  The entry point for providing new software detections
 ##! to this framework is through the :bro:id:`Software::found` function.
@@ -10,39 +10,44 @@
 module Software;
 
 export {
-
+	## The software logging stream identifier.
 	redef enum Log::ID += { LOG };
-
+	
+	## Scripts detecting new types of software need to redef this enum to add
+	## their own specific software types which would then be used when they 
+	## create :bro:type:`Software::Info` records.
 	type Type: enum {
+		## A placeholder type for when the type of software is not known.
 		UNKNOWN,
-		OPERATING_SYSTEM,
-		DATABASE_SERVER,
-		# There are a number of ways to detect printers on the 
-		# network, we just need to codify them in a script and move
-		# this out of here.  It isn't currently used for anything.
-		PRINTER,
 	};
-
+	
+	## A structure to represent the numeric version of software.
 	type Version: record {
-		major:  count  &optional;  ##< Major version number
+		## Major version number
-		minor:  count  &optional;  ##< Minor version number
+		major:  count  &optional;
-		minor2: count  &optional;  ##< Minor subversion number
+		## Minor version number
-		addl:   string &optional;  ##< Additional version string (e.g. "beta42")
+		minor:  count  &optional;
+		## Minor subversion number
+		minor2: count  &optional;
+		## Additional version string (e.g. "beta42")
+		addl:   string &optional;
 	} &log;
-
+	
+	## The record type that is used for representing and logging software.
 	type Info: record {
-		## The time at which the software was first detected.
+		## The time at which the software was detected.
 		ts:               time &log;
 		## The IP address detected running the software.
 		host:             addr &log;
-		## The type of software detected (e.g. WEB_SERVER)
+		## The type of software detected (e.g. :bro:enum:`HTTP::SERVER`).
 		software_type:    Type &log &default=UNKNOWN;
-		## Name of the software (e.g. Apache)
+		## Name of the software (e.g. Apache).
 		name:             string &log;
-		## Version of the software
+		## Version of the software.
 		version:          Version &log;
 		## The full unparsed version string found because the version parsing 
-		## doesn't work 100% reliably and this acts as a fall back in the logs.
+		## doesn't always work reliably in all cases and this acts as a 
+		## fallback in the logs.
 		unparsed_version: string &log &optional;
 		
 		## This can indicate that this software being detected should
@@ -55,37 +60,48 @@ export {
 		force_log:        bool &default=F;
 	};
 	
-	## The hosts whose software should be detected and tracked.
+	## Hosts whose software should be detected and tracked.
 	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
 	const asset_tracking = LOCAL_HOSTS &redef;
 	
-	
 	## Other scripts should call this function when they detect software.
 	## unparsed_version: This is the full string from which the
 	##                   :bro:type:`Software::Info` was extracted.
+	##
+	## id: The connection id where the software was discovered.
+	##
+	## info: A record representing the software discovered.
+	## 
 	## Returns: T if the software was logged, F otherwise.
 	global found: function(id: conn_id, info: Software::Info): bool;
 	
-	## This function can take many software version strings and parse them 
+	## Take many common software version strings and parse them 
 	## into a sensible :bro:type:`Software::Version` record.  There are 
 	## still many cases where scripts may have to have their own specific 
 	## version parsing though.
+	##
+	## unparsed_version: The raw version string.
+	##
+	## host: The host where the software was discovered.
+	##
+	## software_type: The type of software.
+	##
+	## Returns: A complete record ready for the :bro:id:`Software::found` function.
 	global parse: function(unparsed_version: string,
 	                       host: addr,
 	                       software_type: Type): Info;
 	
-	## Compare two versions.
+	## Compare two version records.
+	## 
 	## Returns:  -1 for v1 < v2, 0 for v1 == v2, 1 for v1 > v2.
 	##           If the numerical version numbers match, the addl string
 	##           is compared lexicographically.
 	global cmp_versions: function(v1: Version, v2: Version): int;
 	
-	## This type represents a set of software.  It's used by the 
+	## Type to represent a collection of :bro:type:`Software::Info` records.
-	## :bro:id:`Software::tracked` variable to store all known pieces of
+	## It's indexed with the name of a piece of software such as "Firefox" 
-	## software for a particular host.  It's indexed with the name of a piece
+	## and it yields a :bro:type:`Software::Info` record with more information
-	## of software such as "Firefox" and it yields a 
+	## about the  software.
-	## :bro:type:`Software::Info` record with more information about the 
-	## software.
 	type SoftwareSet: table[string] of Info;
 	
 	## The set of software associated with an address.  Data expires from

scripts/base/protocols/conn/contents.bro

@@ -1,23 +1,27 @@
 ##! This script can be used to extract either the originator's data or the 
 ##! responders data or both.  By default nothing is extracted, and in order 
 ##! to actually extract data the ``c$extract_orig`` and/or the
-##! ``c$extract_resp`` variable must be set to T.  One way to achieve this
+##! ``c$extract_resp`` variable must be set to ``T``.  One way to achieve this
-##! would be to handle the connection_established event elsewhere and set the
+##! would be to handle the :bro:id:`connection_established` event elsewhere
-##! extract_orig and extract_resp options there.  However, there may be trouble
+##! and set the ``extract_orig`` and ``extract_resp`` options there.
-##! with the timing due the event queue delay.
+##! However, there may be trouble with the timing due to event queue delay.
-##! This script does not work well in a cluster context unless it has a 
+##!
-##! remotely mounted disk to write the content files to.
+##! .. note::
+##!
+##!    This script does not work well in a cluster context unless it has a
+##!    remotely mounted disk to write the content files to.
 
 @load base/utils/files
 
 module Conn;
 
 export {
-	## The prefix given to files as they are opened on disk.
+	## The prefix given to files containing extracted connections as they are
+	## opened on disk.
 	const extraction_prefix = "contents" &redef;
 	
-	## If this variable is set to T, then all contents of all files will be 
+	## If this variable is set to ``T``, then all contents of all connections
-	## extracted.
+	## will be  extracted.
 	const default_extract = F &redef;
 }
 

scripts/base/protocols/conn/inactivity.bro

@@ -4,7 +4,7 @@
 module Conn;
 
 export {
-	## Define inactivty timeouts by the service detected being used over
+	## Define inactivity timeouts by the service detected being used over
 	## the connection.
 	const analyzer_inactivity_timeouts: table[AnalyzerTag] of interval = {
 		# For interactive services, allow longer periods of inactivity.

scripts/base/protocols/conn/main.bro

@@ -1,17 +1,33 @@
+##! This script manages the tracking/logging of general information regarding
+##! TCP, UDP, and ICMP traffic.  For UDP and ICMP, "connections" are to
+##! be interpreted using flow semantics (sequence of packets from a source
+##! host/post to a destination host/port).  Further, ICMP "ports" are to
+##! be interpreted as the source port meaning the ICMP message type and
+##! the destination port being the ICMP message code.
+
 @load base/utils/site
 
 module Conn;
 
 export {
+	## The connection logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## The record type which contains column fields of the connection log.
 	type Info: record {
 		## This is the time of the first packet.
 		ts:           time            &log;
+		## A unique identifier of a connection.
 		uid:          string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:           conn_id         &log;
+		## The transport layer protocol of the connection.
 		proto:        transport_proto &log;
+		## An identification of an application protocol being sent over the
+		## the connection.
 		service:      string          &log &optional;
+		## How long the connection lasted.  For 3-way or 4-way connection
+		## tear-downs, this will not include the final ACK.
 		duration:     interval        &log &optional;
 		## The number of payload bytes the originator sent. For TCP
 		## this is taken from sequence numbers and might be inaccurate 
@@ -51,8 +67,8 @@ export {
 		## have been completed prior to the packet loss.
 		missed_bytes: count           &log &default=0;
 
-		## Records the state history of (TCP) connections as
+		## Records the state history of connections as a string of letters.
-		## a string of letters.
+		## For TCP connections the meaning of those letters is:
 		##
 		## ======  ====================================================
 		## Letter  Meaning
@@ -71,7 +87,8 @@ export {
 		## originator and lower case then means the responder.
 		## Also, there is compression. We only record one "d" in each direction,
 		## for instance. I.e., we just record that data went in that direction.
-		## This history is not meant to encode how much data that happened to be.
+		## This history is not meant to encode how much data that happened to
+		## be.
 		history:      string          &log &optional;
 		## Number of packets the originator sent.
 		## Only set if :bro:id:`use_conn_size_analyzer` = T
@@ -85,7 +102,9 @@ export {
 		## Number IP level bytes the responder sent. See ``orig_pkts``.
 		resp_ip_bytes: count      &log &optional;
 	};
-	
+
+	## Event that can be handled to access the :bro:type:`Conn::Info`
+	## record as it is sent on to the logging framework.
 	global log_conn: event(rec: Info);
 }
 

scripts/base/protocols/dns/consts.bro

@@ -4,9 +4,9 @@
 module DNS;
 
 export {
-	const PTR = 12;
+	const PTR = 12;  ##< RR TYPE value for a domain name pointer.
-	const EDNS = 41;
+	const EDNS = 41; ##< An OPT RR TYPE value described by EDNS.
-	const ANY = 255;
+	const ANY = 255; ##< A QTYPE value describing a request for all records.
 
 	## Mapping of DNS query type codes to human readable string representation.
 	const query_types = {
@@ -29,50 +29,43 @@ export {
 		[ANY] = "*",
 	} &default = function(n: count): string { return fmt("query-%d", n); };
 
-	const code_types = {
-		[0] = "X0",
-		[1] = "Xfmt",
-		[2] = "Xsrv",
-		[3] = "Xnam",
-		[4] = "Ximp",
-		[5] = "X[",
-	} &default="?";
-
 	## Errors used for non-TSIG/EDNS types.
 	const base_errors = {
-		[0] = "NOERROR",        ##< No Error
+		[0] = "NOERROR",        # No Error
-		[1] = "FORMERR",        ##< Format Error
+		[1] = "FORMERR",        # Format Error
-		[2] = "SERVFAIL",       ##< Server Failure
+		[2] = "SERVFAIL",       # Server Failure
-		[3] = "NXDOMAIN",       ##< Non-Existent Domain
+		[3] = "NXDOMAIN",       # Non-Existent Domain
-		[4] = "NOTIMP",         ##< Not Implemented
+		[4] = "NOTIMP",         # Not Implemented
-		[5] = "REFUSED",        ##< Query Refused
+		[5] = "REFUSED",        # Query Refused
-		[6] = "YXDOMAIN",       ##< Name Exists when it should not
+		[6] = "YXDOMAIN",       # Name Exists when it should not
-		[7] = "YXRRSET",        ##< RR Set Exists when it should not
+		[7] = "YXRRSET",        # RR Set Exists when it should not
-		[8] = "NXRRSet",        ##< RR Set that should exist does not
+		[8] = "NXRRSet",        # RR Set that should exist does not
-		[9] = "NOTAUTH",        ##< Server Not Authoritative for zone
+		[9] = "NOTAUTH",        # Server Not Authoritative for zone
-		[10] = "NOTZONE",       ##< Name not contained in zone
+		[10] = "NOTZONE",       # Name not contained in zone
-		[11] = "unassigned-11", ##< available for assignment
+		[11] = "unassigned-11", # available for assignment
-		[12] = "unassigned-12", ##< available for assignment
+		[12] = "unassigned-12", # available for assignment
-		[13] = "unassigned-13", ##< available for assignment
+		[13] = "unassigned-13", # available for assignment
-		[14] = "unassigned-14", ##< available for assignment
+		[14] = "unassigned-14", # available for assignment
-		[15] = "unassigned-15", ##< available for assignment
+		[15] = "unassigned-15", # available for assignment
-		[16] = "BADVERS",       ##< for EDNS, collision w/ TSIG
+		[16] = "BADVERS",       # for EDNS, collision w/ TSIG
-		[17] = "BADKEY",        ##< Key not recognized
+		[17] = "BADKEY",        # Key not recognized
-		[18] = "BADTIME",       ##< Signature out of time window
+		[18] = "BADTIME",       # Signature out of time window
-		[19] = "BADMODE",       ##< Bad TKEY Mode
+		[19] = "BADMODE",       # Bad TKEY Mode
-		[20] = "BADNAME",       ##< Duplicate key name
+		[20] = "BADNAME",       # Duplicate key name
-		[21] = "BADALG",        ##< Algorithm not supported
+		[21] = "BADALG",        # Algorithm not supported
-		[22] = "BADTRUNC",      ##< draft-ietf-dnsext-tsig-sha-05.txt
+		[22] = "BADTRUNC",      # draft-ietf-dnsext-tsig-sha-05.txt
-		[3842] = "BADSIG",      ##< 16 <= number collision with EDNS(16);
+		[3842] = "BADSIG",      # 16 <= number collision with EDNS(16);
-		                        ##< this is a translation from TSIG(16)
+		                        # this is a translation from TSIG(16)
 	} &default = function(n: count): string { return fmt("rcode-%d", n); };
 
-	# This deciphers EDNS Z field values.
+	## This deciphers EDNS Z field values.
 	const edns_zfield = {
 		[0]     = "NOVALUE",    # regular entry
 		[32768] = "DNS_SEC_OK", # accepts DNS Sec RRs
 	} &default="?";
 
+	## Possible values of the CLASS field in resource records or QCLASS field
+	## in query messages.
 	const classes = {
 		[1]   = "C_INTERNET",
 		[2]   = "C_CSNET",
@@ -81,4 +74,4 @@ export {
 		[254] = "C_NONE",
 		[255] = "C_ANY",
 	} &default = function(n: count): string { return fmt("qclass-%d", n); };
-}
+}

scripts/base/protocols/dns/main.bro

@@ -1,38 +1,80 @@
+##! Base DNS analysis script which tracks and logs DNS queries along with
+##! their responses.
+
 @load ./consts
 
 module DNS;
 
 export {
+	## The DNS logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## The record type which contains the column fields of the DNS log.
 	type Info: record {
+		## The earliest time at which a DNS protocol message over the
+		## associated connection is observed.
 		ts:            time               &log;
+		## A unique identifier of the connection over which DNS messages
+		## are being transferred.
 		uid:           string             &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:            conn_id            &log;
+		## The transport layer protocol of the connection.
 		proto:         transport_proto    &log;
+		## A 16 bit identifier assigned by the program that generated the
+		## DNS query.  Also used in responses to match up replies to
+		## outstanding queries.
 		trans_id:      count              &log &optional;
+		## The domain name that is the subject of the DNS query.
 		query:         string             &log &optional;
+		## The QCLASS value specifying the class of the query.
 		qclass:        count              &log &optional;
+		## A descriptive name for the class of the query.
 		qclass_name:   string             &log &optional;
+		## A QTYPE value specifying the type of the query.
 		qtype:         count              &log &optional;
+		## A descriptive name for the type of the query.
 		qtype_name:    string             &log &optional;
+		## The response code value in DNS response messages.
 		rcode:         count              &log &optional;
+		## A descriptive name for the response code value.
 		rcode_name:    string             &log &optional;
+		## Whether the message is a query (F) or response (T).
 		QR:            bool               &log &default=F;
+		## The Authoritative Answer bit for response messages specifies that
+		## the responding name server is an authority for the domain name
+		## in the question section.
 		AA:            bool               &log &default=F;
+		## The Truncation bit specifies that the message was truncated.
 		TC:            bool               &log &default=F;
+		## The Recursion Desired bit indicates to a name server to recursively
+		## purse the query.
 		RD:            bool               &log &default=F;
+		## The Recursion Available bit in a response message indicates if
+		## the name server supports recursive queries.
 		RA:            bool               &log &default=F;
+		## A reserved field that is currently supposed to be zero in all
+		## queries and responses.
 		Z:             count              &log &default=0;
+		## The set of resource descriptions in answer of the query.
 		answers:       vector of string   &log &optional;
+		## The caching intervals of the associated RRs described by the
+		## ``answers`` field.
 		TTLs:          vector of interval &log &optional;
 
-		## This value indicates if this request/response pair is ready to be logged.
+		## This value indicates if this request/response pair is ready to be
+		## logged.
 		ready:         bool            &default=F;
+		## The total number of resource records in a reply message's answer
+		## section.
 		total_answers: count           &optional;
+		## The total number of resource records in a reply message's answer,
+		## authority, and additional sections.
 		total_replies: count           &optional;
 	};
 
+	## A record type which tracks the status of DNS queries for a given
+	## :bro:type:`connection`.
 	type State: record {
 		## Indexed by query id, returns Info record corresponding to
 		## query/response which haven't completed yet.
@@ -44,11 +86,21 @@ export {
 		finished_answers: set[count] &optional;
 	};
 
+	## An event that can be handled to access the :bro:type:`DNS::Info`
+	## record as it is sent to the logging framework.
 	global log_dns: event(rec: Info);
 
 	## This is called by the specific dns_*_reply events with a "reply" which
 	## may not represent the full data available from the resource record, but
 	## it's generally considered a summarization of the response(s).
+	##
+	## c: The connection record for which to fill in DNS reply data.
+	##
+	## msg: The DNS message header information for the response.
+	##
+	## ans: The general information of a RR response.
+	##
+	## reply: The specific response information according to RR type/class.
 	global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string);
 }
 

scripts/base/protocols/ftp/file-extract.bro

@@ -1,4 +1,4 @@
-##! File extraction for FTP.
+##! File extraction support for FTP.
 
 @load ./main
 @load base/utils/files
@@ -6,7 +6,7 @@
 module FTP;
 
 export {
-	## Pattern of file mime types to extract from FTP entity bodies.
+	## Pattern of file mime types to extract from FTP transfers.
 	const extract_file_types = /NO_DEFAULT/ &redef;
 
 	## The on-disk prefix for files to be extracted from FTP-data transfers.
@@ -14,10 +14,15 @@ export {
 }
 
 redef record Info += {
-	## The file handle for the file to be extracted
+	## On disk file where it was extracted to.
 	extraction_file:       file &log &optional;
 	
+	## Indicates if the current command/response pair should attempt to 
+	## extract the file if a file was transferred.
 	extract_file:          bool &default=F;
+	
+	## Internal tracking of the total number of files extracted during this 
+	## session.
 	num_extracted_files:   count &default=0;
 };
 
@@ -33,7 +38,6 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	if ( extract_file_types in s$mime_type )
 		{
 		s$extract_file = T;
-		add s$tags["extracted_file"];
 		++s$num_extracted_files;
 		}
 	}

scripts/base/protocols/ftp/main.bro

@@ -1,11 +1,7 @@
 ##! The logging this script does is primarily focused on logging FTP commands
 ##! along with metadata.  For example, if files are transferred, the argument
 ##! will take on the full path that the client is at along with the requested 
-##! file name.  
+##! file name.
-##! 
-##! TODO:
-##!
-##! * Handle encrypted sessions correctly (get an example?)
 
 @load ./utils-commands
 @load base/utils/paths
@@ -14,38 +10,64 @@
 module FTP;
 
 export {
+	## The FTP protocol logging stream identifier.
 	redef enum Log::ID += { LOG };
-
+	
+	## List of commands that should have their command/response pairs logged.
+	const logged_commands = {
+		"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT"
+	} &redef;
+	
 	## This setting changes if passwords used in FTP sessions are captured or not.
 	const default_capture_password = F &redef;
 	
+	## User IDs that can be considered "anonymous".
+	const guest_ids = { "anonymous", "ftp", "guest" } &redef;
+	
 	type Info: record {
+		## Time when the command was sent.
 		ts:               time        &log;
 		uid:              string      &log;
 		id:               conn_id     &log;
+		## User name for the current FTP session.
 		user:             string      &log &default="<unknown>";
+		## Password for the current FTP session if captured.
 		password:         string      &log &optional;
+		## Command given by the client.
 		command:          string      &log &optional;
+		## Argument for the command if one is given.
 		arg:              string      &log &optional;
-		                              
+		
+		## Libmagic "sniffed" file type if the command indicates a file transfer.
 		mime_type:        string      &log &optional;
+		## Libmagic "sniffed" file description if the command indicates a file transfer.
 		mime_desc:        string      &log &optional;
+		## Size of the file if the command indicates a file transfer.
 		file_size:        count       &log &optional;
+		
+		## Reply code from the server in response to the command.
 		reply_code:       count       &log &optional;
+		## Reply message from the server in response to the command.
 		reply_msg:        string      &log &optional;
+		## Arbitrary tags that may indicate a particular attribute of this command.
 		tags:             set[string] &log &default=set();
 		
-		## By setting the CWD to '/.', we can indicate that unless something
+		## Current working directory that this session is in.  By making
+		## the default value '/.', we can indicate that unless something
 		## more concrete is discovered that the existing but unknown
 		## directory is ok to use.
 		cwd:                string  &default="/.";
+		
+		## Command that is currently waiting for a response.
 		cmdarg:             CmdArg  &optional;
+		## Queue for commands that have been sent but not yet responded to 
+		## are tracked here.
 		pending_commands:   PendingCmds;
 		
-		## This indicates if the session is in active or passive mode.
+		## Indicates if the session is in active or passive mode.
 		passive:            bool &default=F;
 		
-		## This determines if the password will be captured for this request.
+		## Determines if the password will be captured for this request.
 		capture_password:   bool &default=default_capture_password;
 	};
 
@@ -56,22 +78,12 @@ export {
 		y: count;
 		z: count;
 	};
-
-	# TODO: add this back in some form.  raise a notice again?
-	#const excessive_filename_len = 250 &redef;
-	#const excessive_filename_trunc_len = 32 &redef;
-
-	## These are user IDs that can be considered "anonymous".
-	const guest_ids = { "anonymous", "ftp", "guest" } &redef;
 	
-	## The list of commands that should have their command/response pairs logged.
+	## Parse FTP reply codes into the three constituent single digit values.
-	const logged_commands = {
-		"APPE", "DELE", "RETR", "STOR", "STOU", "ACCT"
-	} &redef;
-	
-	## This function splits FTP reply codes into the three constituent 
 	global parse_ftp_reply_code: function(code: count): ReplyCode;
-
+	
+	## Event that can be handled to access the :bro:type:`FTP::Info`
+	## record as it is sent on to the logging framework.
 	global log_ftp: event(rec: Info);
 }
 

scripts/base/protocols/ftp/utils-commands.bro

@@ -2,14 +2,22 @@ module FTP;
 
 export {
 	type CmdArg: record {
+		## Time when the command was sent.
 		ts:   time;
+		## Command.
 		cmd:  string &default="<unknown>";
+		## Argument for the command if one was given.
 		arg:  string &default="";
+		## Counter to track how many commands have been executed.
 		seq:  count &default=0;
 	};
-
+	
+	## Structure for tracking pending commands in the event that the client
+	## sends a large number of commands before the server has a chance to 
+	## reply.
 	type PendingCmds: table[count] of CmdArg;
-
+	
+	## Possible response codes for a wide variety of FTP commands.
 	const cmd_reply_code: set[string, count] = {
 		# According to RFC 959
 		["<init>", [120, 220, 421]],

scripts/base/protocols/http/file-extract.bro

@@ -8,29 +8,24 @@
 module HTTP;
 
 export {
-	## Pattern of file mime types to extract from HTTP entity bodies.
+	## Pattern of file mime types to extract from HTTP response entity bodies.
 	const extract_file_types = /NO_DEFAULT/ &redef;
 
 	## The on-disk prefix for files to be extracted from HTTP entity bodies.
 	const extraction_prefix = "http-item" &redef;
 
 	redef record Info += {
-		## This field can be set per-connection to determine if the entity body
+		## On-disk file where the response body was extracted to.
-		## will be extracted.  It must be set to T on or before the first 
-		## entity_body_data event.
-		extracting_file:  bool &default=F;
-	
-		## This is the holder for the file handle as the file is being written
-		## to disk.
 		extraction_file:  file &log &optional;
-	};
+		
-
+		## Indicates if the response body is to be extracted or not.  Must be 
-	redef record State += {
+		## set before or by the first :bro:id:`http_entity_data` event for the
-		entity_bodies:    count &default=0;
+		## content.
+		extract_file:     bool &default=F;
 	};
 }
 
-event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=5
+event http_entity_data(c: connection, is_orig: bool, length: count, data: string) &priority=-5
 	{
 	# Client body extraction is not currently supported in this script.
 	if ( is_orig )
@@ -41,8 +36,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string
 		if ( c$http?$mime_type &&
 		     extract_file_types in c$http$mime_type )
 			{
-			c$http$extracting_file = T;
+			c$http$extract_file = T;
-			local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", ++c$http_state$entity_bodies);
+			}
+			
+		if ( c$http$extract_file )
+			{
+			local suffix = fmt("%s_%d.dat", is_orig ? "orig" : "resp", c$http_state$current_response);
 			local fname = generate_extraction_filename(extraction_prefix, c, suffix);
 			
 			c$http$extraction_file = open(fname);
@@ -50,12 +49,12 @@ event http_entity_data(c: connection, is_orig: bool, length: count, data: string
 			}
 		}
 
-	if ( c$http$extracting_file )
+	if ( c$http?$extraction_file )
 		print c$http$extraction_file, data;
 	}
 
 event http_end_entity(c: connection, is_orig: bool)
 	{
-	if ( c$http$extracting_file )
+	if ( c$http?$extraction_file )
 		close(c$http$extraction_file);
 	}

scripts/base/protocols/http/file-hash.bro

@@ -11,7 +11,8 @@ export {
 	};
 
 	redef record Info += {
-		## The MD5 sum for a file transferred over HTTP will be stored here.
+		## MD5 sum for a file transferred over HTTP calculated from the 
+		## response body.
 		md5:             string   &log &optional;
 		
 		## This value can be set per-transfer to determine per request
@@ -19,8 +20,8 @@ export {
 		## set to T at the time of or before the first chunk of body data.
 		calc_md5:        bool     &default=F;
 		
-		## This boolean value indicates if an MD5 sum is currently being 
+		## Indicates if an MD5 sum is being calculated for the current 
-		## calculated for the current file transfer.
+		## request/response pair.
 		calculating_md5: bool     &default=F;
 	};
 	

scripts/base/protocols/http/file-ident.bro

@@ -1,5 +1,4 @@
-##! This script is involved in the identification of file types in HTTP
+##! Identification of file types in HTTP response bodies with file content sniffing.
-##! response bodies.
 
 @load base/frameworks/signatures
 @load base/frameworks/notice
@@ -15,27 +14,23 @@ module HTTP;
 
 export {
 	redef enum Notice::Type += {
-		# This notice is thrown when the file extension doesn't 
+		## Indicates when the file extension doesn't seem to match the file contents.
-		# seem to match the file contents.
 		Incorrect_File_Type,
 	};
 
 	redef record Info += {
-		## This will record the mime_type identified.
+		## Mime type of response body identified by content sniffing.
 		mime_type:    string   &log &optional;
 		
-		## This indicates that no data of the current file transfer has been
+		## Indicates that no data of the current file transfer has been
 		## seen yet.  After the first :bro:id:`http_entity_data` event, it 
-		## will be set to T.
+		## will be set to F.
 		first_chunk:     bool &default=T;
 	};
-
-	redef enum Tags += {
-		IDENTIFIED_FILE
-	};
 	
-	# Create regexes that *should* in be in the urls for specifics mime types.
+	## Mapping between mime types and regular expressions for URLs
-	# Notices are thrown if the pattern doesn't match the url for the file type.
+	## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the pattern 
+	## doesn't match the mime type that was discovered.
 	const mime_types_extensions: table[string] of pattern = {
 		["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/,
 	} &redef;

scripts/base/protocols/http/main.bro

@@ -1,3 +1,7 @@
+##! Implements base functionality for HTTP analysis.  The logging model is 
+##! to log request/response pairs and all relevant metadata together in 
+##! a single record.
+
 @load base/utils/numbers
 @load base/utils/files
 
@@ -8,6 +12,7 @@ export {
 
 	## Indicate a type of attack or compromise in the record to be logged.
 	type Tags: enum {
+		## Placeholder.
 		EMPTY
 	};
 	
@@ -15,64 +20,69 @@ export {
 	const default_capture_password = F &redef;
 	
 	type Info: record {
-		ts:                      time     &log;
+		## Timestamp for when the request happened.
-		uid:                     string   &log;
+		ts:                      time      &log;
-		id:                      conn_id  &log;
+		uid:                     string    &log;
-		## This represents the pipelined depth into the connection of this 
+		id:                      conn_id   &log;
+		## Represents the pipelined depth into the connection of this 
 		## request/response transaction.
-		trans_depth:             count    &log;
+		trans_depth:             count     &log;
-		## The verb used in the HTTP request (GET, POST, HEAD, etc.).
+		## Verb used in the HTTP request (GET, POST, HEAD, etc.).
-		method:                  string   &log &optional;
+		method:                  string    &log &optional;
-		## The value of the HOST header.
+		## Value of the HOST header.
-		host:                    string   &log &optional;
+		host:                    string    &log &optional;
-		## The URI used in the request.
+		## URI used in the request.
-		uri:                     string   &log &optional;
+		uri:                     string    &log &optional;
-		## The value of the "referer" header.  The comment is deliberately
+		## Value of the "referer" header.  The comment is deliberately
 		## misspelled like the standard declares, but the name used here is
 		## "referrer" spelled correctly.
-		referrer:                string   &log &optional;
+		referrer:                string    &log &optional;
-		## The value of the User-Agent header from the client.
+		## Value of the User-Agent header from the client.
-		user_agent:              string   &log &optional;
+		user_agent:              string    &log &optional;
-		## The actual uncompressed content size of the data transferred from
+		## Actual uncompressed content size of the data transferred from
 		## the client.
-		request_body_len:        count    &log &default=0;
+		request_body_len:        count     &log &default=0;
-		## The actual uncompressed content size of the data transferred from
+		## Actual uncompressed content size of the data transferred from
 		## the server.
 		response_body_len:       count     &log &default=0;
-		## The status code returned by the server.
+		## Status code returned by the server.
 		status_code:             count     &log &optional;
-		## The status message returned by the server.
+		## Status message returned by the server.
 		status_msg:              string    &log &optional;
-		## The last 1xx informational reply code returned by the server.
+		## Last seen 1xx informational reply code returned by the server.
 		info_code:               count     &log &optional;
-		## The last 1xx informational reply message returned by the server.
+		## Last seen 1xx informational reply message returned by the server.
 		info_msg:                string    &log &optional;
-		## The filename given in the Content-Disposition header
+		## Filename given in the Content-Disposition header sent by the server.
-		## sent by the server.
 		filename:                string    &log &optional;
-		## This is a set of indicators of various attributes discovered and
+		## A set of indicators of various attributes discovered and
 		## related to a particular request/response pair.
 		tags:                    set[Tags] &log;
 		
-		## The username if basic-auth is performed for the request.
+		## Username if basic-auth is performed for the request.
 		username:                string    &log &optional;
-		## The password if basic-auth is performed for the request.
+		## Password if basic-auth is performed for the request.
 		password:                string    &log &optional;
 		
-		## This determines if the password will be captured for this request.
+		## Determines if the password will be captured for this request.
 		capture_password:        bool      &default=default_capture_password;
 		
 		## All of the headers that may indicate if the request was proxied.
 		proxied:                 set[string] &log &optional;
 	};
 	
+	## Structure to maintain state for an HTTP connection with multiple 
+	## requests and responses.
 	type State: record {
+		## Pending requests.
 		pending:          table[count] of Info;
-		current_response: count                &default=0;
+		## Current request in the pending queue.
 		current_request:  count                &default=0;
+		## Current response in the pending queue.
+		current_response: count                &default=0;
 	};
 		
-	## The list of HTTP headers typically used to indicate a proxied request.
+	## A list of HTTP headers typically used to indicate proxied requests.
 	const proxy_headers: set[string] = {
 		"FORWARDED",
 		"X-FORWARDED-FOR",
@@ -83,6 +93,8 @@ export {
 		"PROXY-CONNECTION",
 	} &redef;
 	
+	## Event that can be handled to access the HTTP record as it is sent on 
+	## to the logging framework.
 	global log_http: event(rec: Info);
 }
 

scripts/base/protocols/http/utils.bro

@@ -5,8 +5,31 @@
 module HTTP;
 
 export {
+	## Given a string containing a series of key-value pairs separated by "=",
+	## this function can be used to parse out all of the key names.
+	##
+	## data: The raw data, such as a URL or cookie value.
+	##
+	## kv_splitter: A regular expression representing the separator between
+	##              key-value pairs.
+	##
+	## Returns: A vector of strings containing the keys.
 	global extract_keys: function(data: string, kv_splitter: pattern): string_vec;
+	
+	## Creates a URL from an :bro:type:`HTTP::Info` record.  This should handle
+	## edge cases such as proxied requests appropriately.
+	##
+	## rec: An :bro:type:`HTTP::Info` record.
+	##
+	## Returns: A URL, not prefixed by "http://".
 	global build_url: function(rec: Info): string;
+	
+	## Creates a URL from an :bro:type:`HTTP::Info` record.  This should handle
+	## edge cases such as proxied requests appropriately.
+	##
+	## rec: An :bro:type:`HTTP::Info` record.
+	##
+	## Returns: A URL prefixed with "http://".
 	global build_url_http: function(rec: Info): string;
 }
 

scripts/base/protocols/irc/dcc-send.bro

@@ -5,8 +5,9 @@
 ##! but that connection will actually be between B and C which could be 
 ##! analyzed on a different worker.
 ##!
-##! Example line from IRC server indicating that the DCC SEND is about to start:
+
-##!    PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
+# Example line from IRC server indicating that the DCC SEND is about to start:
+#    PRIVMSG my_nick :^ADCC SEND whateverfile.zip 3640061780 1026 41709^A
 
 @load ./main
 @load base/utils/files
@@ -14,24 +15,25 @@
 module IRC;
 
 export {
-	redef enum Tag += { EXTRACTED_FILE };
-
 	## Pattern of file mime types to extract from IRC DCC file transfers.
 	const extract_file_types = /NO_DEFAULT/ &redef;
 
-	## The on-disk prefix for files to be extracted from IRC DCC file transfers.
+	## On-disk prefix for files to be extracted from IRC DCC file transfers.
 	const extraction_prefix = "irc-dcc-item" &redef;
 
 	redef record Info += {
-		dcc_file_name:    string &log &optional;
+		## DCC filename requested.
-		dcc_file_size:    count  &log &optional;
+		dcc_file_name:         string &log &optional;
-		dcc_mime_type:    string &log &optional;
+		## Size of the DCC transfer as indicated by the sender.
+		dcc_file_size:         count  &log &optional;
+		## Sniffed mime type of the file.
+		dcc_mime_type:         string &log &optional;
 		
 		## The file handle for the file to be extracted
-		extraction_file:  file &log &optional;
+		extraction_file:       file &log &optional;
 		
-		## A boolean to indicate if the current file transfer should be extraced.
+		## A boolean to indicate if the current file transfer should be extracted.
-		extract_file:     bool &default=F;
+		extract_file:          bool &default=F;
 		
 		## The count of the number of file that have been extracted during the session.
 		num_extracted_files:   count &default=0;
@@ -54,8 +56,10 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	if ( extract_file_types == irc$dcc_mime_type )
 		{
 		irc$extract_file = T;
-		add irc$tags[EXTRACTED_FILE];
+		}
 		
+	if ( irc$extract_file )
+		{
 		local suffix = fmt("%d.dat", ++irc$num_extracted_files);
 		local fname = generate_extraction_filename(extraction_prefix, c, suffix);
 		irc$extraction_file = open(fname);
@@ -76,7 +80,7 @@ event file_transferred(c: connection, prefix: string, descr: string,
 	Log::write(IRC::LOG, irc);
 	irc$command = tmp;
 
-	if ( irc$extract_file && irc?$extraction_file )
+	if ( irc?$extraction_file )
 		set_contents_file(id, CONTENTS_RESP, irc$extraction_file);
 
 	# Delete these values in case another DCC transfer 

scripts/base/protocols/irc/main.bro

@@ -1,36 +1,38 @@
-##! This is the script that implements the core IRC analysis support.  It only
+##! Implements the core IRC analysis support.  The logging model is to log
-##! logs a very limited subset of the IRC protocol by default.  The points
+##! IRC commands along with the associated response and some additional 
-##! that it logs at are NICK commands, USER commands, and JOIN commands.  It 
+##! metadata about the connection if it's available.
-##! log various bits of meta data as indicated in the :bro:type:`IRC::Info`
-##! record along with the command at the command arguments.
 
 module IRC;
 
 export {
+	
 	redef enum Log::ID += { LOG };
 
-	type Tag: enum { 
-		EMPTY 
-	};
-
 	type Info: record {
+		## Timestamp when the command was seen.
 		ts:       time        &log;
 		uid:      string      &log;
 		id:       conn_id     &log;
+		## Nick name given for the connection.
 		nick:     string      &log &optional;
+		## User name given for the connection.
 		user:     string      &log &optional;
-		channels: set[string] &log &optional;
+		
-		          
+		## Command given by the client.
 		command:  string      &log &optional;
+		## Value for the command given by the client.
 		value:    string      &log &optional;
+		## Any additional data for the command.
 		addl:     string      &log &optional;
-		tags:     set[Tag]    &log;
 	};
 	
+	## Event that can be handled to access the IRC record as it is sent on 
+	## to the logging framework.
 	global irc_log: event(rec: Info);
 }
 
 redef record connection += {
+	## IRC session information.
 	irc:  Info &optional;
 };
 

scripts/base/protocols/ssh/main.bro

@@ -14,15 +14,17 @@
 module SSH;
 
 export {
+	## The SSH protocol logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
 	redef enum Notice::Type += { 
-		## This indicates that a heuristically detected "successful" SSH 
+		## Indicates that a heuristically detected "successful" SSH 
 		## authentication occurred.
 		Login 
 	};
 
 	type Info: record {
+		## Time when the SSH connection began.
 		ts:              time         &log;
 		uid:             string       &log;
 		id:              conn_id      &log;
@@ -34,11 +36,11 @@ export {
 		## would be set for the opposite situation.
 		# TODO: handle local-local and remote-remote better.
 		direction:       Direction    &log &optional;
-		## The software string given by the client.
+		## Software string given by the client.
 		client:          string       &log &optional;
-		## The software string given by the server.
+		## Software string given by the server.
 		server:          string       &log &optional;
-		## The amount of data returned from the server.  This is currently
+		## Amount of data returned from the server.  This is currently
 		## the only measure of the success heuristic and it is logged to 
 		## assist analysts looking at the logs to make their own determination
 		## about the success on a case-by-case basis.
@@ -48,8 +50,8 @@ export {
 		done:            bool         &default=F;
 	};
 	
-	## The size in bytes at which the SSH connection is presumed to be
+	## The size in bytes of data sent by the server at which the SSH 
-	## successful.
+	## connection is presumed to be successful.
 	const authentication_data_size = 5500 &redef;
 	
 	## If true, we tell the event engine to not look at further data
@@ -58,14 +60,16 @@ export {
 	## kinds of analyses (e.g., tracking connection size).
 	const skip_processing_after_detection = F &redef;
 	
-	## This event is generated when the heuristic thinks that a login
+	## Event that is generated when the heuristic thinks that a login
 	## was successful.
 	global heuristic_successful_login: event(c: connection);
 	
-	## This event is generated when the heuristic thinks that a login
+	## Event that is generated when the heuristic thinks that a login
 	## failed.
 	global heuristic_failed_login: event(c: connection);
 	
+	## Event that can be handled to access the :bro:type:`SSH::Info`
+	## record as it is sent on to the logging framework.
 	global log_ssh: event(rec: Info);
 }
 

scripts/base/protocols/ssl/main.bro

@@ -1,3 +1,5 @@
+##! Base SSL analysis script.
+
 @load ./consts
 
 module SSL;
@@ -6,6 +8,7 @@ export {
 	redef enum Log::ID += { LOG };
 
 	type Info: record {
+		## Time when the SSL connection began.
 		ts:               time             &log;
 		uid:              string           &log;
 		id:               conn_id          &log;

scripts/base/protocols/syslog/main.bro

@@ -8,6 +8,7 @@ export {
 	redef enum Log::ID += { LOG };
 	
 	type Info: record {
+		## Timestamp of when the syslog message was seen.
 		ts:        time            &log;
 		uid:       string          &log;
 		id:        conn_id         &log;

scripts/policy/frameworks/control/controllee.bro

@@ -1,3 +1,12 @@
+##! The controllee portion of the control framework.  Load this script if remote
+##! runtime control of the Bro process is desired.
+##!
+##! A controllee only needs to load the controllee script in addition
+##! to the specific analysis scripts desired.  It may also need a node
+##! configured as a controller node in the communications nodes configuration::
+##!
+##!     bro <scripts> frameworks/control/controllee
+
 @load base/frameworks/control
 # If an instance is a controllee, it implicitly needs to listen for remote
 # connections.

scripts/policy/frameworks/control/controller.bro

@@ -1,3 +1,10 @@
+##! This is a utility script that implements the controller interface for the
+##! control framework.  It's intended to be run to control a remote Bro 
+##! and then shutdown.
+##!
+##! It's intended to be used from the command line like this::
+##!     bro <scripts> frameworks/control/controller Control::host=<host_addr> Control::port=<host_port> Control::cmd=<command> [Control::arg=<arg>]
+
 @load base/frameworks/control
 @load base/frameworks/communication
 

scripts/policy/frameworks/metrics/conn-example.bro

@@ -1,3 +1,6 @@
+##! An example of using the metrics framework to collect connection metrics 
+##! aggregated into /24 CIDR ranges.
+
 @load base/frameworks/metrics
 @load base/utils/site
 

scripts/policy/frameworks/metrics/http-example.bro

@@ -1,9 +1,17 @@
+##! Provides an example of aggregating and limiting collection down to 
+##! only local networks.  Additionally, the status code for the response from
+##! the request is added into the metric.
+
 @load base/frameworks/metrics
 @load base/protocols/http
 @load base/utils/site
 
 redef enum Metrics::ID += {
+	## Measures HTTP requests indexed on both the request host and the response
+	## code from the server.
 	HTTP_REQUESTS_BY_STATUS_CODE,
+
+	## Currently unfinished and not working.
 	HTTP_REQUESTS_BY_HOST_HEADER,
 };
 
@@ -11,13 +19,13 @@ event bro_init()
 	{
 	# TODO: these are waiting on a fix with table vals + records before they will work.
 	#Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER, 
-	#                    [$pred(index: Index) = { return Site:is_local_addr(index$host) },
+	#                    [$pred(index: Metrics::Index) = { return Site::is_local_addr(index$host); },
 	#                     $aggregation_mask=24,
-	#                     $break_interval=5mins]);
+	#                     $break_interval=1min]);
-	#
+	
-	## Site::local_nets must be defined in order for this to actually do anything.
+	# Site::local_nets must be defined in order for this to actually do anything.
-	#Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table,
+	Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table,
-	#                                                   $break_interval=5mins]);
+	                                                   $break_interval=1min]);
 	}
 
 event HTTP::log_http(rec: HTTP::Info)

scripts/policy/frameworks/metrics/ssl-example.bro

@@ -1,3 +1,8 @@
+##! Provides an example of using the metrics framework to collect the number
+##! of times a specific server name indicator value is seen in SSL session
+##! establishments.  Names ending in google.com are being filtered out as an
+##! example of the predicate based filtering in metrics filters.
+
 @load base/frameworks/metrics
 @load base/protocols/ssl
 

scripts/policy/frameworks/software/version-changes.bro

@@ -1,3 +1,7 @@
+##! Provides the possibly to define software names that are interesting to 
+##! watch for changes.  A notice is generated if software versions change on a
+##! host.
+
 @load base/frameworks/notice
 @load base/frameworks/software
 
@@ -5,24 +9,17 @@ module Software;
 
 export {
 	redef enum Notice::Type += { 
-		## For certain softwares, a version changing may matter.  In that case, 
+		## For certain software, a version changing may matter.  In that case, 
 		## this notice will be generated.  Software that matters if the version
 		## changes can be configured with the 
 		## :bro:id:`Software::interesting_version_changes` variable.
 		Software_Version_Change,
 	};
 	
-	## Some software is more interesting when the version changes and this
+	## Some software is more interesting when the version changes and this is
 	## a set of all software that should raise a notice when a different 
 	## version is seen on a host.
-	const interesting_version_changes: set[string] = {
+	const interesting_version_changes: set[string] = { } &redef;
-		"SSH"
-	} &redef;
-	
-	## Some software is more interesting when the version changes and this
-	## a set of all software that should raise a notice when a different 
-	## version is seen on a host.
-	const interesting_type_changes: set[string] = {};
 }
 
 event log_software(rec: Info)

scripts/policy/frameworks/software/vulnerable.bro

@@ -1,3 +1,7 @@
+##! Provides a variable to define vulnerable versions of software and if a 
+##! a version of that software as old or older than the defined version a 
+##! notice will be generated.
+
 @load base/frameworks/notice
 @load base/frameworks/software
 
@@ -5,6 +9,7 @@ module Software;
 
 export {
 	redef enum Notice::Type += {
+		## Indicates that a vulnerable version of software was detected.
 		Vulnerable_Version,
 	};
 
@@ -18,6 +23,7 @@ event log_software(rec: Info)
 	if ( rec$name in vulnerable_versions &&
 	     cmp_versions(rec$version, vulnerable_versions[rec$name]) <= 0 )
 		{
-		NOTICE([$note=Vulnerable_Version, $src=rec$host, $msg=software_fmt(rec)]);
+		NOTICE([$note=Vulnerable_Version, $src=rec$host, 
+		        $msg=fmt("A vulnerable version of software was detected: %s", software_fmt(rec))]);
 		}
 	}

scripts/policy/protocols/conn/known-hosts.bro

@@ -8,8 +8,10 @@
 module Known;
 
 export {
+	## The known-hosts logging stream identifier.
 	redef enum Log::ID += { HOSTS_LOG };
-	
+
+	## The record type which contains the column fields of the known-hosts log.
 	type HostsInfo: record {
 		## The timestamp at which the host was detected.
 		ts:      time &log;
@@ -19,7 +21,7 @@ export {
 	};
 	
 	## The hosts whose existence should be logged and tracked.
-	## Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
+	## See :bro:type:`Host` for possible choices.
 	const host_tracking = LOCAL_HOSTS &redef;
 	
 	## The set of all known addresses to store for preventing duplicate 
@@ -28,7 +30,9 @@ export {
 	## Maintain the list of known hosts for 24 hours so that the existence
 	## of each individual address is logged each day.
 	global known_hosts: set[addr] &create_expire=1day &synchronized &redef;
-	
+
+	## An event that can be handled to access the :bro:type:`Known::HostsInfo`
+	## record as it is sent on to the logging framework.
 	global log_known_hosts: event(rec: HostsInfo);
 }
 

scripts/policy/protocols/conn/known-services.bro

@@ -8,29 +8,41 @@
 module Known;
 
 export {
+	## The known-services logging stream identifier.
 	redef enum Log::ID += { SERVICES_LOG };
-	
+
+	## The record type which contains the column fields of the known-services
+	## log.
 	type ServicesInfo: record {
+		## The time at which the service was detected.
 		ts:             time            &log;
+		## The host address on which the service is running.
 		host:           addr            &log;
+		## The port number on which the service is running.
 		port_num:       port            &log;
+		## The transport-layer protocol which the service uses.
 		port_proto:     transport_proto &log;
+		## A set of protocols that match the service's connection payloads.
 		service:        set[string]     &log;
-		
-		done:           bool &default=F;
 	};
 	
 	## The hosts whose services should be tracked and logged.
+	## See :bro:type:`Host` for possible choices.
 	const service_tracking = LOCAL_HOSTS &redef;
-	
+
+	## Tracks the set of daily-detected services for preventing the logging
+	## of duplicates, but can also be inspected by other scripts for
+	## different purposes.
 	global known_services: set[addr, port] &create_expire=1day &synchronized;
-	
+
+	## Event that can be handled to access the :bro:type:`Known::ServicesInfo`
+	## record as it is sent on to the logging framework.
 	global log_known_services: event(rec: ServicesInfo);
 }
 
 redef record connection += {
-	## This field is to indicate whether or not the processing for detecting 
+	# This field is to indicate whether or not the processing for detecting 
-	## and logging the service for this connection is complete.
+	# and logging the service for this connection is complete.
 	known_services_done: bool &default=F;
 };
 

scripts/policy/protocols/ftp/detect.bro

@@ -7,7 +7,7 @@ module FTP;
 
 export {
 	redef enum Notice::Type += {
-		## This indicates that a successful response to a "SITE EXEC" 
+		## Indicates that a successful response to a "SITE EXEC" 
 		## command/arg pair was seen.
 		Site_Exec_Success,
 	};

scripts/policy/protocols/ftp/software.bro

@@ -12,8 +12,10 @@ module FTP;
 
 export {
 	redef enum Software::Type += {
-		FTP_CLIENT,
+		## Identifier for FTP clients in the software framework.
-		FTP_SERVER,
+		CLIENT,
+		## Not currently implemented.
+		SERVER,
 	};
 }
 
@@ -21,7 +23,7 @@ event ftp_request(c: connection, command: string, arg: string) &priority=4
 	{
 	if ( command == "CLNT" )
 		{
-		local si = Software::parse(arg, c$id$orig_h, FTP_CLIENT);
+		local si = Software::parse(arg, c$id$orig_h, CLIENT);
 		Software::found(c$id, si);
 		}
 	}

scripts/policy/protocols/http/detect-MHR.bro

@@ -1,8 +1,8 @@
-##! This script takes MD5 sums of files transferred over HTTP and checks them with
+##! Detect file downloads over HTTP that have MD5 sums matching files in Team 
-##! Team Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
+##! Cymru's Malware Hash Registry (http://www.team-cymru.org/Services/MHR/).
 ##! By default, not all file transfers will have MD5 sums calculated.  Read the
-##! documentation for the :doc:base/protocols/http/file-hash.bro script to see how to 
+##! documentation for the :doc:base/protocols/http/file-hash.bro script to see
-##! configure which transfers will have hashes calculated.
+##! how to configure which transfers will have hashes calculated.
 
 @load base/frameworks/notice
 @load base/protocols/http

scripts/policy/protocols/http/detect-intel.bro

@@ -1,4 +1,4 @@
-##! Intelligence based HTTP detections.
+##! Intelligence based HTTP detections.  Not yet working!
 
 @load base/protocols/http/main
 @load base/protocols/http/utils

scripts/policy/protocols/http/detect-sqli.bro

@@ -16,7 +16,9 @@ export {
 	};
 	
 	redef enum Metrics::ID += {
+		## Metric to track SQL injection attackers.
 		SQLI_ATTACKER,
+		## Metrics to track SQL injection victims.
 		SQLI_VICTIM,
 	};
 
@@ -30,7 +32,7 @@ export {
 		COOKIE_SQLI,
 	};
 	
-	## This defines the threshold that determines if an SQL injection attack
+	## Defines the threshold that determines if an SQL injection attack
 	## is ongoing based on the number of requests that appear to be SQL 
 	## injection attacks.
 	const sqli_requests_threshold = 50 &redef;
@@ -40,7 +42,7 @@ export {
 	## At the end of each interval the counter is reset.
 	const sqli_requests_interval = 5min &redef;
 
-	## This regular expression is used to match URI based SQL injections
+	## Regular expression is used to match URI based SQL injections.
 	const match_sql_injection_uri = 
 		  /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/
 		| /[\?&][^[:blank:]\x00-\x37\|]+?=[\-0-9%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+([xX]?[oO][rR]|[nN]?[aA][nN][dD])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+['"]?(([^a-zA-Z&]+)?=|[eE][xX][iI][sS][tT][sS])/

scripts/policy/protocols/http/detect-webapps.bro

@@ -1,3 +1,5 @@
+##! Detect and log web applications through the software framework.
+
 @load base/frameworks/signatures
 @load base/frameworks/software
 @load base/protocols/http
@@ -10,10 +12,12 @@ redef Signatures::ignored_ids += /^webapp-/;
 
 export {
 	redef enum Software::Type += {
+		## Identifier for web applications in the software framework.
 		WEB_APPLICATION,
 	};
 
 	redef record Software::Info += {
+		## Most root URL where the software was discovered.
 		url:   string &optional &log;
 	};
 }

scripts/policy/protocols/http/software-browser-plugins.bro

@@ -1,5 +1,5 @@
-##! This script take advantage of a few ways that installed plugin information
+##! Detect browser plugins as they leak through requests to Omniture 
-##! leaks from web browsers.
+##! advertising servers.
 
 @load base/protocols/http
 @load base/frameworks/software
@@ -13,6 +13,7 @@ export {
 	};
 	
 	redef enum Software::Type += {
+		## Identifier for browser plugins in the software framework.
 		BROWSER_PLUGIN
 	};
 }

scripts/policy/protocols/http/software.bro

@@ -6,8 +6,11 @@ module HTTP;
 
 export {
 	redef enum Software::Type += {
+		## Identifier for web servers in the software framework.
 		SERVER,
+		## Identifier for app servers in the software framework.
 		APPSERVER,
+		## Identifier for web browsers in the software framework.
 		BROWSER,
 	};
 

scripts/policy/protocols/http/var-extraction-cookies.bro

@@ -1,4 +1,4 @@
-##! This script extracts and logs variables from cookies sent by clients
+##! Extracts and logs variables names from cookies sent by clients.
 
 @load base/protocols/http/main
 @load base/protocols/http/utils
@@ -6,6 +6,7 @@
 module HTTP;
 
 redef record Info += {
+	## Variable names extracted from all cookies.
 	cookie_vars: vector of string &optional &log;
 };
 

scripts/policy/protocols/http/var-extraction-uri.bro

@@ -1,10 +1,12 @@
-##! This script extracts and logs variables from the requested URI
+##! Extracts and log variables from the requested URI in the default HTTP 
+##! logging stream.
 
 @load base/protocols/http
 
 module HTTP;
 
 redef record Info += {
+	## Variable names from the URI.
 	uri_vars:    vector of string &optional &log;
 };
 

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -1,3 +1,5 @@
+##! Detect hosts which are doing password guessing attacks and/or password
+##! bruteforcing over SSH.
 
 @load base/protocols/ssh
 @load base/frameworks/metrics
@@ -13,13 +15,13 @@ export {
 		## determined failed logins.
 		Password_Guessing,
 		## Indicates that a host previously identified as a "password guesser"
-		## has now had a heuristically successful login attempt.
+		## has now had a heuristically successful login attempt.  This is not
+		## currently implemented.
 		Login_By_Password_Guesser,
 	};
 	
 	redef enum Metrics::ID  += {
-		## This metric is to measure failed logins with the hope of detecting
+		## Metric is to measure failed logins.
-		## bruteforcing hosts.
 		FAILED_LOGIN,
 	};
 
@@ -37,7 +39,7 @@ export {
 	## client subnets and the yield value represents server subnets.
 	const ignore_guessers: table[subnet] of subnet &redef;
 
-	## Keeps track of hosts identified as guessing passwords.
+	## Tracks hosts identified as guessing passwords.
 	global password_guessers: set[addr] 
 		&read_expire=guessing_timeout+1hr &synchronized &redef;
 }

scripts/policy/protocols/ssh/geo-data.bro

@@ -1,5 +1,4 @@
-##! This implements all of the additional information and geodata detections 
+##! Geodata based detections for SSH analysis.
-##! for SSH analysis.
 
 @load base/frameworks/notice
 @load base/protocols/ssh
@@ -19,8 +18,8 @@ export {
 		remote_location: geo_location &log &optional;
 	};
 	
-	## The set of countries for which you'd like to throw notices upon 
+	## The set of countries for which you'd like to generate notices upon 
-	## successful login
+	## successful login.
 	const watched_countries: set[string] = {"RO"} &redef;
 }
 

scripts/policy/protocols/ssh/interesting-hostnames.bro

@@ -10,7 +10,7 @@ module SSH;
 
 export {
 	redef enum Notice::Type += {
-		## Generated if a login originates or responds with a host and the 
+		## Generated if a login originates or responds with a host where the 
 		## reverse hostname lookup resolves to a name matched by the 
 		## :bro:id:`SSH::interesting_hostnames` regular expression.
 		Interesting_Hostname_Login,

scripts/policy/protocols/ssh/software.bro

@@ -1,4 +1,4 @@
-##! This script extracts SSH client and server information from SSH 
+##! Extracts SSH client and server information from SSH 
 ##! connections and forwards it to the software framework.
 
 @load base/frameworks/software
@@ -7,7 +7,9 @@ module SSH;
 
 export {
 	redef enum Software::Type += {
+		## Identifier for SSH clients in the software framework.
 		SERVER,
+		## Identifier for SSH servers in the software framework.
 		CLIENT,
 	};
 }

scripts/site/local-manager.bro

@@ -1,9 +1 @@
-##! Local site policy loaded only by the manager in a cluster.
+##! Local site policy loaded only by the manager if Bro is running as a cluster.
-
-@load base/frameworks/notice
-
-# If you are running a cluster you should define your Notice::policy here 
-# so that notice processing occurs on the manager.
-redef Notice::policy += {
-
-};

scripts/site/local-proxy.bro

@@ -1,2 +1 @@
 ##! Local site policy loaded only by the proxies if Bro is running as a cluster.
-

scripts/site/local.bro

@@ -1,22 +1,29 @@
-##! Local site policy. Customize as appropriate. This file will not be 
+##! Local site policy. Customize as appropriate. 
-##! overwritten when upgrading or reinstalling.
+##!
+##! This file will not be overwritten when upgrading or reinstalling!
 
-# Load the script to log which script were loaded during each run
+# This script logs which scripts were loaded during each run.
 @load misc/loaded-scripts
 
 # Apply the default tuning scripts for common tuning settings.
 @load tuning/defaults
 
-# Vulnerable versions of software to generate notices for when discovered.
+# Generate notices when vulnerable versions of software are discovered.
 # The default is to only monitor software found in the address space defined
 # as "local".  Refer to the software framework's documentation for more 
 # information.
 @load frameworks/software/vulnerable
+
+# Example vulnerable software.  This needs to be updated and maintained over
+# time as new vulnerabilities are discovered.
 redef Software::vulnerable_versions += {
 	["Flash"] = [$major=10,$minor=2,$minor2=153,$addl="1"],
 	["Java"] = [$major=1,$minor=6,$minor2=0,$addl="22"],
 };
 
+# Detect software changing (e.g. attacker installing hacked SSHD).
+@load frameworks/software/version-changes
+
 # This adds signatures to detect cleartext forward and reverse windows shells.
 redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
@@ -25,13 +32,15 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 # redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 0] };
 
 # Load all of the scripts that detect software in various protocols.
-@load protocols/http/software
-#@load protocols/http/detect-webapps
 @load protocols/ftp/software
 @load protocols/smtp/software
 @load protocols/ssh/software
+@load protocols/http/software
+# The detect-webapps script could possibly cause performance trouble when 
+# running on live traffic.  Enable it cautiously.
+#@load protocols/http/detect-webapps
 
-# Load the script to detect DNS results pointing toward your Site::local_nets 
+# This script detects DNS results pointing toward your Site::local_nets 
 # where the name is not part of your local DNS zone and is being hosted 
 # externally.  Requires that the Site::local_zones variable is defined.
 @load protocols/dns/detect-external-names
@@ -39,15 +48,12 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 # Script to detect various activity in FTP sessions.
 @load protocols/ftp/detect
 
-# Detect software changing (e.g. attacker installing hacked SSHD).
-@load frameworks/software/version-changes
-
 # Scripts that do asset tracking.
 @load protocols/conn/known-hosts
 @load protocols/conn/known-services
 @load protocols/ssl/known-certs
 
-# Load the script to enable SSL/TLS certificate validation.
+# This script enables SSL/TLS certificate validation.
 @load protocols/ssl/validate-certs
 
 # If you have libGeoIP support built in, do some geographic detections and 
@@ -60,5 +66,5 @@ redef signature_files += "frameworks/signatures/detect-windows-shells.sig";
 
 # Detect MD5 sums in Team Cymru's Malware Hash Registry.
 @load protocols/http/detect-MHR
-# Detect SQL injection attacks
+# Detect SQL injection attacks.
 @load protocols/http/detect-sqli

src/bro.bif

@@ -1,4 +1,8 @@
-##! Defines most of the built-in functions accessible to the scripting layer.
+##! A collection of built-in functions that implement a variety of things
+##! such as general programming algorithms, string processing, math functions,
+##! introspection, type conversion, file/directory manipulation, packet
+##! filtering, inter-process communication and controlling protocol analyzer
+##! behavior.
 
 %%{ // C segment
 #include <math.h>

src/const.bif

@@ -1,5 +1,6 @@
-##! Constants of used with built-in functions and events. Documented in
+##! Declaration of various scripting-layer constants that the Bro core uses
-##! :doc:`/scripts/base/init-bare`. 
+##! internally.  Documentation and default values for the scripting-layer
+##! variables themselves are found in :doc:`/scripts/base/init-bare`.
 
 const ignore_keep_alive_rexmit: bool;
 const skip_http_data: bool;

src/event.bif

@@ -1,4 +1,7 @@
-##! Events generated by Bro's internal event engine.
+##! The events that the C/C++ core of Bro can generate.  This is mostly
+##! consisting of high-level network events that protocol analyzers detect,
+##! but there are also several general-utility events generated by internal
+##! Bro frameworks.
 
 #
 # Documentation conventions:
@@ -4665,6 +4668,7 @@ event ssl_established%(c: connection%);
 ##       defined as part of the SSL/TLS protocol.
 ##
 ## .. bro:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
+<<<<<<< HEAD
 ##    ssl_session_ticket_handshake x509_certificate x509_error x509_extension
 event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%);
 
@@ -4685,6 +4689,24 @@ event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%);
 ##
 ## .. bro:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
 ##    x509_certificate x509_error x509_extension ssl_alert
+=======
+##    x509_certificate x509_error x509_extension
+event ssl_alert%(c: connection, is_orig: bool, level: count, desc: count%);
+
+## Generated for SSL/TLS session ticket handshake messages that are a part
+## of the stateless-server session resumption mechanism as described by
+## :rfc:`4507`.
+##
+## c: The connection.
+##
+## ticket_lifetime_hint: A hint from the server about how long the ticket
+##                       should be stored by the client.
+##
+## ticket: The ticket data.
+##
+## .. bro:see::  ssl_client_hello ssl_established ssl_extension ssl_server_hello
+##    x509_certificate x509_error x509_extension
+>>>>>>> fd74eb8e308915fd6766ec221acfd15633361568
 event ssl_session_ticket_handshake%(c: connection, ticket_lifetime_hint: count, ticket: string%);
 
 ## Generated for x509 certificates seen in SSL/TLS connections. During the initial

src/strings.bif

@@ -1,4 +1,5 @@
-##! Built-in functions for working with strings.
+##! Definitions of built-in functions related to string processing and
+##! manipulation.
 
 
 %%{ // C segment

src/types.bif

@@ -1,4 +1,4 @@
-##! Set of predefined types used with built-in functions and events.
+##! Declaration of various types that the Bro core uses internally.
 
 enum dce_rpc_ptype %{
 	DCE_RPC_REQUEST,
@@ -135,8 +135,8 @@ enum createmode_t %{
 	EXCLUSIVE = 2,
 %}
 
-# Decleare record types that we want to access from the even engine. These are
+# Declare record types that we want to access from the event engine. These are
-# defined in bro.init.
+# defined in init-bare.bro.
 type info_t: record;
 type fattr_t: record;
 type diropargs_t: record;

testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	channels	command	value	addl	tags	dcc_file_name	dcc_file_size	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	extraction_file
-#types	time	string	addr	port	addr	port	string	string	table[string]	string	string	string	table[enum]	string	count	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	file
-1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	-	NICK	bloed	-	(empty)	-	-	-
+1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-
-1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	(empty)	-	-	-
+1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-
-1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	-	JOIN	#easymovies	(empty)	(empty)	-	-	-
+1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-
-1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	-	DCC	#easymovies	(empty)	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	-
+1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	-

testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log

@@ -3,9 +3,9 @@
 #empty_field	(empty)
 #unset_field	-
 #path	irc
-#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	channels	command	value	addl	tags	dcc_file_name	dcc_file_size	dcc_mime_type	extraction_file
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	nick	user	command	value	addl	dcc_file_name	dcc_file_size	dcc_mime_type	extraction_file
-#types	time	string	addr	port	addr	port	string	string	table[string]	string	string	string	table[enum]	string	count	string	file
+#types	time	string	addr	port	addr	port	string	string	string	string	string	string	count	string	file
-1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	-	NICK	bloed	-	(empty)	-	-	-	-
+1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	-	-	NICK	bloed	-	-	-	-	-
-1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	(empty)	-	-	-	-
+1311189164.119437	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	-	USER	sdkfje	sdkfje Montreal.QC.CA.Undernet.org dkdkrwq	-	-	-	-
-1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	-	JOIN	#easymovies	(empty)	(empty)	-	-	-	-
+1311189174.474127	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	JOIN	#easymovies	(empty)	-	-	-	-
-1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	-	DCC	#easymovies	(empty)	IRC::EXTRACTED_FILE	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat
+1311189316.326025	UWkUyAuUGXf	192.168.1.77	57640	66.198.80.67	6667	bloed	sdkfje	DCC	#easymovies	(empty)	ladyvampress-default(2011-07-07)-OS.zip	42208	FAKE_MIME	irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat