Minor documentation cleanups.

scripts/base/frameworks/intel/cluster.bro

@@ -1,8 +1,8 @@
 ##! Cluster transparency support for the intelligence framework.  This is mostly
 ##! oriented toward distributing intelligence information across clusters.
 
+@load ./main
 @load base/frameworks/cluster
-@load ./input
 
 module Intel;
 
@@ -17,14 +17,17 @@ redef record Item += {
 redef have_full_data = F;
 @endif
 
+# Internal event for cluster data distribution.
 global cluster_new_item: event(item: Item);
 
-# Primary intelligence distribution comes from manager.
+# Primary intelligence management is done by the manager:
+# The manager informs the workers about new items and item removal.
 redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/;
-# If a worker finds intelligence and adds it, it should share it back to the manager.
+# A worker queries the manager to insert, remove or indicate the match of an item.
 redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/;
 
 @if ( Cluster::local_node_type() == Cluster::MANAGER )
+# Handling of new worker nodes.
 event remote_connection_handshake_done(p: event_peer)
 	{
 	# When a worker connects, send it the complete minimal data store.
@@ -35,25 +38,21 @@ event remote_connection_handshake_done(p: event_peer)
 		}
 	}
 
+# Handling of matches triggered by worker nodes.
 event Intel::match_no_items(s: Seen) &priority=5
 	{
 	if ( Intel::find(s) )
 		event Intel::match(s, Intel::get_items(s));
 	}
 
+# Handling of item removal triggered by worker nodes.
 event Intel::remove_item(item: Item, purge_indicator: bool)
 	{
 	remove(item, purge_indicator);
 	}
 @endif
 
-event Intel::cluster_new_item(item: Intel::Item) &priority=5
+# Handling of item insertion.
-	{
-	# Ignore locally generated events to avoid event storms.
-	if ( is_remote_event() )
-		Intel::insert(item);
-	}
-
 event Intel::new_item(item: Intel::Item) &priority=5
 	{
 	# The cluster manager always rebroadcasts intelligence.
@@ -65,3 +64,11 @@ event Intel::new_item(item: Intel::Item) &priority=5
 		event Intel::cluster_new_item(item);
 		}
 	}
+
+# Handling of item insertion by remote node.
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}

scripts/base/frameworks/intel/files.bro

@@ -22,8 +22,8 @@ export {
 		## should go here to provide context to the data.
 		f:              fa_file  &optional;
 		## If the data was discovered within a file, the file uid should
-		## go here to provide context to the data. If the *f* field is
+		## go here to provide context to the data. If the file record *f*
-		## provided, this will be automatically filled out.
+		## is provided, this will be automatically filled out.
 		fuid:           string   &optional;
 	};
 

scripts/base/frameworks/intel/input.bro

@@ -1,11 +1,14 @@
+##! Input handling for the intelligence framework. This script implements the
+##! import of intelligence data from files using the input framework.
+
 @load ./main
 
 module Intel;
 
 export {
-	## Intelligence files that will be read off disk.  The files are
+	## Intelligence files that will be read off disk. The files are
-	## reread every time they are updated so updates must be atomic with
+	## reread every time they are updated so updates must be atomic
-	## "mv" instead of writing the file in place.
+	## with "mv" instead of writing the file in place.
 	const read_files: set[string] = {} &redef;
 }
 

scripts/base/frameworks/intel/main.bro

@@ -1,7 +1,6 @@
-##! The intelligence framework provides a way to store and query IP addresses,
+##! The intelligence framework provides a way to store and query intelligence data
-##! and strings (with a str_type).  Metadata can
+##! (e.g. IP addresses, URLs and hashes). The intelligence items can be associated
-##! also be associated with the intelligence, like for making more informed
+##! with metadata to allow informed decisions about matching and handling.
-##! decisions about matching and handling of intelligence.
 
 @load base/frameworks/notice
 
@@ -31,15 +30,15 @@ export {
 		## Public key MD5 hash. (SSH server host keys are a good example.)
 		PUBKEY_HASH,
 	};
+
 	## Set of intelligence data types.
 	type TypeSet: set[Type];
 
 	## Data about an :bro:type:`Intel::Item`.
 	type MetaData: record {
-		## An arbitrary string value representing the data source.
+		## An arbitrary string value representing the data source. This
-		## Typically, the convention for this field will be the source
+		## value is used as unique key to identify a metadata record in
-		## name and feed name separated by a hyphen.
+		## the scope of a single intelligence item.
-		## For example: "source1-c&c".
 		source:      string;
 		## A freeform description for the data.
 		desc:        string      &optional;
@@ -55,7 +54,7 @@ export {
 		## The type of data that the indicator field represents.
 		indicator_type: Type;
 		
-		## Metadata for the item.  Typically represents more deeply
+		## Metadata for the item. Typically represents more deeply
 		## descriptive data for a piece of intelligence.
 		meta:           MetaData;
 	};
@@ -117,11 +116,14 @@ export {
 		sources:  set[string]    &log &default=string_set();
 	};
 
-	## Intelligence data manipulation function.
+	## Function to insert intelligence data. If the indicator is already
+	## present, the associated metadata will be added to the indicator. If
+	## the indicator already contains a metadata record from the same source,
+	## the existing metadata record will be updated.
 	global insert: function(item: Item);
 
 	## Function to remove intelligence data. If purge_indicator is set, the
-	## given meta data is ignored and the indicator is removed completely.
+	## given metadata is ignored and the indicator is removed completely.
 	global remove: function(item: Item, purge_indicator: bool &default = F);
 
 	## Function to declare discovery of a piece of data in order to check
@@ -129,16 +131,17 @@ export {
 	global seen: function(s: Seen);
 
 	## Event to represent a match in the intelligence data from data that
-	## was seen.  On clusters there is no assurance as to where this event
+	## was seen. On clusters there is no assurance as to when this event
 	## will be generated so do not assume that arbitrary global state beyond
 	## the given data will be available.
 	##
-	## This is the primary mechanism where a user will take actions based on
+	## This is the primary mechanism where a user may take actions based on
-	## data within the intelligence framework.
+	## data provided by the intelligence framework.
 	global match: event(s: Seen, items: set[Item]);
 
-	## This hook can be used to extend the intel log by adding data to the
+	## This hook can be used to influence the logging of intelligence hits
-	## Info record. The default information is added with a priority of 5.
+	## (e.g. by adding data to the Info record). The default information is
+	## added with a priority of 5.
 	##
 	## info: The Info record that will be logged.
 	##
@@ -161,7 +164,7 @@ export {
 	##
 	## indicator_type: The indicator type of the expired item.
 	##
-	## metas: The set of meta data describing the expired item.
+	## metas: The set of metadata describing the expired item.
 	##
 	## If all hook handlers are executed, the expiration timeout will be reset.
 	## Otherwise, if one of the handlers terminates using break, the item will
@@ -183,7 +186,7 @@ global purge_item: event(item: Item);
 # if this is a cluster deployment or not.
 const have_full_data = T &redef;
 
-# Table of meta data, indexed by source string.
+# Table of metadata, indexed by source string.
 type MetaDataTable: table[string] of MetaData;
 
 # Expiration handlers.
@@ -215,21 +218,6 @@ event bro_init() &priority=5
 	Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]);
 	}
 
-function find(s: Seen): bool
-	{
-	local ds = have_full_data ? data_store : min_data_store;
-
-	if ( s?$host )
-		{
-		return ((s$host in ds$host_data) ||
-		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
-		}
-	else
-		{
-		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
-		}
-	}
-
 # Function that abstracts expiration of different types.
 function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval
 	{
@@ -275,7 +263,24 @@ function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any
 	return expire_item(indicator, indicator_type, metas);
 	}
 
-# Function to abstract from different data stores for different indicator types.
+# Function to check for intelligence hits.
+function find(s: Seen): bool
+	{
+	local ds = have_full_data ? data_store : min_data_store;
+
+	if ( s?$host )
+		{
+		return ((s$host in ds$host_data) ||
+		        (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0));
+		}
+	else
+		{
+		return ([to_lower(s$indicator), s$indicator_type] in ds$string_data);
+		}
+	}
+
+# Function to retrieve intelligence items while abstracting from different
+# data stores for different indicator types.
 function get_items(s: Seen): set[Item]
 	{
 	local return_data: set[Item];
@@ -365,7 +370,7 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
 
 function insert(item: Item)
 	{
-	# Create and fill out the meta data item.
+	# Create and fill out the metadata item.
 	local meta = item$meta;
 	local meta_tbl: table [string] of MetaData;
 	local is_new: bool = T;
@@ -420,7 +425,7 @@ function insert(item: Item)
 
 	if ( have_full_data )
 		{
-		# Insert new meta data or update if already present
+		# Insert new metadata or update if already present
 		meta_tbl[meta$source] = meta;
 		}
 
@@ -430,8 +435,8 @@ function insert(item: Item)
 		event Intel::new_item(item);
 	}
 
-# Function to remove meta data of an item. The function returns T
+# Function to remove metadata of an item. The function returns T
-# if there is no meta data left for the given indicator.
+# if there is no metadata left for the given indicator.
 function remove_meta_data(item: Item): bool
 	{
 	if ( ! have_full_data )
@@ -466,7 +471,7 @@ function remove(item: Item, purge_indicator: bool)
 		return;
 		}
 
-	# Remove meta data from manager's data store
+	# Remove metadata from manager's data store
 	local no_meta_data = remove_meta_data(item);
 	# Remove whole indicator if necessary
 	if ( no_meta_data || purge_indicator )
@@ -485,14 +490,14 @@ function remove(item: Item, purge_indicator: bool)
 				delete data_store$string_data[item$indicator, item$indicator_type];
 				break;
 			}
-		# Trigger deletion in min data stores
+		# Trigger deletion in minimal data stores
 		event Intel::purge_item(item);
 		}
 	}
 
+# Handling of indicator removal in minimal data stores.
 event purge_item(item: Item)
 	{
-	# Remove data from min data store
 	switch ( item$indicator_type )
 		{
 		case ADDR:

scripts/policy/frameworks/intel/do_expire.bro

@@ -1,10 +1,14 @@
+##! This script enables expiration for intelligence items.
 
 @load base/frameworks/intel
 
 module Intel;
 
+redef item_expiration = 10min;
+
 hook item_expired(indicator: string, indicator_type: Type,
 	metas: set[MetaData]) &priority=-10
 	{
+	# Trigger removal of the expired item.
 	break;
 	}

scripts/policy/frameworks/intel/do_notice.bro

@@ -1,3 +1,4 @@
+##! This script enables notice generation for intelligence matches.
 
 @load base/frameworks/intel
 @load base/frameworks/notice

scripts/policy/frameworks/intel/whitelist.bro

@@ -1,12 +1,12 @@
+##! This script enables whitelisting for intelligence items.
 
 @load base/frameworks/intel
-@load base/frameworks/notice
 
 module Intel;
 
 export {
 	redef record Intel::MetaData += {
-		## Add a field to indicate if this is a whitelisted item.
+		## A boolean value to indicate whether the item is whitelisted.
 		whitelist: bool &default=F;
 	};
 }