diff --git a/scripts/base/frameworks/intel/cluster.bro b/scripts/base/frameworks/intel/cluster.bro index e8fae8327c..0727fb6746 100644 --- a/scripts/base/frameworks/intel/cluster.bro +++ b/scripts/base/frameworks/intel/cluster.bro @@ -1,8 +1,8 @@ ##! Cluster transparency support for the intelligence framework. This is mostly ##! oriented toward distributing intelligence information across clusters. +@load ./main @load base/frameworks/cluster -@load ./input module Intel; @@ -17,14 +17,17 @@ redef record Item += { redef have_full_data = F; @endif +# Internal event for cluster data distribution. global cluster_new_item: event(item: Item); -# Primary intelligence distribution comes from manager. +# Primary intelligence management is done by the manager: +# The manager informs the workers about new items and item removal. redef Cluster::manager2worker_events += /^Intel::(cluster_new_item|purge_item)$/; -# If a worker finds intelligence and adds it, it should share it back to the manager. +# A worker queries the manager to insert, remove or indicate the match of an item. redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|remove_item|match_no_items)$/; @if ( Cluster::local_node_type() == Cluster::MANAGER ) +# Handling of new worker nodes. event remote_connection_handshake_done(p: event_peer) { # When a worker connects, send it the complete minimal data store. @@ -35,25 +38,21 @@ event remote_connection_handshake_done(p: event_peer) } } +# Handling of matches triggered by worker nodes. event Intel::match_no_items(s: Seen) &priority=5 { if ( Intel::find(s) ) event Intel::match(s, Intel::get_items(s)); } +# Handling of item removal triggered by worker nodes. event Intel::remove_item(item: Item, purge_indicator: bool) { remove(item, purge_indicator); } @endif -event Intel::cluster_new_item(item: Intel::Item) &priority=5 - { - # Ignore locally generated events to avoid event storms. - if ( is_remote_event() ) - Intel::insert(item); - } - +# Handling of item insertion. event Intel::new_item(item: Intel::Item) &priority=5 { # The cluster manager always rebroadcasts intelligence. @@ -65,3 +64,11 @@ event Intel::new_item(item: Intel::Item) &priority=5 event Intel::cluster_new_item(item); } } + +# Handling of item insertion by remote node. +event Intel::cluster_new_item(item: Intel::Item) &priority=5 + { + # Ignore locally generated events to avoid event storms. + if ( is_remote_event() ) + Intel::insert(item); + } diff --git a/scripts/base/frameworks/intel/files.bro b/scripts/base/frameworks/intel/files.bro index 89d708cc1e..b786a6fefb 100644 --- a/scripts/base/frameworks/intel/files.bro +++ b/scripts/base/frameworks/intel/files.bro @@ -22,8 +22,8 @@ export { ## should go here to provide context to the data. f: fa_file &optional; ## If the data was discovered within a file, the file uid should - ## go here to provide context to the data. If the *f* field is - ## provided, this will be automatically filled out. + ## go here to provide context to the data. If the file record *f* + ## is provided, this will be automatically filled out. fuid: string &optional; }; diff --git a/scripts/base/frameworks/intel/input.bro b/scripts/base/frameworks/intel/input.bro index 7b494dcd75..9c4d033627 100644 --- a/scripts/base/frameworks/intel/input.bro +++ b/scripts/base/frameworks/intel/input.bro @@ -1,11 +1,14 @@ +##! Input handling for the intelligence framework. This script implements the +##! import of intelligence data from files using the input framework. + @load ./main module Intel; export { - ## Intelligence files that will be read off disk. The files are - ## reread every time they are updated so updates must be atomic with - ## "mv" instead of writing the file in place. + ## Intelligence files that will be read off disk. The files are + ## reread every time they are updated so updates must be atomic + ## with "mv" instead of writing the file in place. const read_files: set[string] = {} &redef; } diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index 027eaaf489..bc7c9187fe 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -1,7 +1,6 @@ -##! The intelligence framework provides a way to store and query IP addresses, -##! and strings (with a str_type). Metadata can -##! also be associated with the intelligence, like for making more informed -##! decisions about matching and handling of intelligence. +##! The intelligence framework provides a way to store and query intelligence data +##! (e.g. IP addresses, URLs and hashes). The intelligence items can be associated +##! with metadata to allow informed decisions about matching and handling. @load base/frameworks/notice @@ -31,15 +30,15 @@ export { ## Public key MD5 hash. (SSH server host keys are a good example.) PUBKEY_HASH, }; + ## Set of intelligence data types. type TypeSet: set[Type]; ## Data about an :bro:type:`Intel::Item`. type MetaData: record { - ## An arbitrary string value representing the data source. - ## Typically, the convention for this field will be the source - ## name and feed name separated by a hyphen. - ## For example: "source1-c&c". + ## An arbitrary string value representing the data source. This + ## value is used as unique key to identify a metadata record in + ## the scope of a single intelligence item. source: string; ## A freeform description for the data. desc: string &optional; @@ -55,7 +54,7 @@ export { ## The type of data that the indicator field represents. indicator_type: Type; - ## Metadata for the item. Typically represents more deeply + ## Metadata for the item. Typically represents more deeply ## descriptive data for a piece of intelligence. meta: MetaData; }; @@ -117,11 +116,14 @@ export { sources: set[string] &log &default=string_set(); }; - ## Intelligence data manipulation function. + ## Function to insert intelligence data. If the indicator is already + ## present, the associated metadata will be added to the indicator. If + ## the indicator already contains a metadata record from the same source, + ## the existing metadata record will be updated. global insert: function(item: Item); ## Function to remove intelligence data. If purge_indicator is set, the - ## given meta data is ignored and the indicator is removed completely. + ## given metadata is ignored and the indicator is removed completely. global remove: function(item: Item, purge_indicator: bool &default = F); ## Function to declare discovery of a piece of data in order to check @@ -129,16 +131,17 @@ export { global seen: function(s: Seen); ## Event to represent a match in the intelligence data from data that - ## was seen. On clusters there is no assurance as to where this event + ## was seen. On clusters there is no assurance as to when this event ## will be generated so do not assume that arbitrary global state beyond ## the given data will be available. ## - ## This is the primary mechanism where a user will take actions based on - ## data within the intelligence framework. + ## This is the primary mechanism where a user may take actions based on + ## data provided by the intelligence framework. global match: event(s: Seen, items: set[Item]); - ## This hook can be used to extend the intel log by adding data to the - ## Info record. The default information is added with a priority of 5. + ## This hook can be used to influence the logging of intelligence hits + ## (e.g. by adding data to the Info record). The default information is + ## added with a priority of 5. ## ## info: The Info record that will be logged. ## @@ -161,7 +164,7 @@ export { ## ## indicator_type: The indicator type of the expired item. ## - ## metas: The set of meta data describing the expired item. + ## metas: The set of metadata describing the expired item. ## ## If all hook handlers are executed, the expiration timeout will be reset. ## Otherwise, if one of the handlers terminates using break, the item will @@ -183,7 +186,7 @@ global purge_item: event(item: Item); # if this is a cluster deployment or not. const have_full_data = T &redef; -# Table of meta data, indexed by source string. +# Table of metadata, indexed by source string. type MetaDataTable: table[string] of MetaData; # Expiration handlers. @@ -215,21 +218,6 @@ event bro_init() &priority=5 Log::create_stream(LOG, [$columns=Info, $ev=log_intel, $path="intel"]); } -function find(s: Seen): bool - { - local ds = have_full_data ? data_store : min_data_store; - - if ( s?$host ) - { - return ((s$host in ds$host_data) || - (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0)); - } - else - { - return ([to_lower(s$indicator), s$indicator_type] in ds$string_data); - } - } - # Function that abstracts expiration of different types. function expire_item(indicator: string, indicator_type: Type, metas: set[MetaData]): interval { @@ -275,7 +263,24 @@ function expire_string_data(data: table[string, Type] of MetaDataTable, idx: any return expire_item(indicator, indicator_type, metas); } -# Function to abstract from different data stores for different indicator types. +# Function to check for intelligence hits. +function find(s: Seen): bool + { + local ds = have_full_data ? data_store : min_data_store; + + if ( s?$host ) + { + return ((s$host in ds$host_data) || + (|matching_subnets(addr_to_subnet(s$host), ds$subnet_data)| > 0)); + } + else + { + return ([to_lower(s$indicator), s$indicator_type] in ds$string_data); + } + } + +# Function to retrieve intelligence items while abstracting from different +# data stores for different indicator types. function get_items(s: Seen): set[Item] { local return_data: set[Item]; @@ -365,7 +370,7 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5 function insert(item: Item) { - # Create and fill out the meta data item. + # Create and fill out the metadata item. local meta = item$meta; local meta_tbl: table [string] of MetaData; local is_new: bool = T; @@ -420,7 +425,7 @@ function insert(item: Item) if ( have_full_data ) { - # Insert new meta data or update if already present + # Insert new metadata or update if already present meta_tbl[meta$source] = meta; } @@ -430,8 +435,8 @@ function insert(item: Item) event Intel::new_item(item); } -# Function to remove meta data of an item. The function returns T -# if there is no meta data left for the given indicator. +# Function to remove metadata of an item. The function returns T +# if there is no metadata left for the given indicator. function remove_meta_data(item: Item): bool { if ( ! have_full_data ) @@ -466,7 +471,7 @@ function remove(item: Item, purge_indicator: bool) return; } - # Remove meta data from manager's data store + # Remove metadata from manager's data store local no_meta_data = remove_meta_data(item); # Remove whole indicator if necessary if ( no_meta_data || purge_indicator ) @@ -485,14 +490,14 @@ function remove(item: Item, purge_indicator: bool) delete data_store$string_data[item$indicator, item$indicator_type]; break; } - # Trigger deletion in min data stores + # Trigger deletion in minimal data stores event Intel::purge_item(item); } } +# Handling of indicator removal in minimal data stores. event purge_item(item: Item) { - # Remove data from min data store switch ( item$indicator_type ) { case ADDR: diff --git a/scripts/policy/frameworks/intel/do_expire.bro b/scripts/policy/frameworks/intel/do_expire.bro index b5f47c2ade..aabe3630e4 100644 --- a/scripts/policy/frameworks/intel/do_expire.bro +++ b/scripts/policy/frameworks/intel/do_expire.bro @@ -1,10 +1,14 @@ +##! This script enables expiration for intelligence items. @load base/frameworks/intel module Intel; +redef item_expiration = 10min; + hook item_expired(indicator: string, indicator_type: Type, metas: set[MetaData]) &priority=-10 { + # Trigger removal of the expired item. break; } diff --git a/scripts/policy/frameworks/intel/do_notice.bro b/scripts/policy/frameworks/intel/do_notice.bro index ed859b468b..fc75a8efee 100644 --- a/scripts/policy/frameworks/intel/do_notice.bro +++ b/scripts/policy/frameworks/intel/do_notice.bro @@ -1,3 +1,4 @@ +##! This script enables notice generation for intelligence matches. @load base/frameworks/intel @load base/frameworks/notice diff --git a/scripts/policy/frameworks/intel/whitelist.bro b/scripts/policy/frameworks/intel/whitelist.bro index 9061ed2a91..527d828881 100644 --- a/scripts/policy/frameworks/intel/whitelist.bro +++ b/scripts/policy/frameworks/intel/whitelist.bro @@ -1,12 +1,12 @@ +##! This script enables whitelisting for intelligence items. @load base/frameworks/intel -@load base/frameworks/notice module Intel; export { redef record Intel::MetaData += { - ## Add a field to indicate if this is a whitelisted item. + ## A boolean value to indicate whether the item is whitelisted. whitelist: bool &default=F; }; }