mirror of
https://github.com/zeek/zeek.git
synced 2025-10-17 14:08:20 +00:00
Major updates to fix the Intel framework API.
This commit is contained in:
Seth Hall
parent
50e319a417
commit
71c5b49bdc
8 changed files with 126 additions and 391 deletions
|
|
@ -2,12 +2,6 @@
|
|||
##! and strings (with a str_type). Metadata can
|
||||
##! also be associated with the intelligence like for making more informated
|
||||
##! decisions about matching and handling of intelligence.
|
||||
#
|
||||
# TODO:
|
||||
# Comments
|
||||
# Better Intel::Item comparison (has_meta)
|
||||
# Generate a notice when messed up data is discovered.
|
||||
# Complete "net" support as an intelligence type.
|
||||
|
||||
@load base/frameworks/notice
|
||||
|
||||
|
|
@ -24,7 +18,7 @@ export {
|
|||
|
||||
## String data needs to be further categoried since it could represent
|
||||
## and number of types of data.
|
||||
type SubType: enum {
|
||||
type StrType: enum {
|
||||
## A complete URL.
|
||||
URL,
|
||||
## User-Agent string, typically HTTP or mail message body.
|
||||
|
|
@ -41,39 +35,13 @@ export {
|
|||
## Certificate hash. Normally for X.509 certificates from the SSL analyzer.
|
||||
CERT_HASH,
|
||||
};
|
||||
|
||||
## Why a piece of intelligence is being added or looked up. The intent a human
|
||||
## placed upon the data when it was decided to be worthwhile as intelligence.
|
||||
type Intent: enum {
|
||||
## Data is to be considered malicious.
|
||||
MALICIOUS,
|
||||
## Data is to be considered sensitive. In many cases this may be
|
||||
## hosts containing contractually or legally restricted data such
|
||||
## as HIPPA, PCI, Sarbanes-Oxley, etc.
|
||||
SENSITIVE,
|
||||
## Data that is never to be seen. This acts like the "canary in
|
||||
## the coal mine". A possibility could be file hashes for
|
||||
## critically important files.
|
||||
CANARY,
|
||||
## Data that is whitelisted. The primary use for this intent is to
|
||||
## locally whitelist false positive data from external feeds.
|
||||
WHITELIST,
|
||||
};
|
||||
|
||||
## Enum to represent where data came from when it was discovered.
|
||||
type Where: enum {
|
||||
## A catchall value to represent data of unknown provenance.
|
||||
ANYWHERE,
|
||||
};
|
||||
|
||||
|
||||
## Data about an :bro:type:`Intel::Item`
|
||||
type MetaData: record {
|
||||
## An arbitrary string value representing the data source. Typically,
|
||||
## the convention for this field will be the source name and feed name
|
||||
## separated by a hyphen. For example: "source1-c&c".
|
||||
source: string;
|
||||
## The intent of the data.
|
||||
intent: Intent;
|
||||
## A freeform description for the data.
|
||||
desc: string &optional;
|
||||
## A URL for more information about the data.
|
||||
|
|
@ -84,215 +52,144 @@ export {
|
|||
host: addr &optional;
|
||||
net: subnet &optional;
|
||||
str: string &optional;
|
||||
str_type: SubType &optional;
|
||||
str_type: StrType &optional;
|
||||
|
||||
meta: MetaData;
|
||||
};
|
||||
|
||||
type Found: record {
|
||||
## Enum to represent where data came from when it was discovered.
|
||||
type Where: enum {
|
||||
## A catchall value to represent data of unknown provenance.
|
||||
ANYWHERE,
|
||||
};
|
||||
|
||||
type Seen: record {
|
||||
host: addr &optional;
|
||||
str: string &optional;
|
||||
str_type: SubType &optional;
|
||||
str_type: StrType &optional;
|
||||
|
||||
where: Where;
|
||||
};
|
||||
|
||||
type Info: record {
|
||||
ts: time &log;
|
||||
## This value should be one of: "info", "warn", "error"
|
||||
level: string &log;
|
||||
message: string &log;
|
||||
item: Item &log;
|
||||
type PolicyItem: record {
|
||||
pred: function(seen: Seen, item: Item): bool &optional;
|
||||
|
||||
log_it: bool &default=T;
|
||||
};
|
||||
|
||||
type Plugin: record {
|
||||
index: function() &optional;
|
||||
match: function(found: Found): bool &optional;
|
||||
lookup: function(found: Found): set[Item] &optional;
|
||||
};
|
||||
|
||||
## Manipulation and query API functions.
|
||||
## Intelligence data manipulation functions.
|
||||
global insert: function(item: Item);
|
||||
global delete_item: function(item: Item): bool;
|
||||
global unique_data: function(): count;
|
||||
|
||||
## Function to declare discovery of a piece of data in order to check
|
||||
## it against known intelligence for matches.
|
||||
global found_in_conn: function(c: connection, found: Found);
|
||||
global seen_in_conn: function(c: connection, seen: Seen);
|
||||
|
||||
## Event to represent a match happening in a connection. On clusters there
|
||||
## is no assurance as to where this event will be generated so don't
|
||||
## assume that arbitrary global state beyond the given data
|
||||
## will be available.
|
||||
global match_in_conn: event(c: connection, found: Found, items: set[Item]);
|
||||
## Intelligence policy variable for handling matches.
|
||||
const policy: set[PolicyItem] = {} &redef;
|
||||
|
||||
global find: function(found: Found): bool;
|
||||
global lookup: function(found: Found): set[Item];
|
||||
|
||||
|
||||
## Plugin API functions
|
||||
global register_custom_matcher: function(str_type: SubType,
|
||||
func: function(found: Found): bool);
|
||||
global register_custom_lookup: function(str_type: SubType,
|
||||
func: function(found: Found): set[Item]);
|
||||
|
||||
## API Events
|
||||
## API Events that indicate when various things happen internally within the
|
||||
## intelligence framework.
|
||||
global new_item: event(item: Item);
|
||||
global updated_item: event(item: Item);
|
||||
global insert_event: event(item: Item);
|
||||
|
||||
## Optionally store metadata. This is primarily used internally depending on
|
||||
## if this is a cluster deployment or not. On clusters, workers probably
|
||||
## shouldn't be storing the full metadata.
|
||||
const store_metadata = T &redef;
|
||||
}
|
||||
|
||||
# Internal handler for conn oriented matches with no metadata base on the store_metadata setting.
|
||||
global match_in_conn_no_items: event(c: connection, found: Found);
|
||||
## Event to represent a match happening in a connection. On clusters there
|
||||
## is no assurance as to where this event will be generated so don't
|
||||
## assume that arbitrary global state beyond the given data
|
||||
## will be available.
|
||||
global match_in_conn: event(c: connection, seen: Seen, items: set[Item]);
|
||||
|
||||
# Internal handler for conn oriented matches with no metadata based on the have_full_data setting.
|
||||
global match_in_conn_no_items: event(c: connection, seen: Seen);
|
||||
|
||||
## Optionally store metadata. This is used internally depending on
|
||||
## if this is a cluster deployment or not.
|
||||
const have_full_data = T &redef;
|
||||
|
||||
type DataStore: record {
|
||||
host_data: table[addr] of set[MetaData];
|
||||
string_data: table[string, SubType] of set[MetaData];
|
||||
net_data: table[subnet] of set[MetaData];
|
||||
string_data: table[string, StrType] of set[MetaData];
|
||||
};
|
||||
global data_store: DataStore;
|
||||
|
||||
global custom_matchers: table[SubType] of set[function(found: Found): bool];
|
||||
global custom_lookup: table[SubType] of set[function(found: Found): set[Item]];
|
||||
|
||||
|
||||
event bro_init() &priority=5
|
||||
function find(seen: Seen): bool
|
||||
{
|
||||
Log::create_stream(Intel::LOG, [$columns=Info]);
|
||||
}
|
||||
|
||||
|
||||
function find(found: Found): bool
|
||||
{
|
||||
if ( found?$host && found$host in data_store$host_data)
|
||||
if ( seen?$host &&
|
||||
seen$host in data_store$net_data )
|
||||
{
|
||||
return T;
|
||||
}
|
||||
else if ( found?$str && found?$str_type &&
|
||||
[found$str, found$str_type] in data_store$string_data )
|
||||
else if ( seen?$str && seen?$str_type &&
|
||||
[seen$str, seen$str_type] in data_store$string_data )
|
||||
{
|
||||
return T;
|
||||
}
|
||||
|
||||
# Finder plugins!
|
||||
for ( plugin in plugins )
|
||||
else
|
||||
{
|
||||
if ( plugin?$match && plugin$match(found) )
|
||||
return T;
|
||||
return F;
|
||||
}
|
||||
|
||||
return F;
|
||||
}
|
||||
|
||||
function lookup(found: Found): set[Item]
|
||||
function get_items(seen: Seen): set[Item]
|
||||
{
|
||||
local item: Item;
|
||||
local return_data: set[Item] = set();
|
||||
|
||||
if ( found?$host )
|
||||
if ( ! have_full_data )
|
||||
{
|
||||
# A reporter warning should be generated here because this function
|
||||
# should never be called from a host that doesn't have the full data.
|
||||
# TODO: do a reporter warning.
|
||||
return return_data;
|
||||
}
|
||||
|
||||
if ( seen?$host )
|
||||
{
|
||||
# See if the host is known about and it has meta values
|
||||
if ( found$host in data_store$host_data )
|
||||
if ( seen$host in data_store$net_data )
|
||||
{
|
||||
for ( m in data_store$host_data[found$host] )
|
||||
for ( m in data_store$net_data[seen$host] )
|
||||
{
|
||||
item = [$host=found$host, $meta=m];
|
||||
# TODO: the lookup should be finding all and not just most specific
|
||||
# and $host/$net should have the correct value.
|
||||
item = [$host=seen$host, $meta=m];
|
||||
add return_data[item];
|
||||
}
|
||||
}
|
||||
}
|
||||
else if ( found?$str && found?$str_type )
|
||||
else if ( seen?$str && seen?$str_type )
|
||||
{
|
||||
# See if the string is known about and it has meta values
|
||||
if ( [found$str, found$str_type] in data_store$string_data )
|
||||
if ( [seen$str, seen$str_type] in data_store$string_data )
|
||||
{
|
||||
for ( m in data_store$string_data[found$str, found$str_type] )
|
||||
for ( m in data_store$string_data[seen$str, seen$str_type] )
|
||||
{
|
||||
item = [$str=found$str, $str_type=found$str_type, $meta=m];
|
||||
item = [$str=seen$str, $str_type=seen$str_type, $meta=m];
|
||||
add return_data[item];
|
||||
}
|
||||
}
|
||||
|
||||
# Check if there are any custom str_type lookup functions and add the values to
|
||||
# the result set.
|
||||
if ( found$str_type in custom_lookup )
|
||||
{
|
||||
for ( lookup_func in custom_lookup[found$str_type] )
|
||||
{
|
||||
# Iterating here because there is no way to merge sets generically.
|
||||
for ( custom_lookup_item in lookup_func(found) )
|
||||
add return_data[custom_lookup_item];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
# TODO: Later we should probably track whitelist matches.
|
||||
# TODO: base this on a set instead of iterating the items.
|
||||
for ( item in return_data )
|
||||
{
|
||||
if ( item$meta$intent == WHITELIST )
|
||||
{
|
||||
return set();
|
||||
}
|
||||
}
|
||||
|
||||
return return_data;
|
||||
}
|
||||
|
||||
function Intel::found_in_conn(c: connection, found: Found)
|
||||
function Intel::seen_in_conn(c: connection, seen: Seen)
|
||||
{
|
||||
if ( find(found) )
|
||||
if ( find(seen) )
|
||||
{
|
||||
if ( store_metadata )
|
||||
if ( have_full_data )
|
||||
{
|
||||
local items = lookup(found);
|
||||
event Intel::match_in_conn(c, found, items);
|
||||
local items = get_items(seen);
|
||||
event Intel::match_in_conn(c, seen, items);
|
||||
}
|
||||
else
|
||||
{
|
||||
event Intel::match_in_conn_no_items(c, found);
|
||||
event Intel::match_in_conn_no_items(c, seen);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function register_custom_matcher(str_type: SubType, func: function(found: Found): bool)
|
||||
{
|
||||
if ( str_type !in custom_matchers )
|
||||
custom_matchers[str_type] = set(func);
|
||||
else
|
||||
add custom_matchers[str_type][func];
|
||||
}
|
||||
|
||||
function register_custom_lookup(str_type: SubType, func: function(found: Found): set[Item])
|
||||
{
|
||||
if ( str_type !in custom_lookup )
|
||||
custom_lookup[str_type] = set(func);
|
||||
else
|
||||
add custom_lookup[str_type][func];
|
||||
}
|
||||
|
||||
function unique_data(): count
|
||||
{
|
||||
return |data_store$host_data| + |data_store$string_data|;
|
||||
}
|
||||
|
||||
#function get_meta(check: MetaData, metas: set[MetaData]): MetaData
|
||||
# {
|
||||
# local check_hash = md5_hash(check);
|
||||
# for ( m in metas )
|
||||
# {
|
||||
# if ( check_hash == md5_hash(m) )
|
||||
# return m;
|
||||
# }
|
||||
#
|
||||
# return [$source=""];
|
||||
# }
|
||||
|
||||
function has_meta(check: MetaData, metas: set[MetaData]): bool
|
||||
{
|
||||
|
|
@ -309,35 +206,41 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
|
|||
|
||||
function insert(item: Item)
|
||||
{
|
||||
local err_msg = "";
|
||||
if ( item?$str && ! item?$str_type )
|
||||
err_msg = "You must provide a str_type for strings or this item doesn't make sense.";
|
||||
|
||||
if ( err_msg == "" )
|
||||
if ( item?$str && !item?$str_type )
|
||||
{
|
||||
# Create and fill out the meta data item.
|
||||
local meta = item$meta;
|
||||
local metas: set[MetaData];
|
||||
event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense. Item: %s", item), "");
|
||||
return;
|
||||
}
|
||||
|
||||
if ( item?$host )
|
||||
{
|
||||
if ( item$host !in data_store$host_data )
|
||||
data_store$host_data[item$host] = set();
|
||||
|
||||
metas = data_store$host_data[item$host];
|
||||
}
|
||||
else if ( item?$str )
|
||||
{
|
||||
if ( [item$str, item$str_type] !in data_store$string_data )
|
||||
data_store$string_data[item$str, item$str_type] = set();
|
||||
# Create and fill out the meta data item.
|
||||
local meta = item$meta;
|
||||
local metas: set[MetaData];
|
||||
|
||||
metas = data_store$string_data[item$str, item$str_type];
|
||||
}
|
||||
else
|
||||
{
|
||||
err_msg = "Malformed intelligence item";
|
||||
}
|
||||
if ( item?$host )
|
||||
{
|
||||
local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128);
|
||||
if ( host !in data_store$net_data )
|
||||
data_store$net_data[host] = set();
|
||||
|
||||
metas = data_store$net_data[host];
|
||||
}
|
||||
else if ( item?$net )
|
||||
{
|
||||
if ( item$net !in data_store$net_data )
|
||||
data_store$net_data[item$net] = set();
|
||||
|
||||
metas = data_store$net_data[item$net];
|
||||
}
|
||||
else if ( item?$str )
|
||||
{
|
||||
if ( [item$str, item$str_type] !in data_store$string_data )
|
||||
data_store$string_data[item$str, item$str_type] = set();
|
||||
|
||||
metas = data_store$string_data[item$str, item$str_type];
|
||||
}
|
||||
|
||||
if ( have_full_data )
|
||||
{
|
||||
for ( m in metas )
|
||||
{
|
||||
if ( meta$source == m$source )
|
||||
|
|
@ -349,6 +252,7 @@ function insert(item: Item)
|
|||
}
|
||||
else
|
||||
{
|
||||
# Same source, different metadata means updated item.
|
||||
event Intel::updated_item(item);
|
||||
break;
|
||||
}
|
||||
|
|
@ -359,19 +263,7 @@ function insert(item: Item)
|
|||
break;
|
||||
}
|
||||
}
|
||||
|
||||
add metas[item$meta];
|
||||
return;
|
||||
}
|
||||
|
||||
if ( err_msg != "" )
|
||||
Log::write(Intel::LOG, [$ts=network_time(), $level="warn", $message=err_msg, $item=item]);
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
event insert_event(item: Item)
|
||||
{
|
||||
insert(item);
|
||||
}
|
||||
|
||||
|
||||
Loading…
Add table
Add a link
Reference in a new issue