mirror of
https://github.com/zeek/zeek.git
synced 2025-10-07 00:58:19 +00:00
Merge remote-tracking branch 'origin/master' into topic/bernhard/hyperloglog
This commit is contained in:
Bernhard Amann
commit
5b9d80e50d
247 changed files with 2729 additions and 5372 deletions
|
|
@ -81,6 +81,13 @@ export {
|
|||
## Returns: The analyzer name corresponding to the tag.
|
||||
global name: function(tag: Analyzer::Tag) : string;
|
||||
|
||||
## Translates an analyzer's name to a tag enum value.
|
||||
##
|
||||
## name: The analyzer name.
|
||||
##
|
||||
## Returns: The analyzer tag corresponding to the name.
|
||||
global get_tag: function(name: string): Analyzer::Tag;
|
||||
|
||||
## Schedules an analyzer for a future connection originating from a given IP
|
||||
## address and port.
|
||||
##
|
||||
|
|
@ -187,6 +194,11 @@ function name(atype: Analyzer::Tag) : string
|
|||
return __name(atype);
|
||||
}
|
||||
|
||||
function get_tag(name: string): Analyzer::Tag
|
||||
{
|
||||
return __tag(name);
|
||||
}
|
||||
|
||||
function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
|
||||
analyzer: Analyzer::Tag, tout: interval) : bool
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,261 +0,0 @@
|
|||
##! An interface for driving the analysis of files, possibly independent of
|
||||
##! any network protocol over which they're transported.
|
||||
|
||||
@load base/bif/file_analysis.bif
|
||||
@load base/frameworks/logging
|
||||
|
||||
module FileAnalysis;
|
||||
|
||||
export {
|
||||
redef enum Log::ID += {
|
||||
## Logging stream for file analysis.
|
||||
LOG
|
||||
};
|
||||
|
||||
## A structure which represents a desired type of file analysis.
|
||||
type AnalyzerArgs: record {
|
||||
## The type of analysis.
|
||||
tag: FileAnalysis::Tag;
|
||||
|
||||
## The local filename to which to write an extracted file. Must be
|
||||
## set when *tag* is :bro:see:`FileAnalysis::ANALYZER_EXTRACT`.
|
||||
extract_filename: string &optional;
|
||||
|
||||
## An event which will be generated for all new file contents,
|
||||
## chunk-wise. Used when *tag* is
|
||||
## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`.
|
||||
chunk_event: event(f: fa_file, data: string, off: count) &optional;
|
||||
|
||||
## An event which will be generated for all new file contents,
|
||||
## stream-wise. Used when *tag* is
|
||||
## :bro:see:`FileAnalysis::ANALYZER_DATA_EVENT`.
|
||||
stream_event: event(f: fa_file, data: string) &optional;
|
||||
} &redef;
|
||||
|
||||
## Contains all metadata related to the analysis of a given file.
|
||||
## For the most part, fields here are derived from ones of the same name
|
||||
## in :bro:see:`fa_file`.
|
||||
type Info: record {
|
||||
## An identifier associated with a single file.
|
||||
id: string &log;
|
||||
|
||||
## Identifier associated with a container file from which this one was
|
||||
## extracted as part of the file analysis.
|
||||
parent_id: string &log &optional;
|
||||
|
||||
## An identification of the source of the file data. E.g. it may be
|
||||
## a network protocol over which it was transferred, or a local file
|
||||
## path which was read, or some other input source.
|
||||
source: string &log &optional;
|
||||
|
||||
## If the source of this file is is a network connection, this field
|
||||
## may be set to indicate the directionality.
|
||||
is_orig: bool &log &optional;
|
||||
|
||||
## The time at which the last activity for the file was seen.
|
||||
last_active: time &log;
|
||||
|
||||
## Number of bytes provided to the file analysis engine for the file.
|
||||
seen_bytes: count &log &default=0;
|
||||
|
||||
## Total number of bytes that are supposed to comprise the full file.
|
||||
total_bytes: count &log &optional;
|
||||
|
||||
## The number of bytes in the file stream that were completely missed
|
||||
## during the process of analysis e.g. due to dropped packets.
|
||||
missing_bytes: count &log &default=0;
|
||||
|
||||
## The number of not all-in-sequence bytes in the file stream that
|
||||
## were delivered to file analyzers due to reassembly buffer overflow.
|
||||
overflow_bytes: count &log &default=0;
|
||||
|
||||
## The amount of time between receiving new data for this file that
|
||||
## the analysis engine will wait before giving up on it.
|
||||
timeout_interval: interval &log &optional;
|
||||
|
||||
## The number of bytes at the beginning of a file to save for later
|
||||
## inspection in *bof_buffer* field.
|
||||
bof_buffer_size: count &log &optional;
|
||||
|
||||
## A mime type provided by libmagic against the *bof_buffer*, or
|
||||
## in the cases where no buffering of the beginning of file occurs,
|
||||
## an initial guess of the mime type based on the first data seen.
|
||||
mime_type: string &log &optional;
|
||||
|
||||
## Whether the file analysis timed out at least once for the file.
|
||||
timedout: bool &log &default=F;
|
||||
|
||||
## Connection UIDS over which the file was transferred.
|
||||
conn_uids: set[string] &log;
|
||||
|
||||
## A set of analysis types done during the file analysis.
|
||||
analyzers: set[FileAnalysis::Tag];
|
||||
|
||||
## Local filenames of extracted files.
|
||||
extracted_files: set[string] &log;
|
||||
|
||||
## An MD5 digest of the file contents.
|
||||
md5: string &log &optional;
|
||||
|
||||
## A SHA1 digest of the file contents.
|
||||
sha1: string &log &optional;
|
||||
|
||||
## A SHA256 digest of the file contents.
|
||||
sha256: string &log &optional;
|
||||
} &redef;
|
||||
|
||||
## A table that can be used to disable file analysis completely for
|
||||
## any files transferred over given network protocol analyzers.
|
||||
const disable: table[Analyzer::Tag] of bool = table() &redef;
|
||||
|
||||
## Event that can be handled to access the Info record as it is sent on
|
||||
## to the logging framework.
|
||||
global log_file_analysis: event(rec: Info);
|
||||
|
||||
## The salt concatenated to unique file handle strings generated by
|
||||
## :bro:see:`get_file_handle` before hashing them in to a file id
|
||||
## (the *id* field of :bro:see:`fa_file`).
|
||||
## Provided to help mitigate the possiblility of manipulating parts of
|
||||
## network connections that factor in to the file handle in order to
|
||||
## generate two handles that would hash to the same file id.
|
||||
const salt = "I recommend changing this." &redef;
|
||||
|
||||
## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
|
||||
## used to determine the length of inactivity that is allowed for a file
|
||||
## before internal state related to it is cleaned up. When used within a
|
||||
## :bro:see:`file_timeout` handler, the analysis will delay timing out
|
||||
## again for the period specified by *t*.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## t: the amount of time the file can remain inactive before discarding.
|
||||
##
|
||||
## Returns: true if the timeout interval was set, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global set_timeout_interval: function(f: fa_file, t: interval): bool;
|
||||
|
||||
## Adds an analyzer to the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## args: the analyzer type to add along with any arguments it takes.
|
||||
##
|
||||
## Returns: true if the analyzer will be added, or false if analysis
|
||||
## for the *id* isn't currently active or the *args*
|
||||
## were invalid for the analyzer type.
|
||||
global add_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
|
||||
|
||||
## Removes an analyzer from the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## args: the analyzer (type and args) to remove.
|
||||
##
|
||||
## Returns: true if the analyzer will be removed, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global remove_analyzer: function(f: fa_file, args: AnalyzerArgs): bool;
|
||||
|
||||
## Stops/ignores any further analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## Returns: true if analysis for the given file will be ignored for the
|
||||
## rest of it's contents, or false if analysis for the *id*
|
||||
## isn't currently active.
|
||||
global stop: function(f: fa_file): bool;
|
||||
}
|
||||
|
||||
redef record fa_file += {
|
||||
info: Info &optional;
|
||||
};
|
||||
|
||||
function set_info(f: fa_file)
|
||||
{
|
||||
if ( ! f?$info )
|
||||
{
|
||||
local tmp: Info;
|
||||
f$info = tmp;
|
||||
}
|
||||
|
||||
f$info$id = f$id;
|
||||
if ( f?$parent_id ) f$info$parent_id = f$parent_id;
|
||||
if ( f?$source ) f$info$source = f$source;
|
||||
if ( f?$is_orig ) f$info$is_orig = f$is_orig;
|
||||
f$info$last_active = f$last_active;
|
||||
f$info$seen_bytes = f$seen_bytes;
|
||||
if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes;
|
||||
f$info$missing_bytes = f$missing_bytes;
|
||||
f$info$overflow_bytes = f$overflow_bytes;
|
||||
f$info$timeout_interval = f$timeout_interval;
|
||||
f$info$bof_buffer_size = f$bof_buffer_size;
|
||||
if ( f?$mime_type ) f$info$mime_type = f$mime_type;
|
||||
if ( f?$conns )
|
||||
for ( cid in f$conns )
|
||||
add f$info$conn_uids[f$conns[cid]$uid];
|
||||
}
|
||||
|
||||
function set_timeout_interval(f: fa_file, t: interval): bool
|
||||
{
|
||||
return __set_timeout_interval(f$id, t);
|
||||
}
|
||||
|
||||
function add_analyzer(f: fa_file, args: AnalyzerArgs): bool
|
||||
{
|
||||
if ( ! __add_analyzer(f$id, args) ) return F;
|
||||
|
||||
set_info(f);
|
||||
add f$info$analyzers[args$tag];
|
||||
|
||||
if ( args$tag == FileAnalysis::ANALYZER_EXTRACT )
|
||||
add f$info$extracted_files[args$extract_filename];
|
||||
|
||||
return T;
|
||||
}
|
||||
|
||||
function remove_analyzer(f: fa_file, args: AnalyzerArgs): bool
|
||||
{
|
||||
return __remove_analyzer(f$id, args);
|
||||
}
|
||||
|
||||
function stop(f: fa_file): bool
|
||||
{
|
||||
return __stop(f$id);
|
||||
}
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(FileAnalysis::LOG,
|
||||
[$columns=Info, $ev=log_file_analysis]);
|
||||
}
|
||||
|
||||
event file_timeout(f: fa_file) &priority=5
|
||||
{
|
||||
set_info(f);
|
||||
f$info$timedout = T;
|
||||
}
|
||||
|
||||
event file_hash(f: fa_file, kind: string, hash: string) &priority=5
|
||||
{
|
||||
set_info(f);
|
||||
switch ( kind ) {
|
||||
case "md5":
|
||||
f$info$md5 = hash;
|
||||
break;
|
||||
case "sha1":
|
||||
f$info$sha1 = hash;
|
||||
break;
|
||||
case "sha256":
|
||||
f$info$sha256 = hash;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=5
|
||||
{
|
||||
set_info(f);
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=-5
|
||||
{
|
||||
Log::write(FileAnalysis::LOG, f$info);
|
||||
}
|
||||
371
scripts/base/frameworks/files/main.bro
Normal file
371
scripts/base/frameworks/files/main.bro
Normal file
|
|
@ -0,0 +1,371 @@
|
|||
##! An interface for driving the analysis of files, possibly independent of
|
||||
##! any network protocol over which they're transported.
|
||||
|
||||
@load base/bif/file_analysis.bif
|
||||
@load base/frameworks/analyzer
|
||||
@load base/frameworks/logging
|
||||
@load base/utils/site
|
||||
|
||||
module Files;
|
||||
|
||||
export {
|
||||
redef enum Log::ID += {
|
||||
## Logging stream for file analysis.
|
||||
LOG
|
||||
};
|
||||
|
||||
## A structure which represents a desired type of file analysis.
|
||||
type AnalyzerArgs: record {
|
||||
## An event which will be generated for all new file contents,
|
||||
## chunk-wise. Used when *tag* is
|
||||
## :bro:see:`Files::ANALYZER_DATA_EVENT`.
|
||||
chunk_event: event(f: fa_file, data: string, off: count) &optional;
|
||||
|
||||
## An event which will be generated for all new file contents,
|
||||
## stream-wise. Used when *tag* is
|
||||
## :bro:see:`Files::ANALYZER_DATA_EVENT`.
|
||||
stream_event: event(f: fa_file, data: string) &optional;
|
||||
} &redef;
|
||||
|
||||
## Contains all metadata related to the analysis of a given file.
|
||||
## For the most part, fields here are derived from ones of the same name
|
||||
## in :bro:see:`fa_file`.
|
||||
type Info: record {
|
||||
## The time when the file was first seen.
|
||||
ts: time &log;
|
||||
|
||||
## An identifier associated with a single file.
|
||||
fuid: string &log;
|
||||
|
||||
## If this file was transferred over a network
|
||||
## connection this should show the host or hosts that
|
||||
## the data sourced from.
|
||||
tx_hosts: set[addr] &log;
|
||||
|
||||
## If this file was transferred over a network
|
||||
## connection this should show the host or hosts that
|
||||
## the data traveled to.
|
||||
rx_hosts: set[addr] &log;
|
||||
|
||||
## Connection UIDS over which the file was transferred.
|
||||
conn_uids: set[string] &log;
|
||||
|
||||
## An identification of the source of the file data. E.g. it may be
|
||||
## a network protocol over which it was transferred, or a local file
|
||||
## path which was read, or some other input source.
|
||||
source: string &log &optional;
|
||||
|
||||
## A value to represent the depth of this file in relation
|
||||
## to its source. In SMTP, it is the depth of the MIME
|
||||
## attachment on the message. In HTTP, it is the depth of the
|
||||
## request within the TCP connection.
|
||||
depth: count &default=0 &log;
|
||||
|
||||
## A set of analysis types done during the file analysis.
|
||||
analyzers: set[string] &log;
|
||||
|
||||
## A mime type provided by libmagic against the *bof_buffer*, or
|
||||
## in the cases where no buffering of the beginning of file occurs,
|
||||
## an initial guess of the mime type based on the first data seen.
|
||||
mime_type: string &log &optional;
|
||||
|
||||
## A filename for the file if one is available from the source
|
||||
## for the file. These will frequently come from
|
||||
## "Content-Disposition" headers in network protocols.
|
||||
filename: string &log &optional;
|
||||
|
||||
## The duration the file was analyzed for.
|
||||
duration: interval &log &default=0secs;
|
||||
|
||||
## If the source of this file is a network connection, this field
|
||||
## indicates if the data originated from the local network or not as
|
||||
## determined by the configured bro:see:`Site::local_nets`.
|
||||
local_orig: bool &log &optional;
|
||||
|
||||
## If the source of this file is a network connection, this field
|
||||
## indicates if the file is being sent by the originator of the connection
|
||||
## or the responder.
|
||||
is_orig: bool &log &optional;
|
||||
|
||||
## Number of bytes provided to the file analysis engine for the file.
|
||||
seen_bytes: count &log &default=0;
|
||||
|
||||
## Total number of bytes that are supposed to comprise the full file.
|
||||
total_bytes: count &log &optional;
|
||||
|
||||
## The number of bytes in the file stream that were completely missed
|
||||
## during the process of analysis e.g. due to dropped packets.
|
||||
missing_bytes: count &log &default=0;
|
||||
|
||||
## The number of not all-in-sequence bytes in the file stream that
|
||||
## were delivered to file analyzers due to reassembly buffer overflow.
|
||||
overflow_bytes: count &log &default=0;
|
||||
|
||||
## Whether the file analysis timed out at least once for the file.
|
||||
timedout: bool &log &default=F;
|
||||
|
||||
## Identifier associated with a container file from which this one was
|
||||
## extracted as part of the file analysis.
|
||||
parent_fuid: string &log &optional;
|
||||
} &redef;
|
||||
|
||||
## A table that can be used to disable file analysis completely for
|
||||
## any files transferred over given network protocol analyzers.
|
||||
const disable: table[Files::Tag] of bool = table() &redef;
|
||||
|
||||
## The salt concatenated to unique file handle strings generated by
|
||||
## :bro:see:`get_file_handle` before hashing them in to a file id
|
||||
## (the *id* field of :bro:see:`fa_file`).
|
||||
## Provided to help mitigate the possiblility of manipulating parts of
|
||||
## network connections that factor in to the file handle in order to
|
||||
## generate two handles that would hash to the same file id.
|
||||
const salt = "I recommend changing this." &redef;
|
||||
|
||||
## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
|
||||
## used to determine the length of inactivity that is allowed for a file
|
||||
## before internal state related to it is cleaned up. When used within a
|
||||
## :bro:see:`file_timeout` handler, the analysis will delay timing out
|
||||
## again for the period specified by *t*.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## t: the amount of time the file can remain inactive before discarding.
|
||||
##
|
||||
## Returns: true if the timeout interval was set, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global set_timeout_interval: function(f: fa_file, t: interval): bool;
|
||||
|
||||
## Adds an analyzer to the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## tag: the analyzer type.
|
||||
##
|
||||
## args: any parameters the analyzer takes.
|
||||
##
|
||||
## Returns: true if the analyzer will be added, or false if analysis
|
||||
## for the *id* isn't currently active or the *args*
|
||||
## were invalid for the analyzer type.
|
||||
global add_analyzer: function(f: fa_file,
|
||||
tag: Files::Tag,
|
||||
args: AnalyzerArgs &default=AnalyzerArgs()): bool;
|
||||
|
||||
## Removes an analyzer from the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## args: the analyzer (type and args) to remove.
|
||||
##
|
||||
## Returns: true if the analyzer will be removed, or false if analysis
|
||||
## for the *id* isn't currently active.
|
||||
global remove_analyzer: function(f: fa_file,
|
||||
tag: Files::Tag,
|
||||
args: AnalyzerArgs &default=AnalyzerArgs()): bool;
|
||||
|
||||
## Stops/ignores any further analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## Returns: true if analysis for the given file will be ignored for the
|
||||
## rest of it's contents, or false if analysis for the *id*
|
||||
## isn't currently active.
|
||||
global stop: function(f: fa_file): bool;
|
||||
|
||||
## Translates an file analyzer enum value to a string with the analyzer's name.
|
||||
##
|
||||
## tag: The analyzer tag.
|
||||
##
|
||||
## Returns: The analyzer name corresponding to the tag.
|
||||
global analyzer_name: function(tag: Files::Tag): string;
|
||||
|
||||
## Provides a text description regarding metadata of the file.
|
||||
## For example, with HTTP it would return a URL.
|
||||
##
|
||||
## f: The file to be described.
|
||||
##
|
||||
## Returns a text description regarding metadata of the file.
|
||||
global describe: function(f: fa_file): string;
|
||||
|
||||
type ProtoRegistration: record {
|
||||
## A callback to generate a file handle on demand when
|
||||
## one is needed by the core.
|
||||
get_file_handle: function(c: connection, is_orig: bool): string;
|
||||
|
||||
## A callback to "describe" a file. In the case of an HTTP
|
||||
## transfer the most obvious description would be the URL.
|
||||
## It's like an extremely compressed version of the normal log.
|
||||
describe: function(f: fa_file): string
|
||||
&default=function(f: fa_file): string { return ""; };
|
||||
};
|
||||
|
||||
## Register callbacks for protocols that work with the Files framework.
|
||||
## The callbacks must uniquely identify a file and each protocol can
|
||||
## only have a single callback registered for it.
|
||||
##
|
||||
## tag: Tag for the protocol analyzer having a callback being registered.
|
||||
##
|
||||
## reg: A :bro:see:`ProtoRegistration` record.
|
||||
##
|
||||
## Returns: true if the protocol being registered was not previously registered.
|
||||
global register_protocol: function(tag: Analyzer::Tag, reg: ProtoRegistration): bool;
|
||||
|
||||
## Register a callback for file analyzers to use if they need to do some manipulation
|
||||
## when they are being added to a file before the core code takes over. This is
|
||||
## unlikely to be interesting for users and should only be called by file analyzer
|
||||
## authors but it *not required*.
|
||||
##
|
||||
## tag: Tag for the file analyzer.
|
||||
##
|
||||
## callback: Function to execute when the given file analyzer is being added.
|
||||
global register_analyzer_add_callback: function(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs));
|
||||
|
||||
## Event that can be handled to access the Info record as it is sent on
|
||||
## to the logging framework.
|
||||
global log_files: event(rec: Info);
|
||||
}
|
||||
|
||||
redef record fa_file += {
|
||||
info: Info &optional;
|
||||
};
|
||||
|
||||
redef record AnalyzerArgs += {
|
||||
# This is used interally for the core file analyzer api.
|
||||
tag: Files::Tag &optional;
|
||||
};
|
||||
|
||||
# Store the callbacks for protocol analyzers that have files.
|
||||
global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
|
||||
|
||||
global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(Files::LOG, [$columns=Info, $ev=log_files]);
|
||||
}
|
||||
|
||||
function set_info(f: fa_file)
|
||||
{
|
||||
if ( ! f?$info )
|
||||
{
|
||||
local tmp: Info = Info($ts=f$last_active,
|
||||
$fuid=f$id);
|
||||
f$info = tmp;
|
||||
}
|
||||
|
||||
if ( f?$parent_id )
|
||||
f$info$parent_fuid = f$parent_id;
|
||||
if ( f?$source )
|
||||
f$info$source = f$source;
|
||||
f$info$duration = f$last_active - f$info$ts;
|
||||
f$info$seen_bytes = f$seen_bytes;
|
||||
if ( f?$total_bytes )
|
||||
f$info$total_bytes = f$total_bytes;
|
||||
f$info$missing_bytes = f$missing_bytes;
|
||||
f$info$overflow_bytes = f$overflow_bytes;
|
||||
if ( f?$is_orig )
|
||||
f$info$is_orig = f$is_orig;
|
||||
if ( f?$mime_type )
|
||||
f$info$mime_type = f$mime_type;
|
||||
}
|
||||
|
||||
function set_timeout_interval(f: fa_file, t: interval): bool
|
||||
{
|
||||
return __set_timeout_interval(f$id, t);
|
||||
}
|
||||
|
||||
function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
|
||||
{
|
||||
# This is to construct the correct args for the core API.
|
||||
args$tag = tag;
|
||||
add f$info$analyzers[Files::analyzer_name(tag)];
|
||||
|
||||
if ( tag in analyzer_add_callbacks )
|
||||
analyzer_add_callbacks[tag](f, args);
|
||||
|
||||
if ( ! __add_analyzer(f$id, args) )
|
||||
{
|
||||
Reporter::warning(fmt("Analyzer %s not added successfully to file %s.", tag, f$id));
|
||||
return F;
|
||||
}
|
||||
return T;
|
||||
}
|
||||
|
||||
function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
|
||||
{
|
||||
analyzer_add_callbacks[tag] = callback;
|
||||
}
|
||||
|
||||
function remove_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
|
||||
{
|
||||
args$tag = tag;
|
||||
return __remove_analyzer(f$id, args);
|
||||
}
|
||||
|
||||
function stop(f: fa_file): bool
|
||||
{
|
||||
return __stop(f$id);
|
||||
}
|
||||
|
||||
function analyzer_name(tag: Files::Tag): string
|
||||
{
|
||||
return __analyzer_name(tag);
|
||||
}
|
||||
|
||||
event file_new(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
}
|
||||
|
||||
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
add f$info$conn_uids[c$uid];
|
||||
local cid = c$id;
|
||||
add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
|
||||
if( |Site::local_nets| > 0 )
|
||||
f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h);
|
||||
|
||||
add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
|
||||
}
|
||||
|
||||
event file_timeout(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
f$info$timedout = T;
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=-10
|
||||
{
|
||||
Log::write(Files::LOG, f$info);
|
||||
}
|
||||
|
||||
function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
|
||||
{
|
||||
local result = (tag !in registered_protocols);
|
||||
registered_protocols[tag] = reg;
|
||||
return result;
|
||||
}
|
||||
|
||||
function describe(f: fa_file): string
|
||||
{
|
||||
local tag = Analyzer::get_tag(f$source);
|
||||
if ( tag !in registered_protocols )
|
||||
return "";
|
||||
|
||||
local handler = registered_protocols[tag];
|
||||
return handler$describe(f);
|
||||
}
|
||||
|
||||
event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
|
||||
{
|
||||
if ( tag !in registered_protocols )
|
||||
return;
|
||||
|
||||
local handler = registered_protocols[tag];
|
||||
set_file_handle(handler$get_file_handle(c, is_orig));
|
||||
}
|
||||
|
|
@ -10,13 +10,14 @@ module Intel;
|
|||
export {
|
||||
redef enum Log::ID += { LOG };
|
||||
|
||||
## String data needs to be further categoried since it could represent
|
||||
## and number of types of data.
|
||||
type StrType: enum {
|
||||
## Enum type to represent various types of intelligence data.
|
||||
type Type: enum {
|
||||
## An IP address.
|
||||
ADDR,
|
||||
## A complete URL without the prefix "http://".
|
||||
URL,
|
||||
## User-Agent string, typically HTTP or mail message body.
|
||||
USER_AGENT,
|
||||
## Software name.
|
||||
SOFTWARE,
|
||||
## Email address.
|
||||
EMAIL,
|
||||
## DNS domain name.
|
||||
|
|
@ -44,18 +45,15 @@ export {
|
|||
|
||||
## Represents a piece of intelligence.
|
||||
type Item: record {
|
||||
## The IP address if the intelligence is about an IP address.
|
||||
host: addr &optional;
|
||||
## The network if the intelligence is about a CIDR block.
|
||||
net: subnet &optional;
|
||||
## The string if the intelligence is about a string.
|
||||
str: string &optional;
|
||||
## The type of data that is in the string if the $str field is set.
|
||||
str_type: StrType &optional;
|
||||
## The intelligence indicator.
|
||||
indicator: string;
|
||||
|
||||
## The type of data that the indicator field represents.
|
||||
indicator_type: Type;
|
||||
|
||||
## Metadata for the item. Typically represents more deeply \
|
||||
## Metadata for the item. Typically represents more deeply
|
||||
## descriptive data for a piece of intelligence.
|
||||
meta: MetaData;
|
||||
meta: MetaData;
|
||||
};
|
||||
|
||||
## Enum to represent where data came from when it was discovered.
|
||||
|
|
@ -65,23 +63,23 @@ export {
|
|||
IN_ANYWHERE,
|
||||
};
|
||||
|
||||
## The $host field and combination of $str and $str_type fields are mutually
|
||||
## exclusive. These records *must* represent either an IP address being
|
||||
## seen or a string being seen.
|
||||
type Seen: record {
|
||||
## The IP address if the data seen is an IP address.
|
||||
host: addr &log &optional;
|
||||
## The string if the data is about a string.
|
||||
str: string &log &optional;
|
||||
## The type of data that is in the string if the $str field is set.
|
||||
str_type: StrType &log &optional;
|
||||
indicator: string &log &optional;
|
||||
|
||||
## The type of data that the indicator represents.
|
||||
indicator_type: Type &log &optional;
|
||||
|
||||
## If the indicator type was :bro:enum:`Intel::ADDR`, then this
|
||||
## field will be present.
|
||||
host: addr &optional;
|
||||
|
||||
## Where the data was discovered.
|
||||
where: Where &log;
|
||||
where: Where &log;
|
||||
|
||||
## If the data was discovered within a connection, the
|
||||
## connection record should go into get to give context to the data.
|
||||
conn: connection &optional;
|
||||
conn: connection &optional;
|
||||
};
|
||||
|
||||
## Record used for the logging framework representing a positive
|
||||
|
|
@ -100,7 +98,7 @@ export {
|
|||
## Where the data was seen.
|
||||
seen: Seen &log;
|
||||
## Sources which supplied data that resulted in this match.
|
||||
sources: set[string] &log;
|
||||
sources: set[string] &log &default=string_set();
|
||||
};
|
||||
|
||||
## Intelligence data manipulation functions.
|
||||
|
|
@ -135,8 +133,8 @@ const have_full_data = T &redef;
|
|||
|
||||
# The in memory data structure for holding intelligence.
|
||||
type DataStore: record {
|
||||
net_data: table[subnet] of set[MetaData];
|
||||
string_data: table[string, StrType] of set[MetaData];
|
||||
host_data: table[addr] of set[MetaData];
|
||||
string_data: table[string, Type] of set[MetaData];
|
||||
};
|
||||
global data_store: DataStore &redef;
|
||||
|
||||
|
|
@ -144,8 +142,8 @@ global data_store: DataStore &redef;
|
|||
# This is primarily for workers to do the initial quick matches and store
|
||||
# a minimal amount of data for the full match to happen on the manager.
|
||||
type MinDataStore: record {
|
||||
net_data: set[subnet];
|
||||
string_data: set[string, StrType];
|
||||
host_data: set[addr];
|
||||
string_data: set[string, Type];
|
||||
};
|
||||
global min_data_store: MinDataStore &redef;
|
||||
|
||||
|
|
@ -157,15 +155,13 @@ event bro_init() &priority=5
|
|||
|
||||
function find(s: Seen): bool
|
||||
{
|
||||
if ( s?$host &&
|
||||
((have_full_data && s$host in data_store$net_data) ||
|
||||
(s$host in min_data_store$net_data)))
|
||||
if ( s?$host )
|
||||
{
|
||||
return T;
|
||||
return ((s$host in min_data_store$host_data) ||
|
||||
(have_full_data && s$host in data_store$host_data));
|
||||
}
|
||||
else if ( s?$str && s?$str_type &&
|
||||
((have_full_data && [s$str, s$str_type] in data_store$string_data) ||
|
||||
([s$str, s$str_type] in min_data_store$string_data)))
|
||||
else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
|
||||
(have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) )
|
||||
{
|
||||
return T;
|
||||
}
|
||||
|
|
@ -177,8 +173,7 @@ function find(s: Seen): bool
|
|||
|
||||
function get_items(s: Seen): set[Item]
|
||||
{
|
||||
local item: Item;
|
||||
local return_data: set[Item] = set();
|
||||
local return_data: set[Item];
|
||||
|
||||
if ( ! have_full_data )
|
||||
{
|
||||
|
|
@ -191,26 +186,23 @@ function get_items(s: Seen): set[Item]
|
|||
if ( s?$host )
|
||||
{
|
||||
# See if the host is known about and it has meta values
|
||||
if ( s$host in data_store$net_data )
|
||||
if ( s$host in data_store$host_data )
|
||||
{
|
||||
for ( m in data_store$net_data[s$host] )
|
||||
for ( m in data_store$host_data[s$host] )
|
||||
{
|
||||
# TODO: the lookup should be finding all and not just most specific
|
||||
# and $host/$net should have the correct value.
|
||||
item = [$host=s$host, $meta=m];
|
||||
add return_data[item];
|
||||
add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)];
|
||||
}
|
||||
}
|
||||
}
|
||||
else if ( s?$str && s?$str_type )
|
||||
else
|
||||
{
|
||||
local lower_indicator = to_lower(s$indicator);
|
||||
# See if the string is known about and it has meta values
|
||||
if ( [s$str, s$str_type] in data_store$string_data )
|
||||
if ( [lower_indicator, s$indicator_type] in data_store$string_data )
|
||||
{
|
||||
for ( m in data_store$string_data[s$str, s$str_type] )
|
||||
for ( m in data_store$string_data[lower_indicator, s$indicator_type] )
|
||||
{
|
||||
item = [$str=s$str, $str_type=s$str_type, $meta=m];
|
||||
add return_data[item];
|
||||
add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -222,6 +214,12 @@ function Intel::seen(s: Seen)
|
|||
{
|
||||
if ( find(s) )
|
||||
{
|
||||
if ( s?$host )
|
||||
{
|
||||
s$indicator = cat(s$host);
|
||||
s$indicator_type = Intel::ADDR;
|
||||
}
|
||||
|
||||
if ( have_full_data )
|
||||
{
|
||||
local items = get_items(s);
|
||||
|
|
@ -250,8 +248,7 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool
|
|||
|
||||
event Intel::match(s: Seen, items: set[Item]) &priority=5
|
||||
{
|
||||
local empty_set: set[string] = set();
|
||||
local info: Info = [$ts=network_time(), $seen=s, $sources=empty_set];
|
||||
local info: Info = [$ts=network_time(), $seen=s];
|
||||
|
||||
if ( s?$conn )
|
||||
{
|
||||
|
|
@ -267,52 +264,37 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
|
|||
|
||||
function insert(item: Item)
|
||||
{
|
||||
if ( item?$str && !item?$str_type )
|
||||
{
|
||||
event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense. Item: %s", item), "");
|
||||
return;
|
||||
}
|
||||
|
||||
# Create and fill out the meta data item.
|
||||
local meta = item$meta;
|
||||
local metas: set[MetaData];
|
||||
|
||||
if ( item?$host )
|
||||
# All intelligence is case insensitive at the moment.
|
||||
local lower_indicator = to_lower(item$indicator);
|
||||
|
||||
if ( item$indicator_type == ADDR )
|
||||
{
|
||||
local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128);
|
||||
local host = to_addr(item$indicator);
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( host !in data_store$net_data )
|
||||
data_store$net_data[host] = set();
|
||||
if ( host !in data_store$host_data )
|
||||
data_store$host_data[host] = set();
|
||||
|
||||
metas = data_store$net_data[host];
|
||||
metas = data_store$host_data[host];
|
||||
}
|
||||
|
||||
add min_data_store$net_data[host];
|
||||
add min_data_store$host_data[host];
|
||||
}
|
||||
else if ( item?$net )
|
||||
else
|
||||
{
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( item$net !in data_store$net_data )
|
||||
data_store$net_data[item$net] = set();
|
||||
if ( [lower_indicator, item$indicator_type] !in data_store$string_data )
|
||||
data_store$string_data[lower_indicator, item$indicator_type] = set();
|
||||
|
||||
metas = data_store$net_data[item$net];
|
||||
metas = data_store$string_data[lower_indicator, item$indicator_type];
|
||||
}
|
||||
|
||||
add min_data_store$net_data[item$net];
|
||||
}
|
||||
else if ( item?$str )
|
||||
{
|
||||
if ( have_full_data )
|
||||
{
|
||||
if ( [item$str, item$str_type] !in data_store$string_data )
|
||||
data_store$string_data[item$str, item$str_type] = set();
|
||||
|
||||
metas = data_store$string_data[item$str, item$str_type];
|
||||
}
|
||||
|
||||
add min_data_store$string_data[item$str, item$str_type];
|
||||
add min_data_store$string_data[lower_indicator, item$indicator_type];
|
||||
}
|
||||
|
||||
local updated = F;
|
||||
|
|
|
|||
|
|
@ -68,6 +68,25 @@ export {
|
|||
## the notice policy.
|
||||
iconn: icmp_conn &optional;
|
||||
|
||||
## A file record if the notice is relted to a file. The
|
||||
## reference to the actual fa_file record will be deleted after applying
|
||||
## the notice policy.
|
||||
f: fa_file &optional;
|
||||
|
||||
## A file unique ID if this notice is related to a file. If the $f
|
||||
## field is provided, this will be automatically filled out.
|
||||
fuid: string &log &optional;
|
||||
|
||||
## A mime type if the notice is related to a file. If the $f field
|
||||
## is provided, this will be automatically filled out.
|
||||
file_mime_type: string &log &optional;
|
||||
|
||||
## Frequently files can be "described" to give a bit more context.
|
||||
## This field will typically be automatically filled out from an
|
||||
## fa_file record. For example, if a notice was related to a
|
||||
## file over HTTP, the URL of the request would be shown.
|
||||
file_desc: string &log &optional;
|
||||
|
||||
## The transport protocol. Filled automatically when either conn, iconn
|
||||
## or p is specified.
|
||||
proto: transport_proto &log &optional;
|
||||
|
|
@ -460,10 +479,28 @@ function apply_policy(n: Notice::Info)
|
|||
if ( ! n?$ts )
|
||||
n$ts = network_time();
|
||||
|
||||
if ( n?$f )
|
||||
{
|
||||
if ( ! n?$fuid )
|
||||
n$fuid = n$f$id;
|
||||
|
||||
if ( ! n?$file_mime_type && n$f?$mime_type )
|
||||
n$file_mime_type = n$f$mime_type;
|
||||
|
||||
n$file_desc = Files::describe(n$f);
|
||||
|
||||
if ( n$f?$conns && |n$f$conns| == 1 )
|
||||
{
|
||||
for ( id in n$f$conns )
|
||||
n$conn = n$f$conns[id];
|
||||
}
|
||||
}
|
||||
|
||||
if ( n?$conn )
|
||||
{
|
||||
if ( ! n?$id )
|
||||
n$id = n$conn$id;
|
||||
|
||||
if ( ! n?$uid )
|
||||
n$uid = n$conn$uid;
|
||||
}
|
||||
|
|
@ -513,13 +550,15 @@ function apply_policy(n: Notice::Info)
|
|||
if ( ! n?$suppress_for )
|
||||
n$suppress_for = default_suppression_interval;
|
||||
|
||||
# Delete the connection record if it's there so we aren't sending that
|
||||
# to remote machines. It can cause problems due to the size of the
|
||||
# connection record.
|
||||
# Delete the connection and file records if they're there so we
|
||||
# aren't sending that to remote machines. It can cause problems
|
||||
# due to the size of those records.
|
||||
if ( n?$conn )
|
||||
delete n$conn;
|
||||
if ( n?$iconn )
|
||||
delete n$iconn;
|
||||
if ( n?$f )
|
||||
delete n$f;
|
||||
}
|
||||
|
||||
function internal_NOTICE(n: Notice::Info)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue