mirror of
https://github.com/zeek/zeek.git
synced 2025-10-07 17:18:20 +00:00
Merge remote-tracking branch 'origin/topic/seth/files-tracking' into topic/seth/files-reassembly-and-mime-updates
Conflicts: testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log
This commit is contained in:
Seth Hall
commit
842dfd8b4a
55 changed files with 868 additions and 590 deletions
|
|
@ -100,8 +100,9 @@ export {
|
|||
## during the process of analysis e.g. due to dropped packets.
|
||||
missing_bytes: count &log &default=0;
|
||||
|
||||
## The number of not all-in-sequence bytes in the file stream that
|
||||
## were delivered to file analyzers due to reassembly buffer overflow.
|
||||
## The number of bytes in the file stream that were not delivered to
|
||||
## stream file analyzers. This could be overlapping bytes or
|
||||
## bytes that couldn't be reassembled.
|
||||
overflow_bytes: count &log &default=0;
|
||||
|
||||
## Whether the file analysis timed out at least once for the file.
|
||||
|
|
@ -124,6 +125,37 @@ export {
|
|||
## generate two handles that would hash to the same file id.
|
||||
const salt = "I recommend changing this." &redef;
|
||||
|
||||
## Decide if you want to automatically attached analyzers to
|
||||
## files based on the detected mime type of the file.
|
||||
const analyze_by_mime_type_automatically = T &redef;
|
||||
|
||||
## The default setting for if the file reassembler is enabled for
|
||||
## each file.
|
||||
const enable_reassembler = T &redef;
|
||||
|
||||
## The default allow per-file reassembly buffer size.
|
||||
const reassembly_buffer_size = 1048576 &redef;
|
||||
|
||||
## Allows the file reassembler to be used if it's necessary because the
|
||||
## file is transferred out of order.
|
||||
##
|
||||
## f: the file.
|
||||
global enable_reassembly: function(f: fa_file);
|
||||
|
||||
## Disables the file reassembler on this file. If the file is not
|
||||
## transferred out of order this will have no effect.
|
||||
##
|
||||
## f: the file.
|
||||
global disable_reassembly: function(f: fa_file);
|
||||
|
||||
## Set the maximum size the reassembly buffer is allowed to grow
|
||||
## for the given file.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## max: Maximum allowed size of the reassembly buffer.
|
||||
global set_reassembly_buffer_size: function(f: fa_file, max: count);
|
||||
|
||||
## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
|
||||
## used to determine the length of inactivity that is allowed for a file
|
||||
## before internal state related to it is cleaned up. When used within
|
||||
|
|
@ -153,15 +185,6 @@ export {
|
|||
tag: Files::Tag,
|
||||
args: AnalyzerArgs &default=AnalyzerArgs()): bool;
|
||||
|
||||
## Adds all analyzers associated with a give MIME type to the analysis of
|
||||
## a file. Note that analyzers added via MIME types cannot take further
|
||||
## arguments.
|
||||
##
|
||||
## f: the file.
|
||||
##
|
||||
## mtype: the MIME type; it will be compared case-insensitive.
|
||||
global add_analyzers_for_mime_type: function(f: fa_file, mtype: string);
|
||||
|
||||
## Removes an analyzer from the analysis of a given file.
|
||||
##
|
||||
## f: the file.
|
||||
|
|
@ -284,6 +307,7 @@ global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table()
|
|||
|
||||
# Store the MIME type to analyzer mappings.
|
||||
global mime_types: table[Analyzer::Tag] of set[string];
|
||||
global mime_type_to_analyzers: table[string] of set[Analyzer::Tag];
|
||||
|
||||
global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
|
||||
|
||||
|
|
@ -313,8 +337,6 @@ function set_info(f: fa_file)
|
|||
f$info$overflow_bytes = f$overflow_bytes;
|
||||
if ( f?$is_orig )
|
||||
f$info$is_orig = f$is_orig;
|
||||
if ( f?$mime_type )
|
||||
f$info$mime_type = f$mime_type;
|
||||
}
|
||||
|
||||
function set_timeout_interval(f: fa_file, t: interval): bool
|
||||
|
|
@ -322,6 +344,21 @@ function set_timeout_interval(f: fa_file, t: interval): bool
|
|||
return __set_timeout_interval(f$id, t);
|
||||
}
|
||||
|
||||
function enable_reassembly(f: fa_file)
|
||||
{
|
||||
__enable_reassembly(f$id);
|
||||
}
|
||||
|
||||
function disable_reassembly(f: fa_file)
|
||||
{
|
||||
__disable_reassembly(f$id);
|
||||
}
|
||||
|
||||
function set_reassembly_buffer_size(f: fa_file, max: count)
|
||||
{
|
||||
__set_reassembly_buffer(f$id, max);
|
||||
}
|
||||
|
||||
function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
|
||||
{
|
||||
add f$info$analyzers[Files::analyzer_name(tag)];
|
||||
|
|
@ -337,15 +374,6 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
|
|||
return T;
|
||||
}
|
||||
|
||||
function add_analyzers_for_mime_type(f: fa_file, mtype: string)
|
||||
{
|
||||
local dummy_args: AnalyzerArgs;
|
||||
local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args);
|
||||
|
||||
for ( tag in analyzers )
|
||||
add f$info$analyzers[Files::analyzer_name(tag)];
|
||||
}
|
||||
|
||||
function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
|
||||
{
|
||||
analyzer_add_callbacks[tag] = callback;
|
||||
|
|
@ -366,42 +394,6 @@ function analyzer_name(tag: Files::Tag): string
|
|||
return __analyzer_name(tag);
|
||||
}
|
||||
|
||||
event file_new(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
|
||||
if ( f?$mime_type )
|
||||
add_analyzers_for_mime_type(f, f$mime_type);
|
||||
}
|
||||
|
||||
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
add f$info$conn_uids[c$uid];
|
||||
local cid = c$id;
|
||||
add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
|
||||
if( |Site::local_nets| > 0 )
|
||||
f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h);
|
||||
|
||||
add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
|
||||
}
|
||||
|
||||
event file_timeout(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
f$info$timedout = T;
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=-10
|
||||
{
|
||||
Log::write(Files::LOG, f$info);
|
||||
}
|
||||
|
||||
function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
|
||||
{
|
||||
local result = (tag !in registered_protocols);
|
||||
|
|
@ -424,13 +416,18 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) :
|
|||
|
||||
function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
|
||||
{
|
||||
if ( ! __register_for_mime_type(tag, mt) )
|
||||
return F;
|
||||
|
||||
if ( tag !in mime_types )
|
||||
{
|
||||
mime_types[tag] = set();
|
||||
|
||||
}
|
||||
add mime_types[tag][mt];
|
||||
|
||||
if ( mt !in mime_type_to_analyzers )
|
||||
{
|
||||
mime_type_to_analyzers[mt] = set();
|
||||
}
|
||||
add mime_type_to_analyzers[mt][tag];
|
||||
|
||||
return T;
|
||||
}
|
||||
|
||||
|
|
@ -462,3 +459,62 @@ event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priorit
|
|||
local handler = registered_protocols[tag];
|
||||
set_file_handle(handler$get_file_handle(c, is_orig));
|
||||
}
|
||||
|
||||
event file_new(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
|
||||
if ( enable_reassembler )
|
||||
{
|
||||
Files::enable_reassembly(f);
|
||||
Files::set_reassembly_buffer_size(f, reassembly_buffer_size);
|
||||
}
|
||||
}
|
||||
|
||||
event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
|
||||
add f$info$conn_uids[c$uid];
|
||||
local cid = c$id;
|
||||
add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
|
||||
if( |Site::local_nets| > 0 )
|
||||
f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h);
|
||||
|
||||
add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
|
||||
}
|
||||
|
||||
event file_mime_type(f: fa_file, mime_type: string) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
|
||||
f$info$mime_type = mime_type;
|
||||
|
||||
|
||||
if ( analyze_by_mime_type_automatically &&
|
||||
mime_type in mime_type_to_analyzers )
|
||||
{
|
||||
local analyzers = mime_type_to_analyzers[mime_type];
|
||||
for ( a in analyzers )
|
||||
{
|
||||
add f$info$analyzers[Files::analyzer_name(a)];
|
||||
Files::add_analyzer(f, a);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
event file_timeout(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
f$info$timedout = T;
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=10
|
||||
{
|
||||
set_info(f);
|
||||
}
|
||||
|
||||
event file_state_remove(f: fa_file) &priority=-10
|
||||
{
|
||||
Log::write(Files::LOG, f$info);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -289,8 +289,8 @@ event Intel::match(s: Seen, items: set[Item]) &priority=5
|
|||
if ( ! info?$fuid )
|
||||
info$fuid = s$f$id;
|
||||
|
||||
if ( ! info?$file_mime_type && s$f?$mime_type )
|
||||
info$file_mime_type = s$f$mime_type;
|
||||
if ( ! info?$file_mime_type && s$f?$info && s$f$info?$mime_type )
|
||||
info$file_mime_type = s$f$info$mime_type;
|
||||
|
||||
if ( ! info?$file_desc )
|
||||
info$file_desc = Files::describe(s$f);
|
||||
|
|
|
|||
|
|
@ -531,8 +531,8 @@ function create_file_info(f: fa_file): Notice::FileInfo
|
|||
local fi: Notice::FileInfo = Notice::FileInfo($fuid = f$id,
|
||||
$desc = Files::describe(f));
|
||||
|
||||
if ( f?$mime_type )
|
||||
fi$mime = f$mime_type;
|
||||
if ( f?$info && f$info?$mime_type )
|
||||
fi$mime = f$info$mime_type;
|
||||
|
||||
if ( f?$conns && |f$conns| == 1 )
|
||||
for ( id in f$conns )
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue