mirror of
https://github.com/zeek/zeek.git
synced 2025-10-04 23:58:20 +00:00
Merge remote-tracking branch 'origin/topic/jsiwek/file-analysis' into topic/seth/file-analysis-exe-analyzer
Conflicts: src/file_analysis/ActionSet.cc
This commit is contained in:
Seth Hall
commit
a624dd61c0
27 changed files with 80205 additions and 159553 deletions
|
|
@ -1,8 +1,7 @@
|
|||
##! TODO add some comments here
|
||||
##! An interface for driving the analysis of files, possibly independent of
|
||||
##! any network protocol over which they're transported.
|
||||
|
||||
@load base/file_analysis.bif
|
||||
|
||||
# TODO: do logging here?
|
||||
@load base/frameworks/logging
|
||||
|
||||
module FileAnalysis;
|
||||
|
|
@ -13,10 +12,6 @@ export {
|
|||
LOG
|
||||
};
|
||||
|
||||
## The default buffer size used to reassemble files.
|
||||
# TODO: what's a reasonable default?
|
||||
const default_reassembly_buffer_size: count = 1024*1024 &redef;
|
||||
|
||||
## The default buffer size used for storing the beginning of files.
|
||||
const default_bof_buffer_size: count = 1024 &redef;
|
||||
|
||||
|
|
@ -24,29 +19,32 @@ export {
|
|||
## before giving up.
|
||||
const default_timeout_interval: interval = 2 mins &redef;
|
||||
|
||||
# Needed a forward declaration for event parameters...
|
||||
type Info: record {};
|
||||
|
||||
## A structure which represents a desired file analysis action to take.
|
||||
type ActionArgs: record {
|
||||
## The type of action.
|
||||
act: Action;
|
||||
## The local filename to which to write an extracted file. Must be
|
||||
## set when *act* is :bro:see:`FileAnalysis::ACTION_EXTRACT`.
|
||||
extract_filename: string &optional;
|
||||
chunk_event: event(info: Info, data: string, off: count) &optional;
|
||||
stream_event: event(info: Info, data: string) &optional;
|
||||
};
|
||||
|
||||
## A structure which contains the results of certain file analysis actions.
|
||||
type ActionResults: record {
|
||||
## An MD5 digest of the file contents.
|
||||
md5: string &optional;
|
||||
## An SHA1 digest of the file contents.
|
||||
sha1: string &optional;
|
||||
## An SHA256 digest of the file contents.
|
||||
sha256: string &optional;
|
||||
};
|
||||
|
||||
## Contains all metadata related to the analysis of a given file, some
|
||||
## of which is logged.
|
||||
## Contains all metadata related to the analysis of a given file.
|
||||
type Info: record {
|
||||
## Unique identifier associated with a single file.
|
||||
## An identifier associated with a single file.
|
||||
file_id: string &log;
|
||||
## Unique identifier associated with the file if it was extracted
|
||||
## from a container file as part of the analysis.
|
||||
|
||||
## Identifier associated with a container file from which this one was
|
||||
## extracted as part of the file analysis.
|
||||
parent_file_id: string &log &optional;
|
||||
|
||||
## An identification of the source of the file data. E.g. it may be
|
||||
|
|
@ -62,18 +60,17 @@ export {
|
|||
|
||||
## Number of bytes provided to the file analysis engine for the file.
|
||||
seen_bytes: count &log &default=0;
|
||||
## Total number of bytes that are supposed to comprise the file content.
|
||||
|
||||
## Total number of bytes that are supposed to comprise the full file.
|
||||
total_bytes: count &log &optional;
|
||||
|
||||
## The number of bytes in the file stream that were completely missed
|
||||
## during the process of analysis e.g. due to dropped packets.
|
||||
## analysis that had to be discarded due to a reassembly buffer size
|
||||
## of *reassembly_buffer_size* being filled.
|
||||
missing_bytes: count &log &default=0;
|
||||
|
||||
## The number of not all-in-sequence bytes in the file stream that
|
||||
## were delivered to file actions/analyzers due to reassembly buffer
|
||||
## size of *reassembly_buffer_size* being filled.
|
||||
## overflow.
|
||||
overflow_bytes: count &log &default=0;
|
||||
|
||||
## The amount of time between receiving new data for this file that
|
||||
|
|
@ -81,61 +78,246 @@ export {
|
|||
timeout_interval: interval &log &default=default_timeout_interval;
|
||||
|
||||
## The number of bytes at the beginning of a file to save for later
|
||||
## inspection in *bof_buffer* field of
|
||||
## :bro:see:`FileAnalysis::ActionResults`.
|
||||
## inspection in *bof_buffer* field.
|
||||
bof_buffer_size: count &log &default=default_bof_buffer_size;
|
||||
|
||||
## The content of the beginning of a file up to *bof_buffer_size* bytes.
|
||||
## This is also the buffer that's used for file/mime type detection.
|
||||
bof_buffer: string &optional;
|
||||
|
||||
## An initial guess at file type.
|
||||
## A file type provided by libmagic against the *bof_buffer*, or
|
||||
## in the cases where no buffering of the beginning of file occurs,
|
||||
## an initial guess of the file type based on the first data seen.
|
||||
file_type: string &log &optional;
|
||||
## An initial guess at mime type.
|
||||
|
||||
## A mime type provided by libmagic against the *bof_buffer*, or
|
||||
## in the cases where no buffering of the beginning of file occurs,
|
||||
## an initial guess of the mime type based on the first data seen.
|
||||
mime_type: string &log &optional;
|
||||
|
||||
## Actions that have been added to the analysis of this file.
|
||||
## Not meant to be modified directly by scripts.
|
||||
## Only meant for inspection by user scripts, not direct modification.
|
||||
actions: table[ActionArgs] of ActionResults;
|
||||
} &redef;
|
||||
|
||||
## TODO: document
|
||||
## Fields that are derived from existing ones, and are set just in time
|
||||
## for logging purposes.
|
||||
redef record FileAnalysis::Info += {
|
||||
## Whether the file analysis timed out at least once for the file.
|
||||
timedout: bool &log &default=F;
|
||||
|
||||
## Connection UIDS over which the file was transferred.
|
||||
conn_uids: set[string] &log &optional;
|
||||
|
||||
## A set of action types taken during the file analysis.
|
||||
actions_taken: set[Action] &log &optional;
|
||||
|
||||
## Local filenames of file extraction actions.
|
||||
extracted_files: set[string] &log &optional;
|
||||
|
||||
## An MD5 digest of the file contents.
|
||||
md5: string &log &optional;
|
||||
|
||||
## A SHA1 digest of the file contents.
|
||||
sha1: string &log &optional;
|
||||
|
||||
## A SHA256 digest of the file contents.
|
||||
sha256: string &log &optional;
|
||||
};
|
||||
|
||||
## Redefined here just so the *info* parameters of the events have the
|
||||
## right type information.
|
||||
redef record ActionArgs += {
|
||||
## An event which will be generated for all new file contents,
|
||||
## chunk-wise.
|
||||
chunk_event: event(info: Info, data: string, off: count) &optional;
|
||||
|
||||
## An event which will be generated for all new file contents,
|
||||
## stream-wise.
|
||||
stream_event: event(info: Info, data: string) &optional;
|
||||
};
|
||||
|
||||
## Evaluated every time a significant event occurs during the course of
|
||||
## file analysis. Fields of the *info* argument may be modified or
|
||||
## other actions may be added or removed inside the body of any handlers
|
||||
## of this hook.
|
||||
global policy: hook(trig: Trigger, info: Info);
|
||||
|
||||
## A table that can be used to disable file analysis completely for
|
||||
## any files transferred over given network protocol analyzers.
|
||||
const disable: table[AnalyzerTag] of bool = table() &redef;
|
||||
|
||||
# TODO: wrapper functions for BiFs ?
|
||||
|
||||
## Event that can be handled to access the Info record as it is sent on
|
||||
## to the logging framework.
|
||||
global log_file_analysis: event(rec: Info);
|
||||
|
||||
## The salt concatenated to unique file handle strings generated by
|
||||
## :bro:see:`FileAnalysis::handle_callbacks` before hashing them
|
||||
## in to a file id (the *file_id* field of :bro:see:`FileAnalysis::Info`).
|
||||
## :bro:see:`get_file_handle` before hashing them in to a file id
|
||||
## (the *file_id* field of :bro:see:`FileAnalysis::Info`).
|
||||
## Provided to help mitigate the possiblility of manipulating parts of
|
||||
## network connections that factor in to the file handle in order to
|
||||
## generate two handles that would hash to the same file id.
|
||||
const salt = "I recommend changing this." &redef;
|
||||
|
||||
## Postpones the timeout of file analysis for a given file.
|
||||
## When used within a :bro:see:`FileAnalysis::policy` handler for
|
||||
## :bro:see:`FileAnalysis::TRIGGER_TIMEOUT`, the analysis will delay
|
||||
## timing out for the period of time indicated by the *timeout_interval*
|
||||
## field of :bro:see:`FileAnalysis::Info`.
|
||||
##
|
||||
## file_id: the file identifier string from the *file_id* field of
|
||||
## :bro:see:`FileAnalysis::Info`.
|
||||
##
|
||||
## Returns: true if the timeout will be postponed, or false if analysis
|
||||
## for the *file_id* isn't currently active.
|
||||
global postpone_timeout: function(file_id: string): bool;
|
||||
|
||||
## Adds an action to the analysis of a given file.
|
||||
##
|
||||
## file_id: the file identifier string from the *file_id* field of
|
||||
## :bro:see:`FileAnalysis::Info`.
|
||||
##
|
||||
## args: the action type to add along with any arguments it takes.
|
||||
##
|
||||
## Returns: true if the action will be added, or false if analysis
|
||||
## for the *file_id* isn't currently active or the *args*
|
||||
## were invalid for the action type.
|
||||
global add_action: function(file_id: string, args: ActionArgs): bool;
|
||||
|
||||
## Removes an action from the analysis of a given file.
|
||||
##
|
||||
## file_id: the file identifier string from the *file_id* field of
|
||||
## :bro:see:`FileAnalysis::Info`.
|
||||
##
|
||||
## args: the action (type and args) to remove.
|
||||
##
|
||||
## Returns: true if the action will be removed, or false if analysis
|
||||
## for the *file_id* isn't currently active.
|
||||
global remove_action: function(file_id: string, args: ActionArgs): bool;
|
||||
|
||||
## Stops/ignores any further analysis of a given file.
|
||||
##
|
||||
## file_id: the file identifier string from the *file_id* field of
|
||||
## :bro:see:`FileAnalysis::Info`.
|
||||
##
|
||||
## Returns: true if analysis for the given file will be ignored for the
|
||||
## rest of it's contents, or false if analysis for the *file_id*
|
||||
## isn't currently active.
|
||||
global stop: function(file_id: string): bool;
|
||||
|
||||
## Sends a sequential stream of data in for file analysis.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## data: bytestring contents of the file to analyze.
|
||||
global data_stream: function(source: string, data: string);
|
||||
|
||||
## Sends a non-sequential chunk of data in for file analysis.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## data: bytestring contents of the file to analyze.
|
||||
##
|
||||
## offset: the offset within the file that this chunk starts.
|
||||
global data_chunk: function(source: string, data: string, offset: count);
|
||||
|
||||
## Signals a content gap in the file bytestream.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## offset: the offset within the file that this gap starts.
|
||||
##
|
||||
## len: the number of bytes that are missing.
|
||||
global gap: function(source: string, offset: count, len: count);
|
||||
|
||||
## Signals the total size of a file.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
##
|
||||
## size: the number of bytes that comprise the full file.
|
||||
global set_size: function(source: string, size: count);
|
||||
|
||||
## Signals the end of a file.
|
||||
## Meant for use when providing external file analysis input (e.g.
|
||||
## from the input framework).
|
||||
##
|
||||
## source: a string that uniquely identifies the logical file that the
|
||||
## data is a part of and describes its source.
|
||||
global eof: function(source: string);
|
||||
}
|
||||
|
||||
function postpone_timeout(file_id: string): bool
|
||||
{
|
||||
return __postpone_timeout(file_id);
|
||||
}
|
||||
|
||||
function add_action(file_id: string, args: ActionArgs): bool
|
||||
{
|
||||
return __add_action(file_id, args);
|
||||
}
|
||||
|
||||
function remove_action(file_id: string, args: ActionArgs): bool
|
||||
{
|
||||
return __remove_action(file_id, args);
|
||||
}
|
||||
|
||||
function stop(file_id: string): bool
|
||||
{
|
||||
return __stop(file_id);
|
||||
}
|
||||
|
||||
function data_stream(source: string, data: string)
|
||||
{
|
||||
__data_stream(source, data);
|
||||
}
|
||||
|
||||
function data_chunk(source: string, data: string, offset: count)
|
||||
{
|
||||
__data_chunk(source, data, offset);
|
||||
}
|
||||
|
||||
function gap(source: string, offset: count, len: count)
|
||||
{
|
||||
__gap(source, offset, len);
|
||||
}
|
||||
|
||||
function set_size(source: string, size: count)
|
||||
{
|
||||
__set_size(source, size);
|
||||
}
|
||||
|
||||
function eof(source: string)
|
||||
{
|
||||
__eof(source);
|
||||
}
|
||||
|
||||
event bro_init() &priority=5
|
||||
{
|
||||
Log::create_stream(FileAnalysis::LOG,
|
||||
[$columns=Info, $ev=log_file_analysis]);
|
||||
}
|
||||
|
||||
redef record FileAnalysis::Info += {
|
||||
conn_uids: set[string] &log &optional;
|
||||
actions_taken: set[Action] &log &optional;
|
||||
extracted_files: set[string] &log &optional;
|
||||
md5: string &log &optional;
|
||||
sha1: string &log &optional;
|
||||
sha256: string &log &optional;
|
||||
};
|
||||
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
|
||||
&priority=5
|
||||
{
|
||||
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
|
||||
info$timedout = T;
|
||||
}
|
||||
|
||||
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
|
||||
&priority=-10
|
||||
&priority=-5
|
||||
{
|
||||
if ( trig != FileAnalysis::TRIGGER_EOF &&
|
||||
trig != FileAnalysis::TRIGGER_DONE ) return;
|
||||
|
|
|
|||
|
|
@ -18,9 +18,6 @@ export {
|
|||
|
||||
local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p];
|
||||
|
||||
local rval = fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time,
|
||||
id_string(c$id));
|
||||
|
||||
if ( info$passive )
|
||||
# FTP client initiates data channel.
|
||||
if ( is_orig )
|
||||
|
|
@ -28,12 +25,14 @@ export {
|
|||
return "";
|
||||
else
|
||||
# Do care about FTP server data.
|
||||
return rval;
|
||||
return cat(ANALYZER_FTP_DATA, " ", c$start_time, " ",
|
||||
id_string(c$id));
|
||||
else
|
||||
# FTP server initiates dta channel.
|
||||
if ( is_orig )
|
||||
# Do care about FTP server data.
|
||||
return rval;
|
||||
return cat(ANALYZER_FTP_DATA, " ", c$start_time, " ",
|
||||
id_string(c$id));
|
||||
else
|
||||
# Don't care about FTP client data.
|
||||
return "";
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ export {
|
|||
## Reply message from the server in response to the command.
|
||||
reply_msg: string &log &optional;
|
||||
## Arbitrary tags that may indicate a particular attribute of this command.
|
||||
tags: set[string] &log &default=set();
|
||||
tags: set[string] &log;
|
||||
|
||||
## Expected FTP data channel.
|
||||
data_channel: ExpectedDataChannel &log &optional;
|
||||
|
|
@ -109,6 +109,7 @@ export {
|
|||
# Add the state tracking information variable to the connection record
|
||||
redef record connection += {
|
||||
ftp: Info &optional;
|
||||
ftp_data_reuse: bool &default=F;
|
||||
};
|
||||
|
||||
# Configure DPD
|
||||
|
|
@ -337,7 +338,6 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
|
|||
}
|
||||
}
|
||||
|
||||
|
||||
event expected_connection_seen(c: connection, a: count) &priority=10
|
||||
{
|
||||
local id = c$id;
|
||||
|
|
@ -357,8 +357,15 @@ event file_transferred(c: connection, prefix: string, descr: string,
|
|||
}
|
||||
}
|
||||
|
||||
event connection_reused(c: connection) &priority=5
|
||||
{
|
||||
if ( "ftp-data" in c$service )
|
||||
c$ftp_data_reuse = T;
|
||||
}
|
||||
|
||||
event connection_state_remove(c: connection) &priority=-5
|
||||
{
|
||||
if ( c$ftp_data_reuse ) return;
|
||||
delete ftp_data_expected[c$id$resp_h, c$id$resp_p];
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -18,11 +18,11 @@ export {
|
|||
if ( ! c?$http ) return "";
|
||||
|
||||
if ( c$http$range_request )
|
||||
return fmt("%s %s %s %s", ANALYZER_HTTP, is_orig, c$id$orig_h,
|
||||
build_url(c$http));
|
||||
return cat(ANALYZER_HTTP, " ", is_orig, " ", c$id$orig_h,
|
||||
" ", build_url(c$http));
|
||||
|
||||
return fmt("%s %s %s %s %s", ANALYZER_HTTP, c$start_time, is_orig,
|
||||
c$http$trans_depth, id_string(c$id));
|
||||
return cat(ANALYZER_HTTP, " ", c$start_time, " ", is_orig,
|
||||
" ", c$http$trans_depth, " ", id_string(c$id));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ module HTTP;
|
|||
|
||||
export {
|
||||
redef enum Notice::Type += {
|
||||
## Indicates when the file extension doesn't seem to match the file contents.
|
||||
## Indicates when the file extension doesn't seem to match the file
|
||||
## contents.
|
||||
Incorrect_File_Type,
|
||||
};
|
||||
|
||||
|
|
@ -18,9 +19,10 @@ export {
|
|||
mime_type: string &log &optional;
|
||||
};
|
||||
|
||||
## Mapping between mime types and regular expressions for URLs
|
||||
## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the pattern
|
||||
## doesn't match the mime type that was discovered.
|
||||
## Mapping between mime type strings (without character set) and
|
||||
## regular expressions for URLs.
|
||||
## The :bro:enum:`HTTP::Incorrect_File_Type` notice is generated if the
|
||||
## pattern doesn't match the mime type that was discovered.
|
||||
const mime_types_extensions: table[string] of pattern = {
|
||||
["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/,
|
||||
} &redef;
|
||||
|
|
@ -49,17 +51,66 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
|
|||
|
||||
c$http$mime_type = info$mime_type;
|
||||
|
||||
if ( info$mime_type !in mime_types_extensions ) next;
|
||||
local mime_str: string = split1(info$mime_type, /;/)[1];
|
||||
|
||||
if ( mime_str !in mime_types_extensions ) next;
|
||||
if ( ! c$http?$uri ) next;
|
||||
if ( mime_types_extensions[info$mime_type] in c$http$uri ) next;
|
||||
if ( mime_types_extensions[mime_str] in c$http$uri ) next;
|
||||
|
||||
local url = build_url_http(c$http);
|
||||
|
||||
if ( url == ignored_incorrect_file_type_urls ) next;
|
||||
|
||||
local message = fmt("%s %s %s", info$mime_type, c$http$method, url);
|
||||
local message = fmt("%s %s %s", mime_str, c$http$method, url);
|
||||
NOTICE([$note=Incorrect_File_Type,
|
||||
$msg=message,
|
||||
$conn=c]);
|
||||
}
|
||||
}
|
||||
|
||||
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
|
||||
&priority=5
|
||||
{
|
||||
if ( trig != FileAnalysis::TRIGGER_NEW_CONN ) return;
|
||||
if ( ! info?$mime_type ) return;
|
||||
if ( ! info?$source ) return;
|
||||
if ( info$source != "HTTP" ) return;
|
||||
if ( ! info?$conns ) return;
|
||||
|
||||
# Spread the mime around (e.g. for partial content, TRIGGER_TYPE only
|
||||
# happens once for the first connection, but if there's subsequent
|
||||
# connections to transfer the same file, they'll be lacking the mime_type
|
||||
# field if we don't do this).
|
||||
for ( cid in info$conns )
|
||||
{
|
||||
local c: connection = info$conns[cid];
|
||||
|
||||
if ( ! c?$http ) next;
|
||||
|
||||
c$http$mime_type = info$mime_type;
|
||||
}
|
||||
}
|
||||
|
||||
# Tracks byte-range request / partial content response mime types, indexed
|
||||
# by [connection, uri] pairs. This is needed because a person can pipeline
|
||||
# byte-range requests over multiple connections to the same uri. Without
|
||||
# the tracking, only the first request in the pipeline for each connection
|
||||
# would get a mime_type field assigned to it (by the FileAnalysis policy hooks).
|
||||
global partial_types: table[conn_id, string] of string &read_expire=5mins;
|
||||
|
||||
# Priority 4 so that it runs before the handler that will write to http.log.
|
||||
event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
|
||||
&priority=4
|
||||
{
|
||||
if ( ! c$http$range_request ) return;
|
||||
if ( ! c$http?$uri ) return;
|
||||
|
||||
if ( c$http?$mime_type )
|
||||
{
|
||||
partial_types[c$id, c$http$uri] = c$http$mime_type;
|
||||
return;
|
||||
}
|
||||
|
||||
if ( [c$id, c$http$uri] in partial_types )
|
||||
c$http$mime_type = partial_types[c$id, c$http$uri];
|
||||
}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ export {
|
|||
function get_file_handle(c: connection, is_orig: bool): string
|
||||
{
|
||||
if ( is_orig ) return "";
|
||||
return fmt("%s %s %s", ANALYZER_IRC_DATA, c$start_time,
|
||||
return cat(ANALYZER_IRC_DATA, " ", c$start_time, " ",
|
||||
id_string(c$id));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -112,6 +112,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
|
|||
local c: connection = info$conns[cid];
|
||||
|
||||
if ( ! c?$smtp ) next;
|
||||
if ( ! c$smtp?$current_entity ) next;
|
||||
|
||||
if ( c$smtp$current_entity$extract_file )
|
||||
{
|
||||
|
|
@ -180,6 +181,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
|
|||
local c: connection = info$conns[cid];
|
||||
|
||||
if ( ! c?$smtp ) next;
|
||||
if ( ! c$smtp?$current_entity ) next;
|
||||
|
||||
c$smtp$current_entity$mime_type = info$mime_type;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,8 +17,8 @@ export {
|
|||
{
|
||||
if ( ! c?$smtp ) return "";
|
||||
|
||||
return fmt("%s %s %s %s", ANALYZER_SMTP, c$start_time,
|
||||
c$smtp$trans_depth, c$smtp_state$mime_level);
|
||||
return cat(ANALYZER_SMTP, " ", c$start_time, " ",
|
||||
c$smtp$trans_depth, " ", c$smtp_state$mime_level);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -118,6 +118,9 @@ public:
|
|||
|
||||
void Describe(ODesc* d) const;
|
||||
|
||||
EventHandlerPtr Tail() { return tail ? tail->handler : EventHandlerPtr(); }
|
||||
val_list* TailArgs() { return tail ? tail->args : 0; }
|
||||
|
||||
protected:
|
||||
void QueueEvent(Event* event);
|
||||
|
||||
|
|
|
|||
|
|
@ -6986,7 +6986,8 @@ event bro_script_loaded%(path: string, level: count%);
|
|||
## belongs. All incoming data to the framework is buffered, and depends
|
||||
## on a handler for this event to return a string value that uniquely
|
||||
## identifies a file. Among all handlers of this event, exactly one must
|
||||
## call :bro:see:`return_file_handle`.
|
||||
## call :bro:see:`return_file_handle`. Handlers of this event must not
|
||||
## change any global state.
|
||||
##
|
||||
## tag: The analyzer which is carrying the file data.
|
||||
##
|
||||
|
|
|
|||
|
|
@ -10,18 +10,23 @@ type Info: record;
|
|||
type ActionArgs: record;
|
||||
type ActionResults: record;
|
||||
|
||||
## An enumeration of possibly-interesting "events" that can occur over
|
||||
## the course of analyzing files. The :bro:see:`FileAnalysis::policy`
|
||||
## hook is called each time a trigger occurs.
|
||||
## An enumeration of significant things that can occur over the course of
|
||||
## analyzing files. The :bro:see:`FileAnalysis::policy` hook is called each
|
||||
## time a trigger occurs.
|
||||
enum Trigger %{
|
||||
|
||||
## Raised when any part of a new file is detected.
|
||||
TRIGGER_NEW,
|
||||
|
||||
## Raised when file is detected being transported over a new network
|
||||
## connection (other than the first).
|
||||
TRIGGER_NEW_CONN,
|
||||
|
||||
## Raised when file analysis has likely seen a complete file. That
|
||||
## is when a number of bytes indicated by the *total_bytes* field of
|
||||
## :bro:see:`FileAnalysis::Info` have been processed. Note that
|
||||
## the *undelivered* field does not have to be zero for this to have
|
||||
## occurred.
|
||||
## :bro:see:`FileAnalysis::Info` have been processed.
|
||||
TRIGGER_DONE,
|
||||
|
||||
## Raised when file analysis for a given file is aborted due
|
||||
## to not seeing any data for it recently. Note that this doesn't
|
||||
## necessarily mean the full file wasn't seen (e.g. if the
|
||||
|
|
@ -30,45 +35,56 @@ enum Trigger %{
|
|||
## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
|
||||
## defer the timeout until later.
|
||||
TRIGGER_TIMEOUT,
|
||||
|
||||
## Raised when the beginning of a file is detected.
|
||||
TRIGGER_BOF,
|
||||
## Raised when the beginning of a file is available and that beginning
|
||||
## is at least the number of bytes indicated by the *bof_buffer_size*
|
||||
## field of :bro:see:`FileAnalysis::Info`.
|
||||
|
||||
## Raised when the beginning of a file is available in the *bof_buffer*
|
||||
## field of :bro:see:`FileAnalysis::Info` and that beginning
|
||||
## is at least the number of bytes indicated by the *bof_buffer_size* field.
|
||||
TRIGGER_BOF_BUFFER,
|
||||
## Raised when an initial guess at the file/mime type of a file is matched
|
||||
## based on magic numbers.
|
||||
|
||||
## Raised when an initial guess at the file/mime type of a file is matched.
|
||||
TRIGGER_TYPE,
|
||||
|
||||
## Raised to signal that no more file data is incoming and it couldn't be
|
||||
## determined whether the full file was actually seen.
|
||||
## determined whether the full file was actually seen and analyzed.
|
||||
TRIGGER_EOF,
|
||||
## The reassembly buffer for the file filled and had to be discarded.
|
||||
## The *undelivered* field of :bro:see:`FileAnalysis::Info` will
|
||||
## indicate the number of bytes, if any, that were not all-in-sequence.
|
||||
## TODO: Is it possible to extend the reassembly buffer when "handling"
|
||||
## this trigger?
|
||||
TRIGGER_REASSEMBLY_BUFFER_FULL,
|
||||
|
||||
## Raised when there's a missing chunk of data in the file stream.
|
||||
TRIGGER_GAP,
|
||||
%}
|
||||
|
||||
## An enumeration of various file analysis actions that can be taken.
|
||||
enum Action %{
|
||||
|
||||
## Extract a file to local filesystem
|
||||
ACTION_EXTRACT,
|
||||
|
||||
## Calculate an MD5 digest of the file's contents.
|
||||
ACTION_MD5,
|
||||
|
||||
## Calculate an SHA1 digest of the file's contents.
|
||||
ACTION_SHA1,
|
||||
|
||||
## Calculate an SHA256 digest of the file's contents.
|
||||
ACTION_SHA256,
|
||||
|
||||
## Deliver the file contents to the script-layer in an event.
|
||||
ACTION_DATA_EVENT,
|
||||
ACTION_PE_ANALYZER,
|
||||
%}
|
||||
|
||||
function FileAnalysis::postpone_timeout%(file_id: string%): bool
|
||||
## :bro:see:`FileAnalysis::postpone_timeout`.
|
||||
function FileAnalysis::__postpone_timeout%(file_id: string%): bool
|
||||
%{
|
||||
using file_analysis::FileID;
|
||||
bool result = file_mgr->PostponeTimeout(FileID(file_id->CheckString()));
|
||||
return new Val(result, TYPE_BOOL);
|
||||
%}
|
||||
|
||||
function FileAnalysis::add_action%(file_id: string, args: any%): bool
|
||||
## :bro:see:`FileAnalysis::add_action`.
|
||||
function FileAnalysis::__add_action%(file_id: string, args: any%): bool
|
||||
%{
|
||||
using file_analysis::FileID;
|
||||
using BifType::Record::FileAnalysis::ActionArgs;
|
||||
|
|
@ -78,7 +94,8 @@ function FileAnalysis::add_action%(file_id: string, args: any%): bool
|
|||
return new Val(result, TYPE_BOOL);
|
||||
%}
|
||||
|
||||
function FileAnalysis::remove_action%(file_id: string, args: any%): bool
|
||||
## :bro:see:`FileAnalysis::remove_action`.
|
||||
function FileAnalysis::__remove_action%(file_id: string, args: any%): bool
|
||||
%{
|
||||
using file_analysis::FileID;
|
||||
using BifType::Record::FileAnalysis::ActionArgs;
|
||||
|
|
@ -88,39 +105,45 @@ function FileAnalysis::remove_action%(file_id: string, args: any%): bool
|
|||
return new Val(result, TYPE_BOOL);
|
||||
%}
|
||||
|
||||
function FileAnalysis::stop%(file_id: string%): bool
|
||||
## :bro:see:`FileAnalysis::stop`.
|
||||
function FileAnalysis::__stop%(file_id: string%): bool
|
||||
%{
|
||||
using file_analysis::FileID;
|
||||
bool result = file_mgr->IgnoreFile(FileID(file_id->CheckString()));
|
||||
return new Val(result, TYPE_BOOL);
|
||||
%}
|
||||
|
||||
function FileAnalysis::data_stream%(source: string, data: string%): any
|
||||
## :bro:see:`FileAnalysis::data_stream`.
|
||||
function FileAnalysis::__data_stream%(source: string, data: string%): any
|
||||
%{
|
||||
file_mgr->DataIn(data->Bytes(), data->Len(), source->CheckString());
|
||||
return 0;
|
||||
%}
|
||||
|
||||
function FileAnalysis::data_chunk%(source: string, data: string,
|
||||
offset: count%): any
|
||||
## :bro:see:`FileAnalysis::data_chunk`.
|
||||
function FileAnalysis::__data_chunk%(source: string, data: string,
|
||||
offset: count%): any
|
||||
%{
|
||||
file_mgr->DataIn(data->Bytes(), data->Len(), offset, source->CheckString());
|
||||
return 0;
|
||||
%}
|
||||
|
||||
function FileAnalysis::gap%(source: string, offset: count, len: count%): any
|
||||
## :bro:see:`FileAnalysis::gap`.
|
||||
function FileAnalysis::__gap%(source: string, offset: count, len: count%): any
|
||||
%{
|
||||
file_mgr->Gap(offset, len, source->CheckString());
|
||||
return 0;
|
||||
%}
|
||||
|
||||
function FileAnalysis::set_size%(source: string, size: count%): any
|
||||
## :bro:see:`FileAnalysis::set_size`.
|
||||
function FileAnalysis::__set_size%(source: string, size: count%): any
|
||||
%{
|
||||
file_mgr->SetSize(size, source->CheckString());
|
||||
return 0;
|
||||
%}
|
||||
|
||||
function FileAnalysis::eof%(source: string%): any
|
||||
## :bro:see:`FileAnalysis::eof`.
|
||||
function FileAnalysis::__eof%(source: string%): any
|
||||
%{
|
||||
file_mgr->EndOfFile(source->CheckString());
|
||||
return 0;
|
||||
|
|
|
|||
|
|
@ -11,13 +11,11 @@ using namespace file_analysis;
|
|||
|
||||
// keep in order w/ declared enum values in file_analysis.bif
|
||||
static ActionInstantiator action_factory[] = {
|
||||
Extract::Instantiate,
|
||||
MD5::Instantiate,
|
||||
SHA1::Instantiate,
|
||||
SHA256::Instantiate,
|
||||
DataEvent::Instantiate,
|
||||
|
||||
PE_Analyzer::Instantiate,
|
||||
file_analysis::Extract::Instantiate,
|
||||
file_analysis::MD5::Instantiate,
|
||||
file_analysis::SHA1::Instantiate,
|
||||
file_analysis::SHA256::Instantiate,
|
||||
file_analysis::DataEvent::Instantiate,
|
||||
};
|
||||
|
||||
static void action_del_func(void* v)
|
||||
|
|
|
|||
|
|
@ -13,6 +13,11 @@ namespace file_analysis {
|
|||
class Info;
|
||||
declare(PDict,Action);
|
||||
|
||||
/**
|
||||
* A set of file analysis actions indexed by ActionArgs. Allows queueing
|
||||
* of addition/removals so that those modifications can happen at well-defined
|
||||
* times (e.g. to make sure a loop iterator isn't invalidated).
|
||||
*/
|
||||
class ActionSet {
|
||||
public:
|
||||
|
||||
|
|
|
|||
|
|
@ -81,7 +81,8 @@ void Info::StaticInit()
|
|||
|
||||
Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
|
||||
: file_id(""), unique(unique), val(0), postpone_timeout(false),
|
||||
need_reassembly(false), done(false), actions(this)
|
||||
first_chunk(true), need_type(false), need_reassembly(false), done(false),
|
||||
actions(this)
|
||||
{
|
||||
StaticInit();
|
||||
|
||||
|
|
@ -134,11 +135,23 @@ void Info::UpdateConnectionFields(Connection* conn)
|
|||
|
||||
Val* conns = val->Lookup(conns_idx);
|
||||
|
||||
bool is_first = false;
|
||||
|
||||
if ( ! conns )
|
||||
{
|
||||
is_first = true;
|
||||
val->Assign(conns_idx, conns = empty_connection_table());
|
||||
}
|
||||
|
||||
Val* idx = get_conn_id_val(conn);
|
||||
conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
|
||||
if ( ! conns->AsTableVal()->Lookup(idx) )
|
||||
{
|
||||
conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
|
||||
if ( ! is_first )
|
||||
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW_CONN,
|
||||
this);
|
||||
}
|
||||
|
||||
Unref(idx);
|
||||
}
|
||||
|
||||
|
|
@ -162,7 +175,7 @@ int Info::Idx(const string& field)
|
|||
{
|
||||
int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str());
|
||||
if ( rval < 0 )
|
||||
reporter->InternalError("Unkown FileAnalysis::Info field: %s",
|
||||
reporter->InternalError("Unknown FileAnalysis::Info field: %s",
|
||||
field.c_str());
|
||||
return rval;
|
||||
}
|
||||
|
|
@ -172,6 +185,13 @@ double Info::GetTimeoutInterval() const
|
|||
return LookupFieldDefaultInterval(timeout_interval_idx);
|
||||
}
|
||||
|
||||
string Info::GetSource() const
|
||||
{
|
||||
Val* v = val->Lookup(source_idx);
|
||||
if ( ! v ) return "";
|
||||
return v->AsStringVal()->CheckString();
|
||||
}
|
||||
|
||||
RecordVal* Info::GetResults(RecordVal* args) const
|
||||
{
|
||||
TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal();
|
||||
|
|
@ -230,18 +250,6 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
|
|||
|
||||
uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
|
||||
|
||||
/* Leaving out this optimization (I think) for now to keep things simpler.
|
||||
// If first chunk satisfies desired size, do everything now without copying.
|
||||
if ( bof_buffer.chunks.empty() && len >= desired_size )
|
||||
{
|
||||
bof_buffer.full = bof_buffer.replayed = true;
|
||||
val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0)));
|
||||
file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
|
||||
// TODO: libmagic stuff
|
||||
return false;
|
||||
}
|
||||
*/
|
||||
|
||||
bof_buffer.chunks.push_back(new BroString(data, len, 0));
|
||||
bof_buffer.size += len;
|
||||
|
||||
|
|
@ -254,18 +262,10 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
|
|||
return true;
|
||||
}
|
||||
|
||||
void Info::ReplayBOF()
|
||||
bool Info::DetectTypes(const u_char* data, uint64 len)
|
||||
{
|
||||
if ( bof_buffer.replayed ) return;
|
||||
bof_buffer.replayed = true;
|
||||
|
||||
if ( bof_buffer.chunks.empty() ) return;
|
||||
|
||||
BroString* bs = concatenate(bof_buffer.chunks);
|
||||
const char* desc = bro_magic_buffer(magic, bs->Bytes(), bs->Len());
|
||||
const char* mime = bro_magic_buffer(magic_mime, bs->Bytes(), bs->Len());
|
||||
|
||||
val->Assign(bof_buffer_idx, new StringVal(bs));
|
||||
const char* desc = bro_magic_buffer(magic, data, len);
|
||||
const char* mime = bro_magic_buffer(magic_mime, data, len);
|
||||
|
||||
if ( desc )
|
||||
val->Assign(file_type_idx, new StringVal(desc));
|
||||
|
|
@ -273,10 +273,29 @@ void Info::ReplayBOF()
|
|||
if ( mime )
|
||||
val->Assign(mime_type_idx, new StringVal(mime));
|
||||
|
||||
return desc || mime;
|
||||
}
|
||||
|
||||
void Info::ReplayBOF()
|
||||
{
|
||||
if ( bof_buffer.replayed ) return;
|
||||
bof_buffer.replayed = true;
|
||||
|
||||
if ( bof_buffer.chunks.empty() )
|
||||
{
|
||||
// Since we missed the beginning, try file type detect on next data in.
|
||||
need_type = true;
|
||||
return;
|
||||
}
|
||||
|
||||
BroString* bs = concatenate(bof_buffer.chunks);
|
||||
val->Assign(bof_buffer_idx, new StringVal(bs));
|
||||
bool have_type = DetectTypes(bs->Bytes(), bs->Len());
|
||||
|
||||
using BifEnum::FileAnalysis::TRIGGER_BOF_BUFFER;
|
||||
file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
|
||||
|
||||
if ( desc || mime )
|
||||
if ( have_type )
|
||||
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
|
||||
|
||||
for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
|
||||
|
|
@ -286,7 +305,17 @@ void Info::ReplayBOF()
|
|||
void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
|
||||
{
|
||||
actions.DrainModifications();
|
||||
// TODO: attempt libmagic stuff here before doing reassembly?
|
||||
|
||||
if ( first_chunk )
|
||||
{
|
||||
if ( DetectTypes(data, len) )
|
||||
{
|
||||
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
|
||||
actions.DrainModifications();
|
||||
}
|
||||
|
||||
first_chunk = false;
|
||||
}
|
||||
|
||||
Action* act = 0;
|
||||
IterCookie* c = actions.InitForIteration();
|
||||
|
|
@ -316,6 +345,17 @@ void Info::DataIn(const u_char* data, uint64 len)
|
|||
|
||||
if ( BufferBOF(data, len) ) return;
|
||||
|
||||
if ( need_type )
|
||||
{
|
||||
if ( DetectTypes(data, len) )
|
||||
{
|
||||
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
|
||||
actions.DrainModifications();
|
||||
}
|
||||
|
||||
need_type = false;
|
||||
}
|
||||
|
||||
Action* act = 0;
|
||||
IterCookie* c = actions.InitForIteration();
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,11 @@ public:
|
|||
*/
|
||||
double GetTimeoutInterval() const;
|
||||
|
||||
/**
|
||||
* @return value of the "source" field from #val record.
|
||||
*/
|
||||
string GetSource() const;
|
||||
|
||||
/**
|
||||
* @return value of the "file_id" field from #val record.
|
||||
*/
|
||||
|
|
@ -155,10 +160,19 @@ protected:
|
|||
*/
|
||||
void ReplayBOF();
|
||||
|
||||
/**
|
||||
* Does file/mime type detection and assigns types (if available) to
|
||||
* corresponding fields in #val.
|
||||
* @return whether a file or mime type was available.
|
||||
*/
|
||||
bool DetectTypes(const u_char* data, uint64 len);
|
||||
|
||||
FileID file_id; /**< A pretty hash that likely identifies file*/
|
||||
string unique; /**< A string that uniquely identifies file */
|
||||
RecordVal* val; /**< \c FileAnalysis::Info from script layer. */
|
||||
bool postpone_timeout; /**< Whether postponing timeout is requested. */
|
||||
bool first_chunk; /**< Track first non-linear chunk. */
|
||||
bool need_type; /**< Flags next data input to be magic typed. */
|
||||
bool need_reassembly; /**< Whether file stream reassembly is needed. */
|
||||
bool done; /**< If this object is about to be deleted. */
|
||||
ActionSet actions;
|
||||
|
|
|
|||
|
|
@ -34,11 +34,17 @@ void Manager::ReceiveHandle(const string& handle)
|
|||
if ( pending.empty() )
|
||||
reporter->InternalError("File analysis underflow");
|
||||
|
||||
PendingFile* pf = pending.front();
|
||||
if ( ! handle.empty() )
|
||||
pf->Finish(handle);
|
||||
delete pf;
|
||||
pending.pop();
|
||||
int use_count = cache.front();
|
||||
cache.pop();
|
||||
|
||||
for ( int i = 0; i < use_count; ++i )
|
||||
{
|
||||
PendingFile* pf = pending.front();
|
||||
if ( ! handle.empty() )
|
||||
pf->Finish(handle);
|
||||
delete pf;
|
||||
pending.pop();
|
||||
}
|
||||
}
|
||||
|
||||
void Manager::EventDrainDone()
|
||||
|
|
@ -330,11 +336,50 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
|
|||
return rval;
|
||||
}
|
||||
|
||||
static bool CheckArgEquality(AnalyzerTag::Tag tag, Connection* conn,
|
||||
bool is_orig, val_list* other_args)
|
||||
{
|
||||
if ( ! other_args ) return false;
|
||||
if ( (*other_args)[0]->AsCount() != (bro_uint_t) tag ) return false;
|
||||
if ( (*other_args)[2]->AsBool() != is_orig ) return false;
|
||||
|
||||
RecordVal* id = (*other_args)[1]->AsRecordVal()->Lookup(
|
||||
connection_type->FieldOffset("id"))->AsRecordVal();
|
||||
|
||||
PortVal* orig_p = id->Lookup(
|
||||
conn_id->FieldOffset("orig_p"))->AsPortVal();
|
||||
|
||||
if ( orig_p->Port() != ntohs(conn->OrigPort()) ) return false;
|
||||
if ( orig_p->PortType() != conn->ConnTransport() ) return false;
|
||||
|
||||
PortVal* resp_p = id->Lookup(
|
||||
conn_id->FieldOffset("resp_p"))->AsPortVal();
|
||||
|
||||
if ( resp_p->Port() != ntohs(conn->RespPort()) ) return false;
|
||||
|
||||
if ( id->Lookup(conn_id->FieldOffset("orig_h"))->AsAddr() !=
|
||||
conn->OrigAddr() ) return false;
|
||||
|
||||
if ( id->Lookup(conn_id->FieldOffset("resp_h"))->AsAddr() !=
|
||||
conn->RespAddr() ) return false;
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
|
||||
bool is_orig)
|
||||
{
|
||||
if ( ! get_file_handle ) return false;
|
||||
|
||||
if ( mgr.Tail() == get_file_handle &&
|
||||
CheckArgEquality(tag, conn, is_orig, mgr.TailArgs()) )
|
||||
{
|
||||
cache.front()++;
|
||||
return true;
|
||||
}
|
||||
|
||||
cache.push(1);
|
||||
|
||||
val_list* vl = new val_list();
|
||||
vl->append(new Val(tag, TYPE_COUNT));
|
||||
vl->append(conn->BuildConnVal());
|
||||
|
|
|
|||
|
|
@ -130,6 +130,7 @@ protected:
|
|||
typedef set<string> StrSet;
|
||||
typedef map<FileID, Info*> IDMap;
|
||||
typedef queue<PendingFile*> PendingQueue;
|
||||
typedef queue<int> HandleCache;
|
||||
|
||||
/**
|
||||
* @return the Info object mapped to \a unique or a null pointer if analysis
|
||||
|
|
@ -164,22 +165,24 @@ protected:
|
|||
*/
|
||||
bool IsIgnored(const string& unique);
|
||||
|
||||
/**
|
||||
* @return whether file analysis is disabled for the given analyzer.
|
||||
*/
|
||||
static bool IsDisabled(AnalyzerTag::Tag tag);
|
||||
|
||||
/**
|
||||
* Queues \c get_file_handle event in order to retrieve unique file handle.
|
||||
* @return true if there is a handler for the event, else false.
|
||||
*/
|
||||
static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
|
||||
bool is_orig);
|
||||
bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
|
||||
bool is_orig);
|
||||
|
||||
/**
|
||||
* @return whether file analysis is disabled for the given analyzer.
|
||||
*/
|
||||
static bool IsDisabled(AnalyzerTag::Tag tag);
|
||||
|
||||
StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
|
||||
IDMap id_map; /**< Map file IDs to \c FileAnalysis::Info records. */
|
||||
StrSet ignored; /**< Ignored files. Will be finally removed on EOF. */
|
||||
PendingQueue pending; /**< Files awaiting a unique handle. */
|
||||
HandleCache cache; /**< The number of times a received file handle can be
|
||||
used to pop the #pending queue. */
|
||||
|
||||
static TableVal* disabled; /**< Table of disabled analyzers. */
|
||||
};
|
||||
|
|
|
|||
|
|
@ -7,6 +7,10 @@
|
|||
|
||||
namespace file_analysis {
|
||||
|
||||
/**
|
||||
* Provides buffering for file contents until the script-layer is able to
|
||||
* return a unique file handle for it.
|
||||
*/
|
||||
class PendingFile {
|
||||
public:
|
||||
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path http
|
||||
#open 2013-03-22-14-37-46
|
||||
#open 2013-03-28-21-35-15
|
||||
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
|
||||
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
|
||||
1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - - - -
|
||||
#close 2013-03-22-14-37-46
|
||||
1333458850.375568 arKYeMETxOg 10.131.47.185 1923 79.101.110.141 80 1 GET o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com /videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1 http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11 0 56320 206 Partial Content - - - (empty) - - - application/octet-stream; charset=binary - -
|
||||
#close 2013-03-28-21-35-15
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
FileAnalysis::TRIGGER_NEW
|
||||
oDwT1BbzjM1, 0, 0
|
||||
FileAnalysis::TRIGGER_TYPE
|
||||
file type is set
|
||||
mime type is set
|
||||
FileAnalysis::TRIGGER_DONE
|
||||
oDwT1BbzjM1, 1022920, 0
|
||||
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
|
||||
|
|
@ -7,6 +10,9 @@ total bytes: 1022920
|
|||
source: HTTP
|
||||
FileAnalysis::TRIGGER_NEW
|
||||
oDwT1BbzjM1, 0, 0
|
||||
FileAnalysis::TRIGGER_TYPE
|
||||
file type is set
|
||||
mime type is set
|
||||
FileAnalysis::TRIGGER_TIMEOUT
|
||||
FileAnalysis::TRIGGER_TIMEOUT
|
||||
FileAnalysis::TRIGGER_EOF
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
FileAnalysis::TRIGGER_NEW
|
||||
7gZBKVUgy4l, 0, 0
|
||||
FileAnalysis::TRIGGER_TYPE
|
||||
file type is set
|
||||
mime type is set
|
||||
FileAnalysis::TRIGGER_NEW_CONN
|
||||
FileAnalysis::TRIGGER_DONE
|
||||
7gZBKVUgy4l, 555523, 0
|
||||
[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
FileAnalysis::TRIGGER_NEW
|
||||
oDwT1BbzjM1, 0, 0
|
||||
FileAnalysis::TRIGGER_TYPE
|
||||
file type is set
|
||||
mime type is set
|
||||
FileAnalysis::TRIGGER_DONE
|
||||
oDwT1BbzjM1, 1022920, 0
|
||||
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
|
||||
|
|
@ -7,6 +10,9 @@ total bytes: 1022920
|
|||
source: HTTP
|
||||
FileAnalysis::TRIGGER_NEW
|
||||
oDwT1BbzjM1, 0, 0
|
||||
FileAnalysis::TRIGGER_TYPE
|
||||
file type is set
|
||||
mime type is set
|
||||
FileAnalysis::TRIGGER_TIMEOUT
|
||||
FileAnalysis::TRIGGER_EOF
|
||||
oDwT1BbzjM1, 206024, 0
|
||||
|
|
|
|||
|
|
@ -1,5 +1,9 @@
|
|||
FileAnalysis::TRIGGER_NEW
|
||||
uHS14uhRKGe, 0, 0
|
||||
FileAnalysis::TRIGGER_TYPE
|
||||
file type is set
|
||||
mime type is set
|
||||
FileAnalysis::TRIGGER_NEW_CONN
|
||||
FileAnalysis::TRIGGER_DONE
|
||||
uHS14uhRKGe, 498702, 0
|
||||
[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]
|
||||
|
|
|
|||
|
|
@ -3,8 +3,8 @@
|
|||
#empty_field (empty)
|
||||
#unset_field -
|
||||
#path file_analysis
|
||||
#open 2013-03-26-20-26-26
|
||||
#fields file_id parent_file_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type conn_uids actions_taken extracted_files md5 sha1 sha256
|
||||
#types string string string time count count count count interval count string string table[string] table[enum] table[string] string string string
|
||||
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
|
||||
#close 2013-03-26-20-26-26
|
||||
#open 2013-03-29-18-28-57
|
||||
#fields file_id parent_file_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type timedout conn_uids actions_taken extracted_files md5 sha1 sha256
|
||||
#types string string string time count count count count interval count string string bool table[string] table[enum] table[string] string string string
|
||||
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set F UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
|
||||
#close 2013-03-29-18-28-57
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -8,141 +8,141 @@
|
|||
|
||||
event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool)
|
||||
{
|
||||
print "modbus_message", c, headers, is_orig;
|
||||
print "modbus_message", c$id, headers, is_orig;
|
||||
}
|
||||
|
||||
event modbus_exception(c: connection, headers: ModbusHeaders, code: count)
|
||||
{
|
||||
print "modbus_exception", c, headers, code;
|
||||
print "modbus_exception", c$id, headers, code;
|
||||
}
|
||||
|
||||
event modbus_read_coils_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
|
||||
{
|
||||
print "modbus_read_coils_request", c, headers, start_address, quantity;
|
||||
print "modbus_read_coils_request", c$id, headers, start_address, quantity;
|
||||
}
|
||||
|
||||
event modbus_read_coils_response(c: connection, headers: ModbusHeaders, coils: ModbusCoils)
|
||||
{
|
||||
print "modbus_read_coils_response", c, headers, coils;
|
||||
print "modbus_read_coils_response", c$id, headers, coils;
|
||||
}
|
||||
|
||||
event modbus_read_discrete_inputs_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
|
||||
{
|
||||
print "modbus_read_discrete_inputs_request", c, headers, start_address, quantity;
|
||||
print "modbus_read_discrete_inputs_request", c$id, headers, start_address, quantity;
|
||||
}
|
||||
|
||||
event modbus_read_discrete_inputs_response(c: connection, headers: ModbusHeaders, coils: ModbusCoils)
|
||||
{
|
||||
print "modbus_read_discrete_inputs_response", c, headers, coils;
|
||||
print "modbus_read_discrete_inputs_response", c$id, headers, coils;
|
||||
}
|
||||
|
||||
event modbus_read_holding_registers_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
|
||||
{
|
||||
print "modbus_read_holding_registers_request", c, headers, start_address, quantity;
|
||||
print "modbus_read_holding_registers_request", c$id, headers, start_address, quantity;
|
||||
}
|
||||
|
||||
event modbus_read_holding_registers_response(c: connection, headers: ModbusHeaders, registers: ModbusRegisters)
|
||||
{
|
||||
print "modbus_read_holding_registers_response", c, headers, registers;
|
||||
print "modbus_read_holding_registers_response", c$id, headers, registers;
|
||||
}
|
||||
|
||||
event modbus_read_input_registers_request(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
|
||||
{
|
||||
print "modbus_read_input_registers_request", c, headers, start_address, quantity;
|
||||
print "modbus_read_input_registers_request", c$id, headers, start_address, quantity;
|
||||
}
|
||||
|
||||
event modbus_read_input_registers_response(c: connection, headers: ModbusHeaders, registers: ModbusRegisters)
|
||||
{
|
||||
print "modbus_read_input_registers_response", c, headers, registers;
|
||||
print "modbus_read_input_registers_response", c$id, headers, registers;
|
||||
}
|
||||
|
||||
event modbus_write_single_coil_request(c: connection, headers: ModbusHeaders, address: count, value: bool)
|
||||
{
|
||||
print "modbus_write_single_coil_request", c, headers, address, value;
|
||||
print "modbus_write_single_coil_request", c$id, headers, address, value;
|
||||
}
|
||||
|
||||
event modbus_write_single_coil_response(c: connection, headers: ModbusHeaders, address: count, value: bool)
|
||||
{
|
||||
print "modbus_write_single_coil_response", c, headers, address, value;
|
||||
print "modbus_write_single_coil_response", c$id, headers, address, value;
|
||||
}
|
||||
|
||||
event modbus_write_single_register_request(c: connection, headers: ModbusHeaders, address: count, value: count)
|
||||
{
|
||||
print "modbus_write_single_register_request", c, headers, address, value;
|
||||
print "modbus_write_single_register_request", c$id, headers, address, value;
|
||||
}
|
||||
|
||||
event modbus_write_single_register_response(c: connection, headers: ModbusHeaders, address: count, value: count)
|
||||
{
|
||||
print "modbus_write_single_register_response", c, headers, address, value;
|
||||
print "modbus_write_single_register_response", c$id, headers, address, value;
|
||||
}
|
||||
|
||||
event modbus_write_multiple_coils_request(c: connection, headers: ModbusHeaders, start_address: count, coils: ModbusCoils)
|
||||
{
|
||||
print "modbus_write_multiple_coils_request", c, headers, start_address, coils;
|
||||
print "modbus_write_multiple_coils_request", c$id, headers, start_address, coils;
|
||||
}
|
||||
|
||||
event modbus_write_multiple_coils_response(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
|
||||
{
|
||||
print "modbus_write_multiple_coils_response", c, headers, start_address, quantity;
|
||||
print "modbus_write_multiple_coils_response", c$id, headers, start_address, quantity;
|
||||
}
|
||||
|
||||
event modbus_write_multiple_registers_request(c: connection, headers: ModbusHeaders, start_address: count, registers: ModbusRegisters)
|
||||
{
|
||||
print "modbus_write_multiple_registers_request", c, headers, start_address, registers;
|
||||
print "modbus_write_multiple_registers_request", c$id, headers, start_address, registers;
|
||||
}
|
||||
|
||||
event modbus_write_multiple_registers_response(c: connection, headers: ModbusHeaders, start_address: count, quantity: count)
|
||||
{
|
||||
print "modbus_write_multiple_registers_response", c, headers, start_address, quantity;
|
||||
print "modbus_write_multiple_registers_response", c$id, headers, start_address, quantity;
|
||||
}
|
||||
|
||||
event modbus_read_file_record_request(c: connection, headers: ModbusHeaders)
|
||||
{
|
||||
print "modbus_read_file_record_request", c, headers;
|
||||
print "modbus_read_file_record_request", c$id, headers;
|
||||
}
|
||||
|
||||
event modbus_read_file_record_response(c: connection, headers: ModbusHeaders)
|
||||
{
|
||||
print "modbus_read_file_record_response", c, headers;
|
||||
print "modbus_read_file_record_response", c$id, headers;
|
||||
}
|
||||
|
||||
event modbus_write_file_record_request(c: connection, headers: ModbusHeaders)
|
||||
{
|
||||
print "modbus_write_file_record_request", c, headers;
|
||||
print "modbus_write_file_record_request", c$id, headers;
|
||||
}
|
||||
|
||||
event modbus_write_file_record_response(c: connection, headers: ModbusHeaders)
|
||||
{
|
||||
print "modbus_write_file_record_response", c, headers;
|
||||
print "modbus_write_file_record_response", c$id, headers;
|
||||
}
|
||||
|
||||
event modbus_mask_write_register_request(c: connection, headers: ModbusHeaders, address: count, and_mask: count, or_mask: count)
|
||||
{
|
||||
print "modbus_mask_write_register_request", c, headers, address, and_mask, or_mask;
|
||||
print "modbus_mask_write_register_request", c$id, headers, address, and_mask, or_mask;
|
||||
}
|
||||
|
||||
event modbus_mask_write_register_response(c: connection, headers: ModbusHeaders, address: count, and_mask: count, or_mask: count)
|
||||
{
|
||||
print "modbus_mask_write_register_response", c, headers, address, and_mask, or_mask;
|
||||
print "modbus_mask_write_register_response", c$id, headers, address, and_mask, or_mask;
|
||||
}
|
||||
|
||||
event modbus_read_write_multiple_registers_request(c: connection, headers: ModbusHeaders, read_start_address: count, read_quantity: count, write_start_address: count, write_registers: ModbusRegisters)
|
||||
{
|
||||
print "modbus_read_write_multiple_registers_request", c, headers, read_start_address, read_quantity, write_start_address, write_registers;
|
||||
print "modbus_read_write_multiple_registers_request", c$id, headers, read_start_address, read_quantity, write_start_address, write_registers;
|
||||
}
|
||||
|
||||
event modbus_read_write_multiple_registers_response(c: connection, headers: ModbusHeaders, written_registers: ModbusRegisters)
|
||||
{
|
||||
print "modbus_read_write_multiple_registers_response", c, headers, written_registers;
|
||||
print "modbus_read_write_multiple_registers_response", c$id, headers, written_registers;
|
||||
}
|
||||
|
||||
event modbus_read_fifo_queue_request(c: connection, headers: ModbusHeaders, start_address: count)
|
||||
{
|
||||
print "modbus_read_fifo_queue_request", c, headers, start_address;
|
||||
print "modbus_read_fifo_queue_request", c$id, headers, start_address;
|
||||
}
|
||||
|
||||
event modbus_read_fifo_queue_response(c: connection, headers: ModbusHeaders, fifos: ModbusRegisters)
|
||||
{
|
||||
print "modbus_read_fifo_queue_response", c, headers, fifos;
|
||||
print "modbus_read_fifo_queue_response", c$id, headers, fifos;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ BEGIN { FS="\t"; OFS="\t"; type_col = -1; desc_col = -1 }
|
|||
/^#fields/ {
|
||||
for ( i = 2; i < NF; ++i )
|
||||
{
|
||||
if ( $i == "mime_type" )
|
||||
if ( $i == "mime_type" || $i == "file_type" )
|
||||
type_col = i-1;
|
||||
if ( $i == "mime_desc" )
|
||||
desc_col = i-1;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue