Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-08 17:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

Merge remote-tracking branch 'origin/topic/jsiwek/faf-experimental' into topic/seth/file-analysis-exe-analyzer

Browse source 
Conflicts:
	src/file_analysis.bif
This commit is contained in:
Seth Hall 2013-04-10 22:59:11 -04:00
commit e0df278de6
74 changed files with 1135 additions and 1544 deletions
Download patch file Download diff file Expand all files Collapse all files

232
scripts/base/frameworks/file-analysis/main.bro
View file

@ -12,49 +12,40 @@ export {
LOG
};
## The default buffer size used for storing the beginning of files.
const default_bof_buffer_size: count = 1024 &redef;
## The default amount of time file analysis will wait for new file data
## before giving up.
const default_timeout_interval: interval = 2 mins &redef;
## A structure which represents a desired file analysis action to take.
type ActionArgs: record {
## The type of action.
act: Action;
## The local filename to which to write an extracted file. Must be
## set when *act* is :bro:see:`FileAnalysis::ACTION_EXTRACT`.
extract_filename: string &optional;
};
## A structure which contains the results of certain file analysis actions.
type ActionResults: record {
## An MD5 digest of the file contents.
md5: string &optional;
## An SHA1 digest of the file contents.
sha1: string &optional;
## An SHA256 digest of the file contents.
sha256: string &optional;
};
## An event which will be generated for all new file contents,
## chunk-wise.
chunk_event: event(f: fa_file, data: string, off: count) &optional;
## An event which will be generated for all new file contents,
## stream-wise.
stream_event: event(f: fa_file, data: string) &optional;
} &redef;
## Contains all metadata related to the analysis of a given file.
## For the most part, fields here are derived from ones of the same name
## in :bro:see:`fa_file`.
type Info: record {
## An identifier associated with a single file.
file_id: string &log;
id: string &log;
## Identifier associated with a container file from which this one was
## extracted as part of the file analysis.
parent_file_id: string &log &optional;
parent_id: string &log &optional;
## An identification of the source of the file data. E.g. it may be
## a network protocol over which it was transferred, or a local file
## path which was read, or some other input source.
source: string &log &optional;
## The set of connections over which the file was transferred.
conns: table[conn_id] of connection &optional;
## The time at which the last activity for the file was seen.
last_active: time &log;
@ -75,15 +66,11 @@ export {
## The amount of time between receiving new data for this file that
## the analysis engine will wait before giving up on it.
timeout_interval: interval &log &default=default_timeout_interval;
timeout_interval: interval &log &optional;
## The number of bytes at the beginning of a file to save for later
## inspection in *bof_buffer* field.
bof_buffer_size: count &log &default=default_bof_buffer_size;
## The content of the beginning of a file up to *bof_buffer_size* bytes.
## This is also the buffer that's used for file/mime type detection.
bof_buffer: string &optional;
bof_buffer_size: count &log &optional;
## A file type provided by libmagic against the *bof_buffer*, or
## in the cases where no buffering of the beginning of file occurs,
@ -95,25 +82,17 @@ export {
## an initial guess of the mime type based on the first data seen.
mime_type: string &log &optional;
## Actions that have been added to the analysis of this file.
## Only meant for inspection by user scripts, not direct modification.
actions: table[ActionArgs] of ActionResults;
} &redef;
## Fields that are derived from existing ones, and are set just in time
## for logging purposes.
redef record FileAnalysis::Info += {
## Whether the file analysis timed out at least once for the file.
timedout: bool &log &default=F;
## Connection UIDS over which the file was transferred.
conn_uids: set[string] &log &optional;
conn_uids: set[string] &log;
## A set of action types taken during the file analysis.
actions_taken: set[Action] &log &optional;
actions_taken: set[Action] &log;
## Local filenames of file extraction actions.
extracted_files: set[string] &log &optional;
extracted_files: set[string] &log;
## An MD5 digest of the file contents.
md5: string &log &optional;
@ -123,25 +102,7 @@ export {
## A SHA256 digest of the file contents.
sha256: string &log &optional;
};
## Redefined here just so the *info* parameters of the events have the
## right type information.
redef record ActionArgs += {
## An event which will be generated for all new file contents,
## chunk-wise.
chunk_event: event(info: Info, data: string, off: count) &optional;
## An event which will be generated for all new file contents,
## stream-wise.
stream_event: event(info: Info, data: string) &optional;
};
## Evaluated every time a significant event occurs during the course of
## file analysis. Fields of the *info* argument may be modified or
## other actions may be added or removed inside the body of any handlers
## of this hook.
global policy: hook(trig: Trigger, info: Info);
} &redef;
## A table that can be used to disable file analysis completely for
## any files transferred over given network protocol analyzers.
@ -153,57 +114,52 @@ export {
## The salt concatenated to unique file handle strings generated by
## :bro:see:`get_file_handle` before hashing them in to a file id
## (the *file_id* field of :bro:see:`FileAnalysis::Info`).
## (the *id* field of :bro:see:`fa_file`).
## Provided to help mitigate the possiblility of manipulating parts of
## network connections that factor in to the file handle in order to
## generate two handles that would hash to the same file id.
const salt = "I recommend changing this." &redef;
## Postpones the timeout of file analysis for a given file.
## When used within a :bro:see:`FileAnalysis::policy` handler for
## :bro:see:`FileAnalysis::TRIGGER_TIMEOUT`, the analysis will delay
## timing out for the period of time indicated by the *timeout_interval*
## field of :bro:see:`FileAnalysis::Info`.
## When used within a :bro:see:`file_timeout` handler for, the analysis
## the analysis will delay timing out for the period of time indicated by
## the *timeout_interval* field of :bro:see:`fa_file`.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## Returns: true if the timeout will be postponed, or false if analysis
## for the *file_id* isn't currently active.
global postpone_timeout: function(file_id: string): bool;
## for the *id* isn't currently active.
global postpone_timeout: function(f: fa_file): bool;
## Adds an action to the analysis of a given file.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## args: the action type to add along with any arguments it takes.
##
## Returns: true if the action will be added, or false if analysis
## for the *file_id* isn't currently active or the *args*
## for the *id* isn't currently active or the *args*
## were invalid for the action type.
global add_action: function(file_id: string, args: ActionArgs): bool;
global add_action: function(f: fa_file, args: ActionArgs): bool;
## Removes an action from the analysis of a given file.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## args: the action (type and args) to remove.
##
## Returns: true if the action will be removed, or false if analysis
## for the *file_id* isn't currently active.
global remove_action: function(file_id: string, args: ActionArgs): bool;
## for the *id* isn't currently active.
global remove_action: function(f: fa_file, args: ActionArgs): bool;
## Stops/ignores any further analysis of a given file.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## Returns: true if analysis for the given file will be ignored for the
## rest of it's contents, or false if analysis for the *file_id*
## rest of it's contents, or false if analysis for the *id*
## isn't currently active.
global stop: function(file_id: string): bool;
global stop: function(f: fa_file): bool;
## Sends a sequential stream of data in for file analysis.
## Meant for use when providing external file analysis input (e.g.
@ -258,24 +214,61 @@ export {
global eof: function(source: string);
}
function postpone_timeout(file_id: string): bool
redef record fa_file += {
info: Info &optional;
};
function set_info(f: fa_file)
{
return __postpone_timeout(file_id);
if ( ! f?$info )
{
local tmp: Info;
f$info = tmp;
}
function add_action(file_id: string, args: ActionArgs): bool
{
return __add_action(file_id, args);
f$info$id = f$id;
if ( f?$parent_id ) f$info$parent_id = f$parent_id;
if ( f?$source ) f$info$source = f$source;
f$info$last_active = f$last_active;
f$info$seen_bytes = f$seen_bytes;
if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes;
f$info$missing_bytes = f$missing_bytes;
f$info$overflow_bytes = f$overflow_bytes;
f$info$timeout_interval = f$timeout_interval;
f$info$bof_buffer_size = f$bof_buffer_size;
if ( f?$file_type ) f$info$file_type = f$file_type;
if ( f?$mime_type ) f$info$mime_type = f$mime_type;
if ( f?$conns )
for ( cid in f$conns )
add f$info$conn_uids[f$conns[cid]$uid];
}
function remove_action(file_id: string, args: ActionArgs): bool
function postpone_timeout(f: fa_file): bool
{
return __remove_action(file_id, args);
return __postpone_timeout(f$id);
}
function stop(file_id: string): bool
function add_action(f: fa_file, args: ActionArgs): bool
{
return __stop(file_id);
if ( ! __add_action(f$id, args) ) return F;
set_info(f);
add f$info$actions_taken[args$act];
if ( args$act == FileAnalysis::ACTION_EXTRACT )
add f$info$extracted_files[args$extract_filename];
return T;
}
function remove_action(f: fa_file, args: ActionArgs): bool
{
return __remove_action(f$id, args);
}
function stop(f: fa_file): bool
{
return __stop(f$id);
}
function data_stream(source: string, data: string)
@ -309,53 +302,34 @@ event bro_init() &priority=5
[$columns=Info, $ev=log_file_analysis]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_timeout(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
info$timedout = T;
set_info(f);
f$info$timedout = T;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-5
event file_hash(f: fa_file, kind: string, hash: string) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_EOF &&
trig != FileAnalysis::TRIGGER_DONE ) return;
set_info(f);
switch ( kind ) {
case "md5":
f$info$md5 = hash;
break;
case "sha1":
f$info$sha1 = hash;
break;
case "sha256":
f$info$sha256 = hash;
break;
}
}
info$conn_uids = set();
if ( info?$conns )
for ( cid in info$conns )
add info$conn_uids[info$conns[cid]$uid];
info$actions_taken = set();
info$extracted_files = set();
for ( act in info$actions )
event file_state_remove(f: fa_file) &priority=5
{
add info$actions_taken[act$act];
local result: FileAnalysis::ActionResults = info$actions[act];
switch ( act$act ) {
case FileAnalysis::ACTION_EXTRACT:
add info$extracted_files[act$extract_filename];
break;
case FileAnalysis::ACTION_MD5:
if ( result?$md5 )
info$md5 = result$md5;
break;
case FileAnalysis::ACTION_SHA1:
if ( result?$sha1 )
info$sha1 = result$sha1;
break;
case FileAnalysis::ACTION_SHA256:
if ( result?$sha256 )
info$sha256 = result$sha256;
break;
case FileAnalysis::ACTION_DATA_EVENT:
# no direct result
break;
}
set_info(f);
}
Log::write(FileAnalysis::LOG, info);
event file_state_remove(f: fa_file) &priority=-5
{
Log::write(FileAnalysis::LOG, f$info);
}

64
scripts/base/init-bare.bro
View file

@ -316,6 +316,70 @@ type connection: record {
tunnel: EncapsulatingConnVector &optional;
};
const default_file_timeout_interval: interval = 2 mins &redef;
const default_file_bof_buffer_size: count = 1024 &redef;
## A file that Bro is analyzing. This is Bro's type for describing the basic
## internal metadata collected about a "file", which is essentially just a
## byte stream that is e.g. pulled from a network connection or possibly
## some other input source.
type fa_file: record {
## An identifier associated with a single file.
id: string;
## Identifier associated with a container file from which this one was
## extracted as part of the file analysis.
parent_id: string &optional;
## An identification of the source of the file data. E.g. it may be
## a network protocol over which it was transferred, or a local file
## path which was read, or some other input source.
source: string &optional;
## The set of connections over which the file was transferred.
conns: table[conn_id] of connection &optional;
## The time at which the last activity for the file was seen.
last_active: time;
## Number of bytes provided to the file analysis engine for the file.
seen_bytes: count &default=0;
## Total number of bytes that are supposed to comprise the full file.
total_bytes: count &optional;
## The number of bytes in the file stream that were completely missed
## during the process of analysis e.g. due to dropped packets.
missing_bytes: count &default=0;
## The number of not all-in-sequence bytes in the file stream that
## were delivered to file actions/analyzers due to reassembly buffer
## overflow.
overflow_bytes: count &default=0;
## The amount of time between receiving new data for this file that
## the analysis engine will wait before giving up on it.
timeout_interval: interval &default=default_file_timeout_interval;
## The number of bytes at the beginning of a file to save for later
## inspection in *bof_buffer* field.
bof_buffer_size: count &default=default_file_bof_buffer_size;
## The content of the beginning of a file up to *bof_buffer_size* bytes.
## This is also the buffer that's used for file/mime type detection.
bof_buffer: string &optional;
## A file type provided by libmagic against the *bof_buffer*, or
## in the cases where no buffering of the beginning of file occurs,
## an initial guess of the file type based on the first data seen.
file_type: string &optional;
## A mime type provided by libmagic against the *bof_buffer*, or
## in the cases where no buffering of the beginning of file occurs,
## an initial guess of the mime type based on the first data seen.
mime_type: string &optional;
} &redef;
## Fields of a SYN packet.
##
## .. bro:see:: connection_SYN_packet

25
scripts/base/protocols/ftp/file-analysis.bro
View file

@ -5,22 +5,21 @@
module FTP;
export {
## Determines whether the default :bro:see:`get_file_handle` handler
## is used to return file handles to the file analysis framework.
## Redefine to true in order to provide a custom handler which overrides
## the default for FTP.
const disable_default_file_handle_provider: bool = F &redef;
## Default file handle provider for FTP.
global get_file_handle: function(c: connection, is_orig: bool): string;
}
function get_handle_string(c: connection): string
{
return fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time, id_string(c$id));
}
function get_file_handle(c: connection, is_orig: bool): string
{
if ( [c$id$resp_h, c$id$resp_p] !in ftp_data_expected ) return "";
local info: FTP::Info = ftp_data_expected[c$id$resp_h, c$id$resp_p];
local rval = fmt("%s %s %s", ANALYZER_FTP_DATA, c$start_time,
id_string(c$id));
if ( info$passive )
# FTP client initiates data channel.
if ( is_orig )
@ -28,23 +27,21 @@ export {
return "";
else
# Do care about FTP server data.
return rval;
return get_handle_string(c);
else
# FTP server initiates dta channel.
if ( is_orig )
# Do care about FTP server data.
return rval;
return get_handle_string(c);
else
# Don't care about FTP client data.
return "";
}
}
module GLOBAL;
event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
{
if ( tag != ANALYZER_FTP_DATA ) return;
if ( FTP::disable_default_file_handle_provider ) return;
return_file_handle(FTP::get_file_handle(c, is_orig));
set_file_handle(FTP::get_file_handle(c, is_orig));
}

91
scripts/base/protocols/ftp/file-extract.bro
View file

@ -24,21 +24,30 @@ redef record Info += {
extract_file: bool &default=F;
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
function get_extraction_name(f: fa_file): string
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "FTP_DATA" ) return;
if ( ! info?$conns ) return;
local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
++extract_count;
return r;
}
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
event file_new(f: fa_file) &priority=5
{
local c: connection = info$conns[cid];
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;
if ( f?$mime_type && extract_file_types in f$mime_type )
{
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=get_extraction_name(f)]);
return;
}
if ( ! f?$conns ) return;
for ( cid in f$conns )
{
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in ftp_data_expected ) next;
@ -46,59 +55,31 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
if ( ! s$extract_file ) next;
if ( ! extracting )
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=get_extraction_name(f)]);
return;
}
}
event file_state_remove(f: fa_file) &priority=4
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
}
}
}
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;
if ( ! f?$info ) return;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "FTP_DATA" ) return;
if ( extract_file_types !in info$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_EOF &&
trig != FileAnalysis::TRIGGER_DONE ) return;
if ( ! info?$source ) return;
if ( info$source != "FTP_DATA" ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT )
for ( filename in f$info$extracted_files )
{
local s: FTP::Info;
s$ts = network_time();
s$tags = set();
s$user = "<ftp-data>";
s$extraction_file = act$extract_filename;
s$extraction_file = filename;
if ( info?$conns )
for ( cid in info$conns )
if ( f?$conns )
for ( cid in f$conns )
{
s$uid = info$conns[cid]$uid;
s$uid = f$conns[cid]$uid;
s$id = cid;
break;
}
Log::write(FTP::LOG, s);

13
scripts/base/protocols/http/file-analysis.bro
View file

@ -6,13 +6,10 @@
module HTTP;
export {
## Determines whether the default :bro:see:`get_file_handle` handler
## is used to return file handles to the file analysis framework.
## Redefine to true in order to provide a custom handler which overrides
## the default HTTP.
const disable_default_file_handle_provider: bool = F &redef;
## Default file handle provider for HTTP.
global get_file_handle: function(c: connection, is_orig: bool): string;
}
function get_file_handle(c: connection, is_orig: bool): string
{
if ( ! c?$http ) return "";
@ -24,13 +21,11 @@ export {
return fmt("%s %s %s %s %s", ANALYZER_HTTP, c$start_time, is_orig,
c$http$trans_depth, id_string(c$id));
}
}
module GLOBAL;
event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
{
if ( tag != ANALYZER_HTTP ) return;
if ( HTTP::disable_default_file_handle_provider ) return;
return_file_handle(HTTP::get_file_handle(c, is_orig));
set_file_handle(HTTP::get_file_handle(c, is_orig));
}

77
scripts/base/protocols/http/file-extract.bro
View file

@ -18,74 +18,67 @@ export {
extraction_file: string &log &optional;
## Indicates if the response body is to be extracted or not. Must be
## set before or by the first :bro:enum:`FileAnalysis::TRIGGER_NEW`
## for the file content.
## set before or by the first :bro:see:`file_new` for the file content.
extract_file: bool &default=F;
};
}
global extract_count: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
function get_extraction_name(f: fa_file): string
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( extract_file_types !in info$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
return r;
}
event file_new(f: fa_file) &priority=5
{
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
local fname: string;
local c: connection;
if ( f?$mime_type && extract_file_types in f$mime_type )
{
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
if ( ! info?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
c = f$conns[cid];
if ( ! c?$http ) next;
c$http$extraction_file = fname;
}
return;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
c = f$conns[cid];
if ( ! c?$http ) next;
if ( c$http$extract_file )
{
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
if ( ! c$http$extract_file ) next;
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
break;
}
if ( extracting )
for ( cid in f$conns )
{
c = f$conns[cid];
if ( ! c?$http ) next;
c$http$extraction_file = fname;
}
}
}

69
scripts/base/protocols/http/file-hash.bro
View file

@ -23,67 +23,46 @@ export {
&redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( generate_md5 in info$mime_type )
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
else if ( info?$conns )
if ( f?$mime_type && generate_md5 in f$mime_type )
{
for ( cid in info$conns )
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
return;
}
if ( ! f?$conns ) return;
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
if ( c$http$calc_md5 )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_MD5]);
if ( ! c$http$calc_md5 ) next;
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
return;
}
}
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_state_remove(f: fa_file) &priority=4
{
if ( trig != FileAnalysis::TRIGGER_DONE &&
trig != FileAnalysis::TRIGGER_EOF ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
if ( ! f?$info ) return;
if ( ! f$info?$md5 ) return;
local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5];
if ( act !in info$actions ) return;
local result = info$actions[act];
if ( ! result?$md5 ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
c$http$md5 = result$md5;
c$http$md5 = f$info$md5;
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_GAP ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
FileAnalysis::remove_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
}

42
scripts/base/protocols/http/file-ident.bro
View file

@ -34,24 +34,22 @@ export {
const ignored_incorrect_file_type_urls = /^$/ &redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
c$http$mime_type = info$mime_type;
c$http$mime_type = f$mime_type;
local mime_str: string = split1(info$mime_type, /;/)[1];
local mime_str: string = split1(f$mime_type, /;/)[1];
if ( mime_str !in mime_types_extensions ) next;
if ( ! c$http?$uri ) next;
@ -68,26 +66,22 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_over_new_connection(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW_CONN ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$conns ) return;
# Spread the mime around (e.g. for partial content, TRIGGER_TYPE only
# Spread the mime around (e.g. for partial content, file_type event only
# happens once for the first connection, but if there's subsequent
# connections to transfer the same file, they'll be lacking the mime_type
# field if we don't do this).
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
c$http$mime_type = info$mime_type;
c$http$mime_type = f$mime_type;
}
}

148
scripts/base/protocols/irc/dcc-send.bro
View file

@ -41,64 +41,29 @@ global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
global extract_count: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
function set_dcc_mime(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "IRC_DATA" ) return;
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
if ( ! s$extract_file ) next;
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
}
s$extraction_file = fname;
s$dcc_mime_type = f$mime_type;
}
}
function set_dcc_mime(info: FileAnalysis::Info)
function set_dcc_extraction_file(f: fa_file, filename: string)
{
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
s$dcc_mime_type = info$mime_type;
}
}
function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
{
if ( ! info?$conns ) return;
for ( cid in info$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
@ -108,13 +73,67 @@ function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
}
}
function log_dcc(info: FileAnalysis::Info)
function get_extraction_name(f: fa_file): string
{
if ( ! info?$conns ) return;
local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
++extract_count;
return r;
}
for ( cid in info$conns )
# this handler sets the IRC::Info mime type
event file_new(f: fa_file) &priority=5
{
local c: connection = info$conns[cid];
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
if ( ! f?$mime_type ) return;
set_dcc_mime(f);
}
# this handler check if file extraction is desired
event file_new(f: fa_file) &priority=5
{
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
local fname: string;
if ( f?$mime_type && extract_file_types in f$mime_type )
{
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
set_dcc_extraction_file(f, fname);
return;
}
if ( ! f?$conns ) return;
for ( cid in f$conns )
{
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
if ( ! s$extract_file ) next;
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
s$extraction_file = fname;
return;
}
}
function log_dcc(f: fa_file)
{
if ( ! f?$conns ) return;
for ( cid in f$conns )
{
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
@ -137,37 +156,12 @@ function log_dcc(info: FileAnalysis::Info)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "IRC_DATA" ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
set_dcc_mime(info);
if ( extract_file_types !in info$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
set_dcc_extraction_file(info, fname);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$source ) return;
if ( info$source != "IRC_DATA" ) return;
log_dcc(info);
log_dcc(f);
}
event irc_dcc_message(c: connection, is_orig: bool,

16
scripts/base/protocols/irc/file-analysis.bro
View file

@ -5,19 +5,14 @@
module IRC;
export {
## Determines whether the default :bro:see:`get_file_handle` handler
## is used to return file handles to the file analysis framework.
## Redefine to true in order to provide a custom handler which overrides
## the default for IRC.
const disable_default_file_handle_provider: bool = F &redef;
## Default file handle provider for IRC.
global get_file_handle: function(c: connection, is_orig: bool): string;
}
function get_file_handle(c: connection, is_orig: bool): string
{
if ( is_orig ) return "";
return fmt("%s %s %s", ANALYZER_IRC_DATA, c$start_time,
id_string(c$id));
}
return fmt("%s %s %s", ANALYZER_IRC_DATA, c$start_time, id_string(c$id));
}
module GLOBAL;
@ -25,6 +20,5 @@ module GLOBAL;
event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
{
if ( tag != ANALYZER_IRC_DATA ) return;
if ( IRC::disable_default_file_handle_provider ) return;
return_file_handle(IRC::get_file_handle(c, is_orig));
set_file_handle(IRC::get_file_handle(c, is_orig));
}

32
scripts/base/protocols/smtp/entities-excerpt.bro
View file

@ -12,38 +12,26 @@ export {
};
## This is the default value for how much of the entity body should be
## included for all MIME entities.
## included for all MIME entities. The lesser of this value and
## :bro:see:`default_file_bof_buffer_size` will be used.
const default_entity_excerpt_len = 0 &redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$bof_buffer ) return;
if ( ! f?$conns ) return;
if ( default_entity_excerpt_len > info$bof_buffer_size )
info$bof_buffer_size = default_entity_excerpt_len;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
for ( cid in f$conns )
{
if ( trig != FileAnalysis::TRIGGER_BOF_BUFFER ) return;
if ( ! info?$bof_buffer ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
for ( cid in info$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( default_entity_excerpt_len > 0 )
c$smtp$current_entity$excerpt =
info$bof_buffer[0:default_entity_excerpt_len];
f$bof_buffer[0:default_entity_excerpt_len];
}
}

133
scripts/base/protocols/smtp/entities.bro
View file

@ -88,6 +88,13 @@ function set_session(c: connection, new_entity: bool)
}
}
function get_extraction_name(f: fa_file): string
{
local r = fmt("%s-%s-%d.dat", extraction_prefix, f$id, extract_count);
++extract_count;
return r;
}
event mime_begin_entity(c: connection) &priority=10
{
if ( ! c?$smtp ) return;
@ -95,21 +102,18 @@ event mime_begin_entity(c: connection) &priority=10
set_session(c, T);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
local fname: string;
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
@ -118,8 +122,8 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
fname = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
@ -129,116 +133,79 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
if ( c$smtp$current_entity$calc_md5 )
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
}
}
function check_extract_by_type(info: FileAnalysis::Info)
function check_extract_by_type(f: fa_file)
{
if ( extract_file_types !in info$mime_type ) return;
if ( extract_file_types !in f$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
local fname: string = get_extraction_name(f);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
c$smtp$current_entity$extraction_file = fname;
}
}
function check_md5_by_type(info: FileAnalysis::Info)
function check_md5_by_type(f: fa_file)
{
if ( never_calc_md5 ) return;
if ( generate_md5 !in info$mime_type ) return;
if ( generate_md5 !in f$mime_type ) return;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$mime_type ) return;
if ( info?$conns )
for ( cid in info$conns )
if ( f?$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
c$smtp$current_entity$mime_type = info$mime_type;
c$smtp$current_entity$mime_type = f$mime_type;
}
check_extract_by_type(info);
check_md5_by_type(info);
check_extract_by_type(f);
check_md5_by_type(f);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_state_remove(f: fa_file) &priority=4
{
if ( trig != FileAnalysis::TRIGGER_GAP ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
# Only log if there was some content.
if ( f$seen_bytes == 0 ) next;
FileAnalysis::remove_action(info$file_id,
[$act=FileAnalysis::ACTION_MD5]);
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_EOF &&
trig != FileAnalysis::TRIGGER_DONE ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
for ( cid in info$conns )
{
local c: connection = info$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
# Only log is there was some content.
if ( info$seen_bytes == 0 ) next;
local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5];
if ( act in info$actions )
{
local result = info$actions[act];
if ( result?$md5 )
c$smtp$current_entity$md5 = result$md5;
}
c$smtp$current_entity$content_len = info$seen_bytes;
if ( f?$info && f$info?$md5 )
c$smtp$current_entity$md5 = f$info$md5;
c$smtp$current_entity$content_len = f$seen_bytes;
Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity);
delete c$smtp$current_entity;
return;

14
scripts/base/protocols/smtp/file-analysis.bro
View file

@ -6,27 +6,21 @@
module SMTP;
export {
## Determines whether the default :bro:see:`get_file_handle` handler
## is used to return file handles to the file analysis framework.
## Redefine to true in order to provide a custom handler which overrides
## the default for SMTP.
const disable_default_file_handle_provider: bool = F &redef;
## Default file handle provider for SMTP.
global get_file_handle: function(c: connection, is_orig: bool): string;
}
function get_file_handle(c: connection, is_orig: bool): string
{
if ( ! c?$smtp ) return "";
return fmt("%s %s %s %s", ANALYZER_SMTP, c$start_time,
c$smtp$trans_depth, c$smtp_state$mime_level);
}
}
module GLOBAL;
event get_file_handle(tag: AnalyzerTag, c: connection, is_orig: bool)
{
if ( tag != ANALYZER_SMTP ) return;
if ( SMTP::disable_default_file_handle_provider ) return;
return_file_handle(SMTP::get_file_handle(c, is_orig));
set_file_handle(SMTP::get_file_handle(c, is_orig));
}

19
scripts/policy/frameworks/intel/smtp-url-extraction.bro
View file

@ -3,13 +3,13 @@
@load base/utils/urls
@load ./where-locations
event intel_mime_data(info: FileAnalysis::Info, data: string)
event intel_mime_data(f: fa_file, data: string)
{
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
local urls = find_all_urls_without_scheme(data);
for ( url in urls )
{
@ -21,14 +21,11 @@ event intel_mime_data(info: FileAnalysis::Info, data: string)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_DATA_EVENT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$stream_event=intel_mime_data]);
}

5
src/CMakeLists.txt
View file

@ -454,9 +454,8 @@ set(bro_SRCS
input/readers/Binary.cc
file_analysis/Manager.cc
file_analysis/Info.cc
file_analysis/InfoTimer.cc
file_analysis/PendingFile.cc
file_analysis/File.cc
file_analysis/FileTimer.cc
file_analysis/FileID.h
file_analysis/Action.h
file_analysis/ActionSet.cc

6
src/Event.cc
View file

@ -6,7 +6,6 @@
#include "Func.h"
#include "NetVar.h"
#include "Trigger.h"
#include "file_analysis/Manager.h"
EventMgr mgr;
@ -111,6 +110,9 @@ void EventMgr::Dispatch()
void EventMgr::Drain()
{
if ( event_queue_flush_point )
QueueEvent(event_queue_flush_point, new val_list());
SegmentProfiler(segment_logger, "draining-events");
draining = true;
@ -125,8 +127,6 @@ void EventMgr::Drain()
// processing, we ensure that it's done at a regular basis by checking
// them here.
Trigger::EvaluatePending();
file_mgr->EventDrainDone();
}
void EventMgr::Describe(ODesc* d) const

2
src/NetVar.cc
View file

@ -9,6 +9,7 @@ RecordType* conn_id;
RecordType* endpoint;
RecordType* endpoint_stats;
RecordType* connection_type;
RecordType* fa_file_type;
RecordType* icmp_conn;
RecordType* icmp_context;
RecordType* SYN_packet;
@ -314,6 +315,7 @@ void init_net_var()
endpoint = internal_type("endpoint")->AsRecordType();
endpoint_stats = internal_type("endpoint_stats")->AsRecordType();
connection_type = internal_type("connection")->AsRecordType();
fa_file_type = internal_type("fa_file")->AsRecordType();
icmp_conn = internal_type("icmp_conn")->AsRecordType();
icmp_context = internal_type("icmp_context")->AsRecordType();
signature_state = internal_type("signature_state")->AsRecordType();

1
src/NetVar.h
View file

@ -12,6 +12,7 @@ extern RecordType* conn_id;
extern RecordType* endpoint;
extern RecordType* endpoint_stats;
extern RecordType* connection_type;
extern RecordType* fa_file_type;
extern RecordType* icmp_conn;
extern RecordType* icmp_context;
extern RecordType* signature_state;

16
src/bro.bif
View file

@ -5538,22 +5538,6 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
return new Val(1, TYPE_BOOL);
%}
## For use within a :bro:see:`get_file_handle` handler to return a unique
## identifier to associate with some buffered input to the file analysis
## framework. The buffered data will then immediately be allowed to pass
## pass through the file analysis framework and execute any policy hooks
## that are available. If an empty string is returned, that signifies that
## the buffered data will be discarded with no further action taken on it.
##
## handle: A string that uniquely identifies a file.
##
## .. bro:see:: get_file_handle FileAnalysis::policy
function return_file_handle%(handle: string%): any
%{
file_mgr->ReceiveHandle(handle->CheckString());
return 0;
%}
# ===========================================================================
#
# Deprecated Functions

35
src/event.bif
View file

@ -6981,12 +6981,15 @@ event reporter_error%(t: time, msg: string, location: string%) &error_handler;
## recursively for each ``@load``.
event bro_script_loaded%(path: string, level: count%);
## Marks a point in the event stream at which the event queue started flushing.
event event_queue_flush_point%(%);
## This event is handled to provide feedback to the file analysis framework
## about how to identify the logical "file" to which some data/input
## belongs. All incoming data to the framework is buffered, and depends
## on a handler for this event to return a string value that uniquely
## identifies a file. Among all handlers of this event, exactly one must
## call :bro:see:`return_file_handle`.
## identifies a file. Among all handlers of this event, the last one to
## call :bro:see:`set_file_handle` will "win".
##
## tag: The analyzer which is carrying the file data.
##
@ -6994,9 +6997,35 @@ event bro_script_loaded%(path: string, level: count%);
##
## is_orig: The direction the file data is flowing over the connection.
##
## .. bro:see:: return_file_handle
## .. bro:see:: set_file_handle
event get_file_handle%(tag: count, c: connection, is_orig: bool%);
# TODO: document
event file_new%(f: fa_file%);
# TODO: give the new connection
event file_over_new_connection%(f: fa_file%);
event file_timeout%(f: fa_file%);
# TODO: give size of gap
event file_gap%(f: fa_file%);
## This event is generated each time file analysis is ending for a given file.
##
## f: The file.
event file_state_remove%(f: fa_file%);
## This event is generated each time file analysis generates a digest of the
## file contents.
##
## f: The file.
##
## kind: The type of digest algorithm.
##
## hash: The result of the hashing.
##
## .. bro:see:: FileAnalysis::add_action FileAnalysis::ACTION_MD5
## FileAnalysis::ACTION_SHA1 FileAnalysis::ACTION_SHA256
event file_hash%(f: fa_file, kind: string, hash: string%);
## Deprecated. Will be removed.
event stp_create_endp%(c: connection, e: int, is_orig: bool%);

87
src/file_analysis.bif
View file

@ -6,75 +6,6 @@ module FileAnalysis;
#include "file_analysis/Manager.h"
%%}
type Info: record;
type ActionArgs: record;
type ActionResults: record;
## An enumeration of significant things that can occur over the course of
## analyzing files. The :bro:see:`FileAnalysis::policy` hook is called each
## time a trigger occurs.
enum Trigger %{
## Raised when any part of a new file is detected.
TRIGGER_NEW,
## Raised when file is detected being transported over a new network
## connection (other than the first).
TRIGGER_NEW_CONN,
## Raised when file analysis has likely seen a complete file. That
## is when a number of bytes indicated by the *total_bytes* field of
## :bro:see:`FileAnalysis::Info` have been processed.
TRIGGER_DONE,
## Raised when file analysis for a given file is aborted due
## to not seeing any data for it recently. Note that this doesn't
## necessarily mean the full file wasn't seen (e.g. if the
## :bro:see:`FileAnalysis::Info` record indicates the file *total_bytes*
## isn't known). Use :bro:see:`FileAnalysis::postpone_timeout`
## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
## defer the timeout until later.
TRIGGER_TIMEOUT,
## Raised when the beginning of a file is detected.
TRIGGER_BOF,
## Raised when the beginning of a file is available in the *bof_buffer*
## field of :bro:see:`FileAnalysis::Info` and that beginning
## is at least the number of bytes indicated by the *bof_buffer_size* field.
TRIGGER_BOF_BUFFER,
## Raised when an initial guess at the file/mime type of a file is matched.
TRIGGER_TYPE,
## Raised to signal that no more file data is incoming and it couldn't be
## determined whether the full file was actually seen and analyzed.
TRIGGER_EOF,
## Raised when there's a missing chunk of data in the file stream.
TRIGGER_GAP,
%}
## An enumeration of various file analysis actions that can be taken.
enum Action %{
## Extract a file to local filesystem
ACTION_EXTRACT,
## Calculate an MD5 digest of the file's contents.
ACTION_MD5,
## Calculate an SHA1 digest of the file's contents.
ACTION_SHA1,
## Calculate an SHA256 digest of the file's contents.
ACTION_SHA256,
## Deliver the file contents to the script-layer in an event.
ACTION_DATA_EVENT,
ACTION_PE_ANALYZER,
%}
## :bro:see:`FileAnalysis::postpone_timeout`.
function FileAnalysis::__postpone_timeout%(file_id: string%): bool
%{
@ -149,6 +80,24 @@ function FileAnalysis::__eof%(source: string%): any
return 0;
%}
module GLOBAL;
## For use within a :bro:see:`get_file_handle` handler to return a unique
## identifier to associate with some buffered input to the file analysis
## framework. The buffered data will then immediately be allowed to pass
## pass through the file analysis framework and execute any policy hooks
## that are available. If an empty string is returned, that signifies that
## the buffered data will be discarded with no further action taken on it.
##
## handle: A string that uniquely identifies a file.
##
## .. bro:see:: get_file_handle FileAnalysis::policy
function set_file_handle%(handle: string%): any
%{
file_mgr->SetHandle(handle->CheckString());
return 0;
%}
# Define file analysis framework events.
#event FileAnalysis::windows_pe_dosstub%(fi: FileAnalysis::Info, sig: string, checksum: count%);

16
src/file_analysis/Action.h
View file

@ -8,10 +8,10 @@ namespace file_analysis {
typedef BifEnum::FileAnalysis::Action ActionTag;
class Info;
class File;
/**
* Base class for actions that can be attached to a file_analysis::Info object.
* Base class for actions that can be attached to a file_analysis::File object.
*/
class Action {
public:
@ -67,9 +67,9 @@ public:
RecordVal* Args() const { return args; }
/**
* @return the file_analysis::Info object to which the action is attached.
* @return the file_analysis::File object to which the action is attached.
*/
Info* GetInfo() const { return info; }
File* GetFile() const { return file; }
/**
* @return the action tag equivalent of the 'act' field from the ActionArgs
@ -84,17 +84,17 @@ public:
protected:
Action(RecordVal* arg_args, Info* arg_info)
Action(RecordVal* arg_args, File* arg_file)
: tag(Action::ArgsTag(arg_args)), args(arg_args->Ref()->AsRecordVal()),
info(arg_info)
file(arg_file)
{}
ActionTag tag;
RecordVal* args;
Info* info;
File* file;
};
typedef Action* (*ActionInstantiator)(RecordVal* args, Info* info);
typedef Action* (*ActionInstantiator)(RecordVal* args, File* file);
} // namespace file_analysis

24
src/file_analysis/ActionSet.cc
View file

@ -1,5 +1,5 @@
#include "ActionSet.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
#include "Extract.h"
#include "DataEvent.h"
@ -25,7 +25,7 @@ static void action_del_func(void* v)
delete (Action*) v;
}
ActionSet::ActionSet(Info* arg_info) : info(arg_info)
ActionSet::ActionSet(File* arg_file) : file(arg_file)
{
TypeList* t = new TypeList();
t->Append(BifType::Record::FileAnalysis::ActionArgs->Ref());
@ -54,7 +54,7 @@ bool ActionSet::AddAction(RecordVal* args)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d skipped for file id"
" %s: already exists", Action::ArgsTag(args),
info->GetFileID().c_str());
file->GetID().c_str());
delete key;
return true;
}
@ -94,7 +94,7 @@ bool ActionSet::Add::Perform(ActionSet* set)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d skipped for file id"
" %s: already exists", act->Tag(),
act->GetInfo()->GetFileID().c_str());
act->GetFile()->GetID().c_str());
Abort();
return true;
}
@ -116,12 +116,12 @@ bool ActionSet::RemoveAction(ActionTag tag, HashKey* key)
if ( ! act )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove action %d for file id %s",
tag, info->GetFileID().c_str());
tag, file->GetID().c_str());
return false;
}
DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", act->Tag(),
info->GetFileID().c_str());
file->GetID().c_str());
delete act;
return true;
}
@ -151,12 +151,12 @@ HashKey* ActionSet::GetKey(const RecordVal* args) const
Action* ActionSet::InstantiateAction(RecordVal* args) const
{
Action* act = action_factory[Action::ArgsTag(args)](args, info);
Action* act = action_factory[Action::ArgsTag(args)](args, file);
if ( ! act )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d failed for file id",
" %s", Action::ArgsTag(args), info->GetFileID().c_str());
" %s", Action::ArgsTag(args), file->GetID().c_str());
return 0;
}
@ -166,11 +166,9 @@ Action* ActionSet::InstantiateAction(RecordVal* args) const
void ActionSet::InsertAction(Action* act, HashKey* key)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act->Tag(),
info->GetFileID().c_str());
file->GetID().c_str());
action_map.Insert(key, act);
delete key;
info->GetVal()->Lookup(Info::actions_idx)->AsTableVal()->Assign(act->Args(),
new RecordVal(BifType::Record::FileAnalysis::ActionResults));
}
void ActionSet::DrainModifications()
@ -178,7 +176,7 @@ void ActionSet::DrainModifications()
if ( mod_queue.empty() ) return;
DBG_LOG(DBG_FILE_ANALYSIS, "Start flushing action mod queue of file id %s",
info->GetFileID().c_str());
file->GetID().c_str());
do
{
Modification* mod = mod_queue.front();
@ -187,5 +185,5 @@ void ActionSet::DrainModifications()
mod_queue.pop();
} while ( ! mod_queue.empty() );
DBG_LOG(DBG_FILE_ANALYSIS, "End flushing action mod queue of file id %s",
info->GetFileID().c_str());
file->GetID().c_str());
}

6
src/file_analysis/ActionSet.h
View file

@ -10,7 +10,7 @@
namespace file_analysis {
class Info;
class File;
declare(PDict,Action);
/**
@ -21,7 +21,7 @@ declare(PDict,Action);
class ActionSet {
public:
ActionSet(Info* arg_info);
ActionSet(File* arg_file);
~ActionSet();
@ -63,7 +63,7 @@ protected:
void InsertAction(Action* act, HashKey* key);
bool RemoveAction(ActionTag tag, HashKey* key);
Info* info;
File* file;
CompositeHash* action_hash; /**< ActionArgs hashes Action map lookup. */
PDict(Action) action_map; /**< Actions indexed by ActionArgs. */

14
src/file_analysis/DataEvent.cc
View file

@ -7,13 +7,13 @@
using namespace file_analysis;
DataEvent::DataEvent(RecordVal* args, Info* info,
DataEvent::DataEvent(RecordVal* args, File* file,
EventHandlerPtr ce, EventHandlerPtr se)
: Action(args, info), chunk_event(ce), stream_event(se)
: Action(args, file), chunk_event(ce), stream_event(se)
{
}
Action* DataEvent::Instantiate(RecordVal* args, Info* info)
Action* DataEvent::Instantiate(RecordVal* args, File* file)
{
using BifType::Record::FileAnalysis::ActionArgs;
@ -36,7 +36,7 @@ Action* DataEvent::Instantiate(RecordVal* args, Info* info)
if ( stream_val )
stream = event_registry->Lookup(stream_val->AsFunc()->Name());
return new DataEvent(args, info, chunk, stream);
return new DataEvent(args, file, chunk, stream);
}
bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
@ -44,9 +44,10 @@ bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
if ( ! chunk_event ) return true;
val_list* args = new val_list;
args->append(info->GetVal()->Ref());
args->append(file->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
args->append(new Val(offset, TYPE_COUNT));
mgr.QueueEvent(chunk_event, args);
return true;
@ -57,8 +58,9 @@ bool DataEvent::DeliverStream(const u_char* data, uint64 len)
if ( ! stream_event ) return true;
val_list* args = new val_list;
args->append(info->GetVal()->Ref());
args->append(file->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
mgr.QueueEvent(stream_event, args);
return true;

6
src/file_analysis/DataEvent.h
View file

@ -4,7 +4,7 @@
#include <string>
#include "Val.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
namespace file_analysis {
@ -15,7 +15,7 @@ namespace file_analysis {
class DataEvent : public Action {
public:
static Action* Instantiate(RecordVal* args, Info* info);
static Action* Instantiate(RecordVal* args, File* file);
virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
@ -23,7 +23,7 @@ public:
protected:
DataEvent(RecordVal* args, Info* info,
DataEvent(RecordVal* args, File* file,
EventHandlerPtr ce, EventHandlerPtr se);
EventHandlerPtr chunk_event;

8
src/file_analysis/Extract.cc
View file

@ -5,8 +5,8 @@
using namespace file_analysis;
Extract::Extract(RecordVal* args, Info* info, const string& arg_filename)
: Action(args, info), filename(arg_filename)
Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
: Action(args, file), filename(arg_filename)
{
fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
@ -25,7 +25,7 @@ Extract::~Extract()
safe_close(fd);
}
Action* Extract::Instantiate(RecordVal* args, Info* info)
Action* Extract::Instantiate(RecordVal* args, File* file)
{
using BifType::Record::FileAnalysis::ActionArgs;
const char* field = "extract_filename";
@ -33,7 +33,7 @@ Action* Extract::Instantiate(RecordVal* args, Info* info)
if ( ! v ) return 0;
return new Extract(args, info, v->AsString()->CheckString());
return new Extract(args, file, v->AsString()->CheckString());
}
bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)

6
src/file_analysis/Extract.h
View file

@ -4,7 +4,7 @@
#include <string>
#include "Val.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
namespace file_analysis {
@ -15,7 +15,7 @@ namespace file_analysis {
class Extract : public Action {
public:
static Action* Instantiate(RecordVal* args, Info* info);
static Action* Instantiate(RecordVal* args, File* file);
virtual ~Extract();
@ -23,7 +23,7 @@ public:
protected:
Extract(RecordVal* args, Info* info, const string& arg_filename);
Extract(RecordVal* args, File* file, const string& arg_filename);
string filename;
int fd;

188
src/file_analysis/Info.cc → src/file_analysis/File.cc
View file

@ -1,14 +1,15 @@
#include <string>
#include <openssl/md5.h>
#include "Info.h"
#include "InfoTimer.h"
#include "File.h"
#include "FileTimer.h"
#include "FileID.h"
#include "Manager.h"
#include "Reporter.h"
#include "Val.h"
#include "Type.h"
#include "Analyzer.h"
#include "Event.h"
using namespace file_analysis;
@ -32,33 +33,32 @@ static RecordVal* get_conn_id_val(const Connection* conn)
return v;
}
int Info::file_id_idx = -1;
int Info::parent_file_id_idx = -1;
int Info::source_idx = -1;
int Info::conns_idx = -1;
int Info::last_active_idx = -1;
int Info::seen_bytes_idx = -1;
int Info::total_bytes_idx = -1;
int Info::missing_bytes_idx = -1;
int Info::overflow_bytes_idx = -1;
int Info::timeout_interval_idx = -1;
int Info::bof_buffer_size_idx = -1;
int Info::bof_buffer_idx = -1;
int Info::file_type_idx = -1;
int Info::mime_type_idx = -1;
int Info::actions_idx = -1;
int File::id_idx = -1;
int File::parent_id_idx = -1;
int File::source_idx = -1;
int File::conns_idx = -1;
int File::last_active_idx = -1;
int File::seen_bytes_idx = -1;
int File::total_bytes_idx = -1;
int File::missing_bytes_idx = -1;
int File::overflow_bytes_idx = -1;
int File::timeout_interval_idx = -1;
int File::bof_buffer_size_idx = -1;
int File::bof_buffer_idx = -1;
int File::file_type_idx = -1;
int File::mime_type_idx = -1;
magic_t Info::magic = 0;
magic_t Info::magic_mime = 0;
magic_t File::magic = 0;
magic_t File::magic_mime = 0;
string Info::salt;
string File::salt;
void Info::StaticInit()
void File::StaticInit()
{
if ( file_id_idx != -1 ) return;
if ( id_idx != -1 ) return;
file_id_idx = Idx("file_id");
parent_file_id_idx = Idx("parent_file_id");
id_idx = Idx("id");
parent_id_idx = Idx("parent_id");
source_idx = Idx("source");
conns_idx = Idx("conns");
last_active_idx = Idx("last_active");
@ -71,7 +71,6 @@ void Info::StaticInit()
bof_buffer_idx = Idx("bof_buffer");
file_type_idx = Idx("file_type");
mime_type_idx = Idx("mime_type");
actions_idx = Idx("actions");
bro_init_magic(&magic, MAGIC_NONE);
bro_init_magic(&magic_mime, MAGIC_MIME);
@ -79,26 +78,26 @@ void Info::StaticInit()
salt = BifConst::FileAnalysis::salt->CheckString();
}
Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
: file_id(""), unique(unique), val(0), postpone_timeout(false),
first_chunk(true), need_type(false), need_reassembly(false), done(false),
File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
: id(""), unique(unique), val(0), postpone_timeout(false),
first_chunk(true), missed_bof(false), need_reassembly(false), done(false),
actions(this)
{
StaticInit();
char id[20];
char tmp[20];
uint64 hash[2];
string msg(unique + salt);
MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
reinterpret_cast<u_char*>(hash));
uitoa_n(hash[0], id, sizeof(id), 62);
uitoa_n(hash[0], tmp, sizeof(tmp), 62);
DBG_LOG(DBG_FILE_ANALYSIS, "Creating new Info object %s (%s)", id,
DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s (%s)", tmp,
unique.c_str());
val = new RecordVal(BifType::Record::FileAnalysis::Info);
val->Assign(file_id_idx, new StringVal(id));
file_id = FileID(id);
val = new RecordVal(fa_file_type);
val->Assign(id_idx, new StringVal(tmp));
id = FileID(tmp);
if ( conn )
{
@ -113,23 +112,23 @@ Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
UpdateLastActivityTime();
}
Info::~Info()
File::~File()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", file_id.c_str());
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
Unref(val);
}
void Info::UpdateLastActivityTime()
void File::UpdateLastActivityTime()
{
val->Assign(last_active_idx, new Val(network_time, TYPE_TIME));
}
double Info::GetLastActivityTime() const
double File::GetLastActivityTime() const
{
return val->Lookup(last_active_idx)->AsTime();
}
void Info::UpdateConnectionFields(Connection* conn)
void File::UpdateConnectionFields(Connection* conn)
{
if ( ! conn ) return;
@ -148,14 +147,13 @@ void Info::UpdateConnectionFields(Connection* conn)
{
conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
if ( ! is_first )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW_CONN,
this);
file_mgr->FileEvent(file_over_new_connection, this);
}
Unref(idx);
}
uint64 Info::LookupFieldDefaultCount(int idx) const
uint64 File::LookupFieldDefaultCount(int idx) const
{
Val* v = val->LookupWithDefault(idx);
uint64 rval = v->AsCount();
@ -163,7 +161,7 @@ uint64 Info::LookupFieldDefaultCount(int idx) const
return rval;
}
double Info::LookupFieldDefaultInterval(int idx) const
double File::LookupFieldDefaultInterval(int idx) const
{
Val* v = val->LookupWithDefault(idx);
double rval = v->AsInterval();
@ -171,46 +169,31 @@ double Info::LookupFieldDefaultInterval(int idx) const
return rval;
}
int Info::Idx(const string& field)
int File::Idx(const string& field)
{
int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str());
int rval = fa_file_type->FieldOffset(field.c_str());
if ( rval < 0 )
reporter->InternalError("Unknown FileAnalysis::Info field: %s",
field.c_str());
reporter->InternalError("Unknown fa_file field: %s", field.c_str());
return rval;
}
double Info::GetTimeoutInterval() const
double File::GetTimeoutInterval() const
{
return LookupFieldDefaultInterval(timeout_interval_idx);
}
RecordVal* Info::GetResults(RecordVal* args) const
{
TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal();
RecordVal* rval = actions_table->Lookup(args)->AsRecordVal();
if ( ! rval )
{
rval = new RecordVal(BifType::Record::FileAnalysis::ActionResults);
actions_table->Assign(args, rval);
}
return rval;
}
void Info::IncrementByteCount(uint64 size, int field_idx)
void File::IncrementByteCount(uint64 size, int field_idx)
{
uint64 old = LookupFieldDefaultCount(field_idx);
val->Assign(field_idx, new Val(old + size, TYPE_COUNT));
}
void Info::SetTotalBytes(uint64 size)
void File::SetTotalBytes(uint64 size)
{
val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT));
}
bool Info::IsComplete() const
bool File::IsComplete() const
{
Val* total = val->Lookup(total_bytes_idx);
if ( ! total ) return false;
@ -219,42 +202,27 @@ bool Info::IsComplete() const
return false;
}
void Info::ScheduleInactivityTimer() const
void File::ScheduleInactivityTimer() const
{
timer_mgr->Add(new InfoTimer(network_time, file_id, GetTimeoutInterval()));
timer_mgr->Add(new FileTimer(network_time, id, GetTimeoutInterval()));
}
bool Info::AddAction(RecordVal* args)
bool File::AddAction(RecordVal* args)
{
return done ? false : actions.QueueAddAction(args);
}
bool Info::RemoveAction(const RecordVal* args)
bool File::RemoveAction(const RecordVal* args)
{
return done ? false : actions.QueueRemoveAction(args);
}
bool Info::BufferBOF(const u_char* data, uint64 len)
bool File::BufferBOF(const u_char* data, uint64 len)
{
if ( bof_buffer.full || bof_buffer.replayed ) return false;
if ( bof_buffer.chunks.size() == 0 )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_BOF, this);
uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
/* Leaving out this optimization (I think) for now to keep things simpler.
// If first chunk satisfies desired size, do everything now without copying.
if ( bof_buffer.chunks.empty() && len >= desired_size )
{
bof_buffer.full = bof_buffer.replayed = true;
val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0)));
file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
// TODO: libmagic stuff
return false;
}
*/
bof_buffer.chunks.push_back(new BroString(data, len, 0));
bof_buffer.size += len;
@ -267,7 +235,7 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
return true;
}
bool Info::DetectTypes(const u_char* data, uint64 len)
bool File::DetectTypes(const u_char* data, uint64 len)
{
const char* desc = bro_magic_buffer(magic, data, len);
const char* mime = bro_magic_buffer(magic_mime, data, len);
@ -281,7 +249,7 @@ bool Info::DetectTypes(const u_char* data, uint64 len)
return desc || mime;
}
void Info::ReplayBOF()
void File::ReplayBOF()
{
if ( bof_buffer.replayed ) return;
bof_buffer.replayed = true;
@ -289,36 +257,33 @@ void Info::ReplayBOF()
if ( bof_buffer.chunks.empty() )
{
// Since we missed the beginning, try file type detect on next data in.
need_type = true;
missed_bof = true;
return;
}
BroString* bs = concatenate(bof_buffer.chunks);
val->Assign(bof_buffer_idx, new StringVal(bs));
bool have_type = DetectTypes(bs->Bytes(), bs->Len());
using BifEnum::FileAnalysis::TRIGGER_BOF_BUFFER;
file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
DetectTypes(bs->Bytes(), bs->Len());
if ( have_type )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
file_mgr->FileEvent(file_new, this);
mgr.Drain(); // need immediate feedback about actions to add
for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
}
void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
void File::DataIn(const u_char* data, uint64 len, uint64 offset)
{
actions.DrainModifications();
if ( first_chunk )
{
if ( DetectTypes(data, len) )
{
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
// TODO: this should all really be delayed until we attempt reassembly
DetectTypes(data, len);
file_mgr->FileEvent(file_new, this);
mgr.Drain(); // need immediate feedback about actions to add
actions.DrainModifications();
}
first_chunk = false;
}
@ -344,21 +309,19 @@ void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
IncrementByteCount(len, seen_bytes_idx);
}
void Info::DataIn(const u_char* data, uint64 len)
void File::DataIn(const u_char* data, uint64 len)
{
actions.DrainModifications();
if ( BufferBOF(data, len) ) return;
if ( need_type )
if ( missed_bof )
{
if ( DetectTypes(data, len) )
{
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
DetectTypes(data, len);
file_mgr->FileEvent(file_new, this);
mgr.Drain(); // need immediate feedback about actions to add
actions.DrainModifications();
}
need_type = false;
missed_bof = false;
}
Action* act = 0;
@ -383,7 +346,7 @@ void Info::DataIn(const u_char* data, uint64 len)
IncrementByteCount(len, seen_bytes_idx);
}
void Info::EndOfFile()
void File::EndOfFile()
{
if ( done ) return;
@ -403,15 +366,12 @@ void Info::EndOfFile()
actions.QueueRemoveAction(act->Args());
}
if ( IsComplete() )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, this);
else
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, this);
file_mgr->FileEvent(file_state_remove, this);
actions.DrainModifications();
}
void Info::Gap(uint64 offset, uint64 len)
void File::Gap(uint64 offset, uint64 len)
{
actions.DrainModifications();
@ -428,7 +388,7 @@ void Info::Gap(uint64 offset, uint64 len)
actions.QueueRemoveAction(act->Args());
}
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, this);
file_mgr->FileEvent(file_gap, this);
actions.DrainModifications();
IncrementByteCount(len, missing_bytes_idx);

39
src/file_analysis/Info.h → src/file_analysis/File.h
View file

@ -1,5 +1,5 @@
#ifndef FILE_ANALYSIS_INFO_H
#define FILE_ANALYSIS_INFO_H
#ifndef FILE_ANALYSIS_FILE_H
#define FILE_ANALYSIS_FILE_H
#include <string>
#include <vector>
@ -15,12 +15,14 @@
namespace file_analysis {
/**
* Wrapper class around \c FileAnalysis::Info record values from script layer.
* Wrapper class around \c fa_file record values from script layer.
*/
class Info {
class File {
friend class Manager;
public:
~Info();
~File();
/**
* @return the #val record.
@ -33,16 +35,9 @@ public:
double GetTimeoutInterval() const;
/**
* @return value of the "file_id" field from #val record.
* @return value of the "id" field from #val record.
*/
FileID GetFileID() const { return file_id; }
/**
* @return looks up the value of the "actions" field in the #val record at
* the index corresponding to \a args. If there was no value at
* the index, it is created.
*/
RecordVal* GetResults(RecordVal* args) const;
FileID GetID() const { return id; }
/**
* @return the string which uniquely identifies the file.
@ -113,12 +108,10 @@ public:
protected:
friend class Manager;
/**
* Constructor; only file_analysis::Manager should be creating these.
*/
Info(const string& unique, Connection* conn = 0,
File(const string& unique, Connection* conn = 0,
AnalyzerTag::Tag tag = AnalyzerTag::Error);
/**
@ -162,12 +155,12 @@ protected:
*/
bool DetectTypes(const u_char* data, uint64 len);
FileID file_id; /**< A pretty hash that likely identifies file*/
FileID id; /**< A pretty hash that likely identifies file */
string unique; /**< A string that uniquely identifies file */
RecordVal* val; /**< \c FileAnalysis::Info from script layer. */
RecordVal* val; /**< \c fa_file from script layer. */
bool postpone_timeout; /**< Whether postponing timeout is requested. */
bool first_chunk; /**< Track first non-linear chunk. */
bool need_type; /**< Flags next data input to be magic typed. */
bool missed_bof; /**< Flags that we missed start of file. */
bool need_reassembly; /**< Whether file stream reassembly is needed. */
bool done; /**< If this object is about to be deleted. */
ActionSet actions;
@ -198,9 +191,8 @@ protected:
static string salt;
public:
static int file_id_idx;
static int parent_file_id_idx;
static int id_idx;
static int parent_id_idx;
static int source_idx;
static int conns_idx;
static int last_active_idx;
@ -213,7 +205,6 @@ public:
static int bof_buffer_idx;
static int file_type_idx;
static int mime_type_idx;
static int actions_idx;
};
} // namespace file_analysis

20
src/file_analysis/InfoTimer.cc → src/file_analysis/FileTimer.cc
View file

@ -1,23 +1,23 @@
#include "Manager.h"
#include "Info.h"
#include "File.h"
using namespace file_analysis;
InfoTimer::InfoTimer(double t, const FileID& id, double interval)
FileTimer::FileTimer(double t, const FileID& id, double interval)
: Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id)
{
DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s",
file_id.c_str(), interval);
}
void InfoTimer::Dispatch(double t, int is_expire)
void FileTimer::Dispatch(double t, int is_expire)
{
Info* info = file_mgr->Lookup(file_id);
File* file = file_mgr->Lookup(file_id);
if ( ! info ) return;
if ( ! file ) return;
double last_active = info->GetLastActivityTime();
double last_active = file->GetLastActivityTime();
double inactive_time = t > last_active ? t - last_active : 0.0;
DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, "
@ -26,13 +26,13 @@ void InfoTimer::Dispatch(double t, int is_expire)
if ( last_active == 0.0 )
{
// was created when network_time was zero, so re-schedule w/ valid time
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
file->UpdateLastActivityTime();
file->ScheduleInactivityTimer();
return;
}
if ( inactive_time >= info->GetTimeoutInterval() )
if ( inactive_time >= file->GetTimeoutInterval() )
file_mgr->Timeout(file_id);
else if ( ! is_expire )
info->ScheduleInactivityTimer();
file->ScheduleInactivityTimer();
}

10
src/file_analysis/InfoTimer.h → src/file_analysis/FileTimer.h
View file

@ -1,5 +1,5 @@
#ifndef FILE_ANALYSIS_INFOTIMER_H
#define FILE_ANALYSIS_INFOTIMER_H
#ifndef FILE_ANALYSIS_FILETIMER_H
#define FILE_ANALYSIS_FILETIMER_H
#include <string>
#include "Timer.h"
@ -10,13 +10,13 @@ namespace file_analysis {
/**
* Timer to periodically check if file analysis for a given file is inactive.
*/
class InfoTimer : public Timer {
class FileTimer : public Timer {
public:
InfoTimer(double t, const FileID& id, double interval);
FileTimer(double t, const FileID& id, double interval);
/**
* Check inactivity of file_analysis::Info corresponding to #file_id,
* Check inactivity of file_analysis::File corresponding to #file_id,
* reschedule if active, else call file_analysis::Manager::Timeout.
*/
void Dispatch(double t, int is_expire);

16
src/file_analysis/Hash.cc
View file

@ -2,15 +2,13 @@
#include "Hash.h"
#include "util.h"
#include "Event.h"
using namespace file_analysis;
Hash::Hash(RecordVal* args, Info* info, HashVal* hv, const char* field)
: Action(args, info), hash(hv), fed(false)
Hash::Hash(RecordVal* args, File* file, HashVal* hv, const char* arg_kind)
: Action(args, file), hash(hv), fed(false), kind(arg_kind)
{
using BifType::Record::FileAnalysis::ActionResults;
if ( (result_field_idx = ActionResults->FieldOffset(field)) < 0 )
reporter->InternalError("Missing ActionResults field: %s", field);
hash->Init();
}
@ -45,6 +43,10 @@ void Hash::Finalize()
{
if ( ! hash->IsValid() || ! fed ) return;
StringVal* sv = hash->Get();
info->GetResults(args)->Assign(result_field_idx, sv);
val_list* vl = new val_list();
vl->append(file->GetVal()->Ref());
vl->append(new StringVal(kind));
vl->append(hash->Get());
mgr.QueueEvent(file_hash, vl);
}

30
src/file_analysis/Hash.h
View file

@ -5,7 +5,7 @@
#include "Val.h"
#include "OpaqueVal.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
namespace file_analysis {
@ -26,51 +26,51 @@ public:
protected:
Hash(RecordVal* args, Info* info, HashVal* hv, const char* field);
Hash(RecordVal* args, File* file, HashVal* hv, const char* kind);
void Finalize();
HashVal* hash;
bool fed;
int result_field_idx;
const char* kind;
};
class MD5 : public Hash {
public:
static Action* Instantiate(RecordVal* args, Info* info)
{ return new MD5(args, info); }
static Action* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new MD5(args, file) : 0; }
protected:
MD5(RecordVal* args, Info* info)
: Hash(args, info, new MD5Val(), "md5")
MD5(RecordVal* args, File* file)
: Hash(args, file, new MD5Val(), "md5")
{}
};
class SHA1 : public Hash {
public:
static Action* Instantiate(RecordVal* args, Info* info)
{ return new SHA1(args, info); }
static Action* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new SHA1(args, file) : 0; }
protected:
SHA1(RecordVal* args, Info* info)
: Hash(args, info, new SHA1Val(), "sha1")
SHA1(RecordVal* args, File* file)
: Hash(args, file, new SHA1Val(), "sha1")
{}
};
class SHA256 : public Hash {
public:
static Action* Instantiate(RecordVal* args, Info* info)
{ return new SHA256(args, info); }
static Action* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new SHA256(args, file) : 0; }
protected:
SHA256(RecordVal* args, Info* info)
: Hash(args, info, new SHA256Val(), "sha256")
SHA256(RecordVal* args, File* file)
: Hash(args, file, new SHA256Val(), "sha256")
{}
};

196
src/file_analysis/Manager.cc
View file

@ -2,7 +2,7 @@
#include <string>
#include "Manager.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
#include "Var.h"
#include "Event.h"
@ -29,78 +29,58 @@ void Manager::Terminate()
Timeout(keys[i], true);
}
void Manager::ReceiveHandle(const string& handle)
void Manager::SetHandle(const string& handle)
{
if ( pending.empty() )
reporter->InternalError("File analysis underflow");
PendingFile* pf = pending.front();
if ( ! handle.empty() )
pf->Finish(handle);
delete pf;
pending.pop();
}
void Manager::EventDrainDone()
{
if ( pending.empty() ) return;
reporter->Error("Too few return_file_handle() calls, discarding pending"
" file analysis input.");
while ( ! pending.empty() )
{
delete pending.front();
pending.pop();
}
current_handle = handle;
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
{
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingDataInChunk(data, len, offset, tag, conn));
GetFileHandle(tag, conn, is_orig);
DataIn(data, len, offset, GetFile(current_handle, conn, tag));
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
const string& unique)
{
DataIn(data, len, offset, GetInfo(unique));
DataIn(data, len, offset, GetFile(unique));
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
Info* info)
File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->DataIn(data, len, offset);
file->DataIn(data, len, offset);
if ( info->IsComplete() )
RemoveFile(info->GetUnique());
if ( file->IsComplete() )
RemoveFile(file->GetUnique());
}
void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig)
{
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingDataInStream(data, len, tag, conn));
GetFileHandle(tag, conn, is_orig);
DataIn(data, len, GetFile(current_handle, conn, tag));
}
void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
{
DataIn(data, len, GetInfo(unique));
DataIn(data, len, GetFile(unique));
}
void Manager::DataIn(const u_char* data, uint64 len, Info* info)
void Manager::DataIn(const u_char* data, uint64 len, File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->DataIn(data, len);
file->DataIn(data, len);
if ( info->IsComplete() )
RemoveFile(info->GetUnique());
if ( file->IsComplete() )
RemoveFile(file->GetUnique());
}
void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
@ -112,8 +92,9 @@ void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
{
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingEOF(tag, conn));
GetFileHandle(tag, conn, is_orig);
EndOfFile(current_handle);
}
void Manager::EndOfFile(const string& unique)
@ -125,102 +106,98 @@ void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig)
{
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingGap(offset, len, tag, conn));
GetFileHandle(tag, conn, is_orig);
Gap(offset, len, GetFile(current_handle, conn, tag));
}
void Manager::Gap(uint64 offset, uint64 len, const string& unique)
{
Gap(offset, len, GetInfo(unique));
Gap(offset, len, GetFile(unique));
}
void Manager::Gap(uint64 offset, uint64 len, Info* info)
void Manager::Gap(uint64 offset, uint64 len, File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->Gap(offset, len);
file->Gap(offset, len);
}
void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig)
{
if ( IsDisabled(tag) ) return;
if ( ! QueueHandleEvent(tag, conn, is_orig) ) return;
pending.push(new PendingSize(size, tag, conn));
GetFileHandle(tag, conn, is_orig);
SetSize(size, GetFile(current_handle, conn, tag));
}
void Manager::SetSize(uint64 size, const string& unique)
{
SetSize(size, GetInfo(unique));
SetSize(size, GetFile(unique));
}
void Manager::SetSize(uint64 size, Info* info)
void Manager::SetSize(uint64 size, File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->SetTotalBytes(size);
file->SetTotalBytes(size);
if ( info->IsComplete() )
RemoveFile(info->GetUnique());
if ( file->IsComplete() )
RemoveFile(file->GetUnique());
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
void Manager::FileEvent(EventHandlerPtr h, File* file)
{
if ( IsIgnored(info->GetUnique()) ) return;
if ( ! h ) return;
if ( IsIgnored(file->GetUnique()) ) return;
const ID* id = global_scope()->Lookup("FileAnalysis::policy");
assert(id);
const Func* hook = id->ID_Val()->AsFunc();
val_list * vl = new val_list();
vl->append(file->GetVal()->Ref());
val_list vl(2);
vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger));
vl.append(info->val->Ref());
info->postpone_timeout = false;
Val* result = hook->Call(&vl);
Unref(result);
mgr.QueueEvent(h, vl);
}
bool Manager::PostponeTimeout(const FileID& file_id) const
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return false;
if ( ! file ) return false;
info->postpone_timeout = true;
file->postpone_timeout = true;
return true;
}
bool Manager::AddAction(const FileID& file_id, RecordVal* args) const
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return false;
if ( ! file ) return false;
return info->AddAction(args);
return file->AddAction(args);
}
bool Manager::RemoveAction(const FileID& file_id, const RecordVal* args) const
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return false;
if ( ! file ) return false;
return info->RemoveAction(args);
return file->RemoveAction(args);
}
Info* Manager::GetInfo(const string& unique, Connection* conn,
File* Manager::GetFile(const string& unique, Connection* conn,
AnalyzerTag::Tag tag)
{
if ( unique.empty() ) return 0;
if ( IsIgnored(unique) ) return 0;
Info* rval = str_map[unique];
File* rval = str_map[unique];
if ( ! rval )
{
rval = str_map[unique] = new Info(unique, conn, tag);
FileID id = rval->GetFileID();
rval = str_map[unique] = new File(unique, conn, tag);
FileID id = rval->GetID();
if ( id_map[id] )
{
@ -229,7 +206,6 @@ Info* Manager::GetInfo(const string& unique, Connection* conn,
}
id_map[id] = rval;
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval);
rval->ScheduleInactivityTimer();
if ( IsIgnored(unique) ) return 0;
}
@ -242,7 +218,7 @@ Info* Manager::GetInfo(const string& unique, Connection* conn,
return rval;
}
Info* Manager::Lookup(const FileID& file_id) const
File* Manager::Lookup(const FileID& file_id) const
{
IDMap::const_iterator it = id_map.find(file_id);
@ -253,25 +229,28 @@ Info* Manager::Lookup(const FileID& file_id) const
void Manager::Timeout(const FileID& file_id, bool is_terminating)
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return;
if ( ! file ) return;
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
file->postpone_timeout = false;
if ( info->postpone_timeout && ! is_terminating )
FileEvent(file_timeout, file);
mgr.Drain(); // need immediate feedback about whether to postpone
if ( file->postpone_timeout && ! is_terminating )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s",
info->GetFileID().c_str());
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
file->GetID().c_str());
file->UpdateLastActivityTime();
file->ScheduleInactivityTimer();
return;
}
DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
info->GetFileID().c_str());
file->GetID().c_str());
RemoveFile(info->GetUnique());
RemoveFile(file->GetUnique());
}
bool Manager::IgnoreFile(const FileID& file_id)
@ -295,7 +274,7 @@ bool Manager::RemoveFile(const string& unique)
it->second->EndOfFile();
FileID id = it->second->GetFileID();
FileID id = it->second->GetID();
DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", id.c_str());
@ -313,6 +292,21 @@ bool Manager::IsIgnored(const string& unique)
return ignored.find(unique) != ignored.end();
}
void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
{
current_handle.clear();
if ( ! get_file_handle ) return;
val_list* vl = new val_list();
vl->append(new Val(tag, TYPE_COUNT));
vl->append(c->BuildConnVal());
vl->append(new Val(is_orig, TYPE_BOOL));
mgr.QueueEvent(get_file_handle, vl);
mgr.Drain(); // need file handle immediately so we don't have to buffer data
}
bool Manager::IsDisabled(AnalyzerTag::Tag tag)
{
if ( ! disabled )
@ -329,17 +323,3 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
return rval;
}
bool Manager::QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
bool is_orig)
{
if ( ! get_file_handle ) return false;
val_list* vl = new val_list();
vl->append(new Val(tag, TYPE_COUNT));
vl->append(conn->BuildConnVal());
vl->append(new Val(is_orig, TYPE_BOOL));
mgr.QueueEvent(get_file_handle, vl);
return true;
}

78
src/file_analysis/Manager.h
View file

@ -12,11 +12,11 @@
#include "Val.h"
#include "Analyzer.h"
#include "Timer.h"
#include "EventHandler.h"
#include "Info.h"
#include "InfoTimer.h"
#include "File.h"
#include "FileTimer.h"
#include "FileID.h"
#include "PendingFile.h"
namespace file_analysis {
@ -24,6 +24,8 @@ namespace file_analysis {
* Main entry point for interacting with file analysis.
*/
class Manager {
friend class FileTimer;
public:
Manager();
@ -36,17 +38,9 @@ public:
void Terminate();
/**
* Associates a handle with the next element in the #pending queue, which
* will immediately push that element all the way through the file analysis
* framework, possibly evaluating any policy hooks.
* Take in a unique file handle string to identifiy incoming file data.
*/
void ReceiveHandle(const string& handle);
/**
* Called when all events have been drained from the event queue.
* There should be no pending file input/data at this point.
*/
void EventDrainDone();
void SetHandle(const string& handle);
/**
* Pass in non-sequential file data.
@ -56,7 +50,7 @@ public:
void DataIn(const u_char* data, uint64 len, uint64 offset,
const string& unique);
void DataIn(const u_char* data, uint64 len, uint64 offset,
Info* info);
File* file);
/**
* Pass in sequential file data.
@ -64,7 +58,7 @@ public:
void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig);
void DataIn(const u_char* data, uint64 len, const string& unique);
void DataIn(const u_char* data, uint64 len, Info* info);
void DataIn(const u_char* data, uint64 len, File* file);
/**
* Signal the end of file data.
@ -79,7 +73,7 @@ public:
void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
void Gap(uint64 offset, uint64 len, const string& unique);
void Gap(uint64 offset, uint64 len, Info* info);
void Gap(uint64 offset, uint64 len, File* file);
/**
* Provide the expected number of bytes that comprise a file.
@ -87,7 +81,7 @@ public:
void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
void SetSize(uint64 size, const string& unique);
void SetSize(uint64 size, Info* info);
void SetSize(uint64 size, File* file);
/**
* Starts ignoring a file, which will finally be removed from internal
@ -97,8 +91,8 @@ public:
bool IgnoreFile(const FileID& file_id);
/**
* If called during \c FileAnalysis::policy evaluation for a
* \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout.
* If called during a \c file_timeout event handler, requests deferral of
* analysis timeout.
*/
bool PostponeTimeout(const FileID& file_id) const;
@ -117,44 +111,40 @@ public:
bool RemoveAction(const FileID& file_id, const RecordVal* args) const;
/**
* Calls the \c FileAnalysis::policy hook.
* Queues an event related to the file's life-cycle.
*/
void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info);
void FileEvent(EventHandlerPtr h, File* file);
protected:
friend class InfoTimer;
friend class PendingFile;
typedef map<string, Info*> StrMap;
typedef map<string, File*> StrMap;
typedef set<string> StrSet;
typedef map<FileID, Info*> IDMap;
typedef queue<PendingFile*> PendingQueue;
typedef map<FileID, File*> IDMap;
/**
* @return the Info object mapped to \a unique or a null pointer if analysis
* is being ignored for the associated file. An Info object may be
* @return the File object mapped to \a unique or a null pointer if analysis
* is being ignored for the associated file. An File object may be
* created if a mapping doesn't exist, and if it did exist, the
* activity time is refreshed along with any connection-related
* fields.
*/
Info* GetInfo(const string& unique, Connection* conn = 0,
File* GetFile(const string& unique, Connection* conn = 0,
AnalyzerTag::Tag tag = AnalyzerTag::Error);
/**
* @return the Info object mapped to \a file_id, or a null pointer if no
* @return the File object mapped to \a file_id, or a null pointer if no
* mapping exists.
*/
Info* Lookup(const FileID& file_id) const;
File* Lookup(const FileID& file_id) const;
/**
* Evaluate timeout policy for a file and remove the Info object mapped to
* Evaluate timeout policy for a file and remove the File object mapped to
* \a file_id if needed.
*/
void Timeout(const FileID& file_id, bool is_terminating = ::terminating);
/**
* Immediately remove file_analysis::Info object associated with \a unique.
* Immediately remove file_analysis::File object associated with \a unique.
* @return false if file string did not map to anything, else true.
*/
bool RemoveFile(const string& unique);
@ -164,22 +154,22 @@ protected:
*/
bool IsIgnored(const string& unique);
/**
* Sets #current_handle to a unique file handle string based on what the
* \c get_file_handle event derives from the connection params. The
* event queue is flushed so that we can get the handle value immediately.
*/
void GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig);
/**
* @return whether file analysis is disabled for the given analyzer.
*/
static bool IsDisabled(AnalyzerTag::Tag tag);
/**
* Queues \c get_file_handle event in order to retrieve unique file handle.
* @return true if there is a handler for the event, else false.
*/
static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
IDMap id_map; /**< Map file IDs to \c FileAnalysis::Info records. */
StrMap str_map; /**< Map unique string to file_analysis::File. */
IDMap id_map; /**< Map file ID to file_analysis::File records. */
StrSet ignored; /**< Ignored files. Will be finally removed on EOF. */
PendingQueue pending; /**< Files awaiting a unique handle. */
string current_handle; /**< Last file handle set by get_file_handle event.*/
static TableVal* disabled; /**< Table of disabled analyzers. */
};

111
src/file_analysis/PendingFile.cc
View file

@ -1,111 +0,0 @@
#include "PendingFile.h"
#include "Manager.h"
using namespace file_analysis;
static void copy_data(const u_char** dst, const u_char* src, uint64 len)
{
u_char* tmp = new u_char[len];
memcpy(tmp, src, len);
*dst = tmp;
}
static string conn_str(Connection* c)
{
char op[256], rp[256];
modp_ulitoa10(ntohs(c->OrigPort()), op);
modp_ulitoa10(ntohs(c->RespPort()), rp);
string rval = c->OrigAddr().AsString() + ":" + op + "->" +
c->RespAddr().AsString() + ":" + rp;
return rval;
}
PendingFile::PendingFile(Connection* arg_conn, AnalyzerTag::Tag arg_tag)
: conn(arg_conn), tag(arg_tag)
{
Ref(conn);
DBG_LOG(DBG_FILE_ANALYSIS, "New pending file: %s", conn_str(conn).c_str());
}
PendingFile::~PendingFile()
{
Unref(conn);
DBG_LOG(DBG_FILE_ANALYSIS, "Delete pending file: %s",
conn_str(conn).c_str());
}
Info* PendingFile::GetInfo(const string& handle) const
{
return file_mgr->GetInfo(handle, conn, tag);
}
PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
uint64 arg_offset,
AnalyzerTag::Tag arg_tag,
Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), len(arg_len),
offset(arg_offset)
{
copy_data(&data, arg_data, len);
}
void PendingDataInChunk::Finish(const string& handle) const
{
file_mgr->DataIn(data, len, offset, GetInfo(handle));
}
PendingDataInChunk::~PendingDataInChunk()
{
delete [] data;
}
PendingDataInStream::PendingDataInStream(const u_char* arg_data, uint64 arg_len,
AnalyzerTag::Tag arg_tag,
Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), len(arg_len)
{
copy_data(&data, arg_data, len);
}
void PendingDataInStream::Finish(const string& handle) const
{
file_mgr->DataIn(data, len, GetInfo(handle));
}
PendingDataInStream::~PendingDataInStream()
{
delete [] data;
}
PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len,
AnalyzerTag::Tag arg_tag, Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), offset(arg_offset),
len(arg_len)
{
}
void PendingGap::Finish(const string& handle) const
{
file_mgr->Gap(offset, len, GetInfo(handle));
}
PendingEOF::PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn)
: PendingFile(arg_conn, arg_tag)
{
}
void PendingEOF::Finish(const string& handle) const
{
file_mgr->EndOfFile(handle);
}
PendingSize::PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
Connection* arg_conn)
: PendingFile(arg_conn, arg_tag), size(arg_size)
{
}
void PendingSize::Finish(const string& handle) const
{
file_mgr->SetSize(size, GetInfo(handle));
}

103
src/file_analysis/PendingFile.h
View file

@ -1,103 +0,0 @@
#ifndef FILE_ANALYSIS_PENDINGFILE_H
#define FILE_ANALYSIS_PENDINGFILE_H
#include "AnalyzerTags.h"
#include "Conn.h"
#include "Info.h"
namespace file_analysis {
/**
* Provides buffering for file contents until the script-layer is able to
* return a unique file handle for it.
*/
class PendingFile {
public:
virtual ~PendingFile();
virtual void Finish(const string& handle) const = 0;
protected:
PendingFile(Connection* arg_conn,
AnalyzerTag::Tag arg_tag = AnalyzerTag::Error);
Info* GetInfo(const string& handle) const;
Connection* conn;
AnalyzerTag::Tag tag;
};
class PendingDataInChunk : public PendingFile {
public:
PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
uint64 arg_offset, AnalyzerTag::Tag arg_tag,
Connection* arg_conn);
virtual ~PendingDataInChunk();
virtual void Finish(const string& handle) const;
protected:
const u_char* data;
uint64 len;
uint64 offset;
};
class PendingDataInStream : public PendingFile {
public:
PendingDataInStream(const u_char* arg_data, uint64 arg_len,
AnalyzerTag::Tag arg_tag, Connection* arg_conn);
virtual ~PendingDataInStream();
virtual void Finish(const string& handle) const;
protected:
const u_char* data;
uint64 len;
};
class PendingGap : public PendingFile {
public:
PendingGap(uint64 arg_offset, uint64 arg_len, AnalyzerTag::Tag arg_tag,
Connection* arg_conn);
virtual void Finish(const string& handle) const;
protected:
uint64 offset;
uint64 len;
};
class PendingEOF : public PendingFile {
public:
PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn);
virtual void Finish(const string& handle) const;
};
class PendingSize : public PendingFile {
public:
PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
Connection* arg_conn);
virtual void Finish(const string& handle) const;
protected:
uint64 size;
};
} // namespace file_analysis
#endif

25
src/types.bif
View file

@ -226,3 +226,28 @@ type gtp_rai: record;
type gtp_qos_profile: record;
type gtp_private_extension: record;
type gtp_gsn_addr: record;
module FileAnalysis;
type ActionArgs: record;
## An enumeration of various file analysis actions that can be taken.
enum Action %{
## Extract a file to local filesystem
ACTION_EXTRACT,
## Calculate an MD5 digest of the file's contents.
ACTION_MD5,
## Calculate an SHA1 digest of the file's contents.
ACTION_SHA1,
## Calculate an SHA256 digest of the file's contents.
ACTION_SHA256,
## Deliver the file contents to the script-layer in an event.
ACTION_DATA_EVENT,
%}
module GLOBAL;

6
testing/btest/Baseline/istate.events-ssl/sender.http.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#open 2013-03-22-21-05-55
#open 2013-04-10-15-49-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-03-22-21-05-56
1365608977.146651 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-04-10-15-49-38

6
testing/btest/Baseline/istate.events/sender.http.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#open 2013-03-22-21-03-17
#open 2013-04-10-15-48-08
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-03-22-21-03-18
1365608887.935644 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-04-10-15-48-09

23
testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out
View file

@ -1,9 +1,8 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
file_stream, Cx92a0ym5R8, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea
@ -14,15 +13,15 @@ file_stream, Cx92a0ym5R8, 476, ormat for RFC2253^J compliance. (Jon Siwek)^J^
file_chunk, Cx92a0ym5R8, 476, 2524, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the
file_stream, Cx92a0ym5R8, 1024, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP
file_chunk, Cx92a0ym5R8, 1024, 3000, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP
FileAnalysis::TRIGGER_DONE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705
source: HTTP
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
MD5: 397168fd09991a0e712254df7bc639ac
SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
file_stream, Cx92a0ym5R8, 476, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J
file_chunk, Cx92a0ym5R8, 476, 4024, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J
file_stream, Cx92a0ym5R8, 205, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J
file_chunk, Cx92a0ym5R8, 205, 4500, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J
FILE_STATE_REMOVE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705
source: HTTP
MD5: 397168fd09991a0e712254df7bc639ac
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18

16
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout
View file

@ -1,21 +1,21 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
source: HTTP
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_EOF
FILE_TIMEOUT
FILE_TIMEOUT
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]
total bytes: 1022920

9
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out
View file

@ -1,12 +1,11 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705

7
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out
View file

@ -1,2 +1,7 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FILE_BOF_BUFFER
^J0.26 | 201
FILE_TYPE
file type is set
mime type is set

11
testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out
View file

@ -1,15 +1,14 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
sidhzrR4IT8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
The Nationa
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
sidhzrR4IT8, 16557, 0
[orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp]
source: FTP_DATA
SHA1: 44586aed07cfe19cad25076af98f535585cd5797
MD5: 7192a8075196267203adb3dfaa5c908d
SHA1: 44586aed07cfe19cad25076af98f535585cd5797
SHA256: 202674eba48e832690a4475113acf8b16a3f6c82c04c94b36bb2c7ce457ac8d2

11
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out
View file

@ -1,15 +1,14 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
kg59rqyYxN, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
{^J "origin
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
kg59rqyYxN, 197, 0
[orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp]
source: HTTP
SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31
MD5: 5baba7eea57bc8a42a92c817ed566d72
SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31
SHA256: 202b775be087f5af98e95120e42769a9b3488f84c5aa79c4f4c1093d348f849c

11
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out
View file

@ -1,16 +1,15 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705
source: HTTP
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
MD5: 397168fd09991a0e712254df7bc639ac
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out
View file

@ -1,10 +1,10 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
7gZBKVUgy4l, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FileAnalysis::TRIGGER_DONE
FILE_OVER_NEW_CONNECTION
FILE_STATE_REMOVE
7gZBKVUgy4l, 555523, 0
[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]
[orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp]

14
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out
View file

@ -1,20 +1,20 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
source: HTTP
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_EOF
FILE_TIMEOUT
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]
total bytes: 1022920

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out
View file

@ -1,10 +1,10 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
uHS14uhRKGe, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FileAnalysis::TRIGGER_DONE
FILE_OVER_NEW_CONNECTION
FILE_STATE_REMOVE
uHS14uhRKGe, 498702, 0
[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]
[orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp]

55
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out
View file

@ -1,78 +1,73 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
aFQKI8SPOL2, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
/*^J********
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
aFQKI8SPOL2, 2675, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
source: HTTP
SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
MD5: b932c3310ce47e158d1a5a42e0b01279
SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42
FileAnalysis::TRIGGER_NEW
FILE_NEW
CCU3vUEr06l, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
//-- Google
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
CCU3vUEr06l, 21421, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
source: HTTP
SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
MD5: e732f7bf1d7cb4eedcb1661697d7bc8c
SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da
FileAnalysis::TRIGGER_NEW
FILE_NEW
HCzA0dVwDPj, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
GIF89a^D\0^D\0\xb3
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
HCzA0dVwDPj, 94, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
total bytes: 94
source: HTTP
SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
MD5: d903de7e30db1691d3130ba5eae6b9a7
SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315
FileAnalysis::TRIGGER_NEW
FILE_NEW
a1Zu1fteVEf, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
\x89PNG^M^J^Z^J\0\0\0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
a1Zu1fteVEf, 2349, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
total bytes: 2349
source: HTTP
SHA1: 560eab5a0177246827a94042dd103916d8765ac7
MD5: e0029eea80812e9a8e57b8d05d52938a
SHA1: 560eab5a0177246827a94042dd103916d8765ac7
SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4
FileAnalysis::TRIGGER_NEW
FILE_NEW
xXlF7wFdsR, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
\x89PNG^M^J^Z^J\0\0\0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
xXlF7wFdsR, 27579, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
total bytes: 27579
source: HTTP
SHA1: ee2b41bdef85de14ef332da14fc392f110b84249
MD5: 30aa926344f58019d047e85ba049ca1e
SHA1: ee2b41bdef85de14ef332da14fc392f110b84249
SHA256: eb482bda230a215b90aedbfe1eee72b8193608df76a319aaf11fb85511579a1e

22
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out
View file

@ -1,32 +1,30 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
v5HLI7MxPQh, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
hello world
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
v5HLI7MxPQh, 11, 0
[orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp]
total bytes: 11
source: HTTP
SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3
SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
FileAnalysis::TRIGGER_NEW
FILE_NEW
PZS1XGHkIf1, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
{^J "origin
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
PZS1XGHkIf1, 366, 0
[orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp]
total bytes: 366
source: HTTP
SHA1: 6a1582672c203210c6d18d700322060b676365e7
MD5: c9337794df612aeaa901dcf9fa446bca
SHA1: 6a1582672c203210c6d18d700322060b676365e7
SHA256: 8eb24c16df7cb45cb6a1790b0d26ad2571f754228d0ac111b3ac59adbfecbeb8

11
testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout
View file

@ -1,14 +1,13 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
nYgPNGLrZf9, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
#separator
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
nYgPNGLrZf9, 311, 0
source: ../input.log
SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1
MD5: bf4dfa6169b74146da5236e918743599
SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1
SHA256: 4e573192c5ea75da72494812fe24dae53a577837b2079df012fd464903d68a6f

11
testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out
View file

@ -1,15 +1,14 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
wqKMAamJVSb, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
PK^C^D^T\0\0\0^H\0\xae
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
wqKMAamJVSb, 42208, 0
[orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp]
source: IRC_DATA
SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218
MD5: 8c0803242f549c2780cb88b9a9215c65
SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218
SHA256: e4f0b0b9d7580e7a22dc1093c8db4df7d0115a4f3b03cc2875cc69705f0d0204

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path file_analysis
#open 2013-03-29-18-28-57
#fields file_id parent_file_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type timedout conn_uids actions_taken extracted_files md5 sha1 sha256
#open 2013-04-04-21-22-26
#fields id parent_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type timedout conn_uids actions_taken extracted_files md5 sha1 sha256
#types string string string time count count count count interval count string string bool table[string] table[enum] table[string] string string string
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set F UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
#close 2013-03-29-18-28-57
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set F UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
#close 2013-04-04-21-22-26

33
testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out
View file

@ -1,45 +1,42 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
cwR7l6Zctxb, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
Hello^M^J^M^J ^M
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
cwR7l6Zctxb, 79, 0
[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
source: SMTP
SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
MD5: 92bca2e6cdcde73647125da7dccbdd07
SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
FileAnalysis::TRIGGER_NEW
FILE_NEW
ZAOEQmRyxv1, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
<html xmlns
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
ZAOEQmRyxv1, 1918, 0
[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
source: SMTP
SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
MD5: d194c6359c85bb88b54caee18b1e9b44
SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
SHA256: b9556e92ddbe52379b64804136f830d111cafe7fcd78e54817fe40f3bc24268d
FileAnalysis::TRIGGER_NEW
FILE_NEW
Ltd7QO7jEv3, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF_BUFFER
Version 4.9
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
Ltd7QO7jEv3, 10823, 0
[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
source: SMTP
SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665
MD5: a968bb0f9f9d95835b2e74c845877e87
SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665
SHA256: d5c4e7248840932b9d74ea2f3b3ae142c723a863abf5fd0599f9dd1171697e12

4
testing/btest/core/leaks/file-analysis-http-get.bro
View file

@ -8,7 +8,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

16
testing/btest/istate/events-ssl.bro
View file

@ -41,12 +41,16 @@ redef ssl_ca_certificate = "../ca_cert.pem";
redef ssl_private_key = "../bro.pem";
redef ssl_passphrase = "my-password";
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=10
# File-analysis fields in http.log won't get set on receiver side correctly,
# one problem is with the way serialization may send a unique ID in place
# of a full value and expect the remote side to associate that unique ID with
# a value it received at an earlier time. So sometimes modifications the sender
# makes to the value aren't seen on the receiver (in this case, the mime_type
# field).
event file_new(f: fa_file) &priority=10
{
FileAnalysis::stop(info$file_id);
delete f$mime_type;
FileAnalysis::stop(f);
}
@TEST-END-FILE
@ -63,7 +67,7 @@ event bro_init()
redef peer_description = "events-rcv";
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T, $retry=1sec]
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $ssl=T, $retry=1sec]
};
redef ssl_ca_certificate = "../ca_cert.pem";

15
testing/btest/istate/events.bro
View file

@ -36,12 +36,15 @@ redef peer_description = "events-send";
# it gets propagated but that's ok.)
redef tcp_close_delay = 0secs;
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=10
# File-analysis fields in http.log won't get set on receiver side correctly,
# one problem is with the way serialization may send a unique ID in place
# of a full value and expect the remote side to associate that unique ID with
# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type
# field).
event file_new(f: fa_file) &priority=10
{
FileAnalysis::stop(info$file_id);
delete f$mime_type;
FileAnalysis::stop(f);
}
@TEST-END-FILE
@ -58,7 +61,7 @@ event bro_init()
redef peer_description = "events-rcv";
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $retry=1sec]
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $retry=1sec]
};
event remote_connection_closed(p: event_peer)

13
testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro
View file

@ -5,25 +5,22 @@
global cnt: count = 0;
global timeout_cnt: count = 0;
redef FileAnalysis::default_timeout_interval=2sec;
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
local rval: string = fmt("%s-file%d", info$file_id, cnt);
local rval: string = fmt("%s-file%d", f$id, cnt);
++cnt;
return rval;
};
redef exit_only_after_terminate = T;
redef default_file_timeout_interval = 2sec;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
event file_timeout(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
if ( timeout_cnt < 1 )
FileAnalysis::postpone_timeout(info$file_id);
FileAnalysis::postpone_timeout(f);
else
terminate();
++timeout_cnt;

14
testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro
View file

@ -3,18 +3,16 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
event file_new(f: fa_file) &priority=-10
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
for ( act in test_file_actions )
FileAnalysis::remove_action(info$file_id, act);
local filename = test_get_file_name(info);
FileAnalysis::remove_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::remove_action(f, act);
local filename = test_get_file_name(f);
FileAnalysis::remove_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}

5
testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro
View file

@ -2,8 +2,7 @@
# @TEST-EXEC: btest-diff get.out
# @TEST-EXEC: test ! -s Cx92a0ym5R8-file
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
event file_new(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
FileAnalysis::stop(info$file_id);
FileAnalysis::stop(f);
}

2
testing/btest/scripts/base/frameworks/file-analysis/ftp.bro
View file

@ -4,7 +4,7 @@
redef test_file_analysis_source = "FTP_DATA";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return "thefile";
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/get.bro
View file

@ -7,7 +7,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro
View file

@ -17,9 +17,9 @@ global cnt: count = 0;
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
local rval: string = fmt("%s-file%d", info$file_id, cnt);
local rval: string = fmt("%s-file%d", f$id, cnt);
++cnt;
return rval;
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro
View file

@ -8,7 +8,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/post.bro
View file

@ -5,7 +5,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

9
testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro
View file

@ -5,9 +5,9 @@
redef exit_only_after_terminate = T;
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};
@TEST-START-FILE input.log
@ -42,10 +42,7 @@ event bro_init()
Input::remove("input");
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-10
event file_state_remove(f: fa_file) &priority=-10
{
if ( trig != FileAnalysis::TRIGGER_EOF ) return;
terminate();
}

2
testing/btest/scripts/base/frameworks/file-analysis/irc.bro
View file

@ -4,7 +4,7 @@
redef test_file_analysis_source = "IRC_DATA";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return "thefile";
};

4
testing/btest/scripts/base/frameworks/file-analysis/logging.bro
View file

@ -3,7 +3,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

2
testing/btest/scripts/base/frameworks/file-analysis/smtp.bro
View file

@ -8,7 +8,7 @@ redef test_file_analysis_source = "SMTP";
global mycnt: count = 0;
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
local rval: string = fmt("thefile%d", mycnt);
++mycnt;

2
testing/external/subdir-btest.cfg vendored
View file

@ -17,4 +17,4 @@ TRACES=%(testbase)s/Traces
SCRIPTS=%(testbase)s/../scripts
DIST=%(testbase)s/../../..
BUILD=%(testbase)s/../../../build
BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX
#BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX

128
testing/scripts/file-analysis-test.bro
View file

@ -3,105 +3,101 @@ global test_file_analysis_source: string = "" &redef;
global test_file_actions: set[FileAnalysis::ActionArgs];
global test_get_file_name: function(info: FileAnalysis::Info): string =
function(info: FileAnalysis::Info): string { return ""; } &redef;
global test_get_file_name: function(f: fa_file): string =
function(f: fa_file): string { return ""; } &redef;
global test_print_file_data_events: bool = F &redef;
event file_chunk(info: FileAnalysis::Info, data: string, off: count)
event file_chunk(f: fa_file, data: string, off: count)
{
if ( test_print_file_data_events )
print "file_chunk", info$file_id, |data|, off, data;
print "file_chunk", f$id, |data|, off, data;
}
event file_stream(info: FileAnalysis::Info, data: string)
event file_stream(f: fa_file, data: string)
{
if ( test_print_file_data_events )
print "file_stream", info$file_id, |data|, data;
print "file_stream", f$id, |data|, data;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
event file_new(f: fa_file)
{
print trig;
print "FILE_NEW";
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
print f$id, f$seen_bytes, f$missing_bytes;
if ( test_file_analysis_source == "" ||
info$source == test_file_analysis_source )
f$source == test_file_analysis_source )
{
for ( act in test_file_actions )
FileAnalysis::add_action(info$file_id, act);
FileAnalysis::add_action(f, act);
local filename: string = test_get_file_name(info);
local filename: string = test_get_file_name(f);
if ( filename != "" )
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_DATA_EVENT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
if ( f?$bof_buffer )
{
print "FILE_BOF_BUFFER";
print f$bof_buffer[0:10];
}
case FileAnalysis::TRIGGER_TYPE:
if ( f?$file_type || f?$mime_type )
print "FILE_TYPE";
# not actually printing the values due to libmagic variances
if ( info?$file_type )
if ( f?$file_type )
{
print "file type is set";
if ( info?$mime_type )
f$file_type = "set";
}
if ( f?$mime_type )
{
print "mime type is set";
break;
f$mime_type = "set";
}
}
case FileAnalysis::TRIGGER_EOF:
fallthrough;
case FileAnalysis::TRIGGER_DONE:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info?$conns )
for ( cid in info$conns )
event file_over_new_connection(f: fa_file)
{
print "FILE_OVER_NEW_CONNECTION";
}
event file_timeout(f: fa_file)
{
print "FILE_TIMEOUT";
}
event file_gap(f: fa_file)
{
print "FILE_GAP";
}
event file_state_remove(f: fa_file)
{
print "FILE_STATE_REMOVE";
print f$id, f$seen_bytes, f$missing_bytes;
if ( f?$conns )
for ( cid in f$conns )
print cid;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
if ( f?$total_bytes )
print "total bytes: " + fmt("%s", f$total_bytes);
if ( f?$source )
print "source: " + f$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
if ( info$actions[act]?$md5 )
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
if ( info$actions[act]?$sha1 )
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
if ( info$actions[act]?$sha256 )
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
if ( ! f?$info ) return;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
# avoids libmagic variances across systems
if ( info?$mime_type )
info$mime_type = "set";
if ( info?$file_type )
info$file_type = "set";
if ( f$info?$md5 )
print fmt("MD5: %s", f$info$md5);
if ( f$info?$sha1 )
print fmt("SHA1: %s", f$info$sha1);
if ( f$info?$sha256 )
print fmt("SHA256: %s", f$info$sha256);
}
event bro_init()