Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-17 22:18:20 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

FileAnalysis: checkpoint in middle of big reorganization.

Browse source 
- FileAnalysis::Info is now just a record used for logging, the fa_file
  record type is defined in init-bare.bro as the analogue to a
  connection record.

- Starting to transfer policy hook triggers and analyzer results to
  events.
This commit is contained in:
Jon Siwek 2013-04-09 15:49:58 -05:00
parent e73a261262
commit 641154f8e8
68 changed files with 855 additions and 871 deletions
Download patch file Download diff file Expand all files Collapse all files

221
scripts/base/frameworks/file-analysis/main.bro
View file

@ -12,49 +12,40 @@ export {
LOG
};
## The default buffer size used for storing the beginning of files.
const default_bof_buffer_size: count = 1024 &redef;
## The default amount of time file analysis will wait for new file data
## before giving up.
const default_timeout_interval: interval = 2 mins &redef;
## A structure which represents a desired file analysis action to take.
type ActionArgs: record {
## The type of action.
act: Action;
## The local filename to which to write an extracted file. Must be
## set when *act* is :bro:see:`FileAnalysis::ACTION_EXTRACT`.
extract_filename: string &optional;
};
## A structure which contains the results of certain file analysis actions.
type ActionResults: record {
## An MD5 digest of the file contents.
md5: string &optional;
## An SHA1 digest of the file contents.
sha1: string &optional;
## An SHA256 digest of the file contents.
sha256: string &optional;
};
## An event which will be generated for all new file contents,
## chunk-wise.
chunk_event: event(f: fa_file, data: string, off: count) &optional;
## An event which will be generated for all new file contents,
## stream-wise.
stream_event: event(f: fa_file, data: string) &optional;
} &redef;
## Contains all metadata related to the analysis of a given file.
## For the most part, fields here are derived from ones of the same name
## in :bro:see:`fa_file`.
type Info: record {
## An identifier associated with a single file.
file_id: string &log;
id: string &log;
## Identifier associated with a container file from which this one was
## extracted as part of the file analysis.
parent_file_id: string &log &optional;
parent_id: string &log &optional;
## An identification of the source of the file data. E.g. it may be
## a network protocol over which it was transferred, or a local file
## path which was read, or some other input source.
source: string &log &optional;
## The set of connections over which the file was transferred.
conns: table[conn_id] of connection &optional;
## The time at which the last activity for the file was seen.
last_active: time &log;
@ -75,15 +66,11 @@ export {
## The amount of time between receiving new data for this file that
## the analysis engine will wait before giving up on it.
timeout_interval: interval &log &default=default_timeout_interval;
timeout_interval: interval &log &optional;
## The number of bytes at the beginning of a file to save for later
## inspection in *bof_buffer* field.
bof_buffer_size: count &log &default=default_bof_buffer_size;
## The content of the beginning of a file up to *bof_buffer_size* bytes.
## This is also the buffer that's used for file/mime type detection.
bof_buffer: string &optional;
bof_buffer_size: count &log &optional;
## A file type provided by libmagic against the *bof_buffer*, or
## in the cases where no buffering of the beginning of file occurs,
@ -95,25 +82,17 @@ export {
## an initial guess of the mime type based on the first data seen.
mime_type: string &log &optional;
## Actions that have been added to the analysis of this file.
## Only meant for inspection by user scripts, not direct modification.
actions: table[ActionArgs] of ActionResults;
} &redef;
## Fields that are derived from existing ones, and are set just in time
## for logging purposes.
redef record FileAnalysis::Info += {
## Whether the file analysis timed out at least once for the file.
timedout: bool &log &default=F;
## Connection UIDS over which the file was transferred.
conn_uids: set[string] &log &optional;
conn_uids: set[string] &log;
## A set of action types taken during the file analysis.
actions_taken: set[Action] &log &optional;
actions_taken: set[Action] &log;
## Local filenames of file extraction actions.
extracted_files: set[string] &log &optional;
extracted_files: set[string] &log;
## An MD5 digest of the file contents.
md5: string &log &optional;
@ -123,25 +102,13 @@ export {
## A SHA256 digest of the file contents.
sha256: string &log &optional;
};
## Redefined here just so the *info* parameters of the events have the
## right type information.
redef record ActionArgs += {
## An event which will be generated for all new file contents,
## chunk-wise.
chunk_event: event(info: Info, data: string, off: count) &optional;
## An event which will be generated for all new file contents,
## stream-wise.
stream_event: event(info: Info, data: string) &optional;
};
} &redef;
## Evaluated every time a significant event occurs during the course of
## file analysis. Fields of the *info* argument may be modified or
## other actions may be added or removed inside the body of any handlers
## of this hook.
global policy: hook(trig: Trigger, info: Info);
global policy: hook(trig: Trigger, f: fa_file);
## A table that can be used to disable file analysis completely for
## any files transferred over given network protocol analyzers.
@ -153,7 +120,7 @@ export {
## The salt concatenated to unique file handle strings generated by
## :bro:see:`get_file_handle` before hashing them in to a file id
## (the *file_id* field of :bro:see:`FileAnalysis::Info`).
## (the *id* field of :bro:see:`fa_file`).
## Provided to help mitigate the possiblility of manipulating parts of
## network connections that factor in to the file handle in order to
## generate two handles that would hash to the same file id.
@ -163,47 +130,43 @@ export {
## When used within a :bro:see:`FileAnalysis::policy` handler for
## :bro:see:`FileAnalysis::TRIGGER_TIMEOUT`, the analysis will delay
## timing out for the period of time indicated by the *timeout_interval*
## field of :bro:see:`FileAnalysis::Info`.
## field of :bro:see:`fa_file`.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## Returns: true if the timeout will be postponed, or false if analysis
## for the *file_id* isn't currently active.
global postpone_timeout: function(file_id: string): bool;
## for the *id* isn't currently active.
global postpone_timeout: function(f: fa_file): bool;
## Adds an action to the analysis of a given file.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## args: the action type to add along with any arguments it takes.
##
## Returns: true if the action will be added, or false if analysis
## for the *file_id* isn't currently active or the *args*
## for the *id* isn't currently active or the *args*
## were invalid for the action type.
global add_action: function(file_id: string, args: ActionArgs): bool;
global add_action: function(f: fa_file, args: ActionArgs): bool;
## Removes an action from the analysis of a given file.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## args: the action (type and args) to remove.
##
## Returns: true if the action will be removed, or false if analysis
## for the *file_id* isn't currently active.
global remove_action: function(file_id: string, args: ActionArgs): bool;
## for the *id* isn't currently active.
global remove_action: function(f: fa_file, args: ActionArgs): bool;
## Stops/ignores any further analysis of a given file.
##
## file_id: the file identifier string from the *file_id* field of
## :bro:see:`FileAnalysis::Info`.
## f: the file.
##
## Returns: true if analysis for the given file will be ignored for the
## rest of it's contents, or false if analysis for the *file_id*
## rest of it's contents, or false if analysis for the *id*
## isn't currently active.
global stop: function(file_id: string): bool;
global stop: function(f: fa_file): bool;
## Sends a sequential stream of data in for file analysis.
## Meant for use when providing external file analysis input (e.g.
@ -258,24 +221,61 @@ export {
global eof: function(source: string);
}
function postpone_timeout(file_id: string): bool
redef record fa_file += {
info: Info &optional;
};
function set_info(f: fa_file)
{
return __postpone_timeout(file_id);
if ( ! f?$info )
{
local tmp: Info;
f$info = tmp;
}
function add_action(file_id: string, args: ActionArgs): bool
{
return __add_action(file_id, args);
f$info$id = f$id;
if ( f?$parent_id ) f$info$parent_id = f$parent_id;
if ( f?$source ) f$info$source = f$source;
f$info$last_active = f$last_active;
f$info$seen_bytes = f$seen_bytes;
if ( f?$total_bytes ) f$info$total_bytes = f$total_bytes;
f$info$missing_bytes = f$missing_bytes;
f$info$overflow_bytes = f$overflow_bytes;
f$info$timeout_interval = f$timeout_interval;
f$info$bof_buffer_size = f$bof_buffer_size;
if ( f?$file_type ) f$info$file_type = f$file_type;
if ( f?$mime_type ) f$info$mime_type = f$mime_type;
if ( f?$conns )
for ( cid in f$conns )
add f$info$conn_uids[f$conns[cid]$uid];
}
function remove_action(file_id: string, args: ActionArgs): bool
function postpone_timeout(f: fa_file): bool
{
return __remove_action(file_id, args);
return __postpone_timeout(f$id);
}
function stop(file_id: string): bool
function add_action(f: fa_file, args: ActionArgs): bool
{
return __stop(file_id);
if ( ! __add_action(f$id, args) ) return F;
set_info(f);
add f$info$actions_taken[args$act];
if ( args$act == FileAnalysis::ACTION_EXTRACT )
add f$info$extracted_files[args$extract_filename];
return T;
}
function remove_action(f: fa_file, args: ActionArgs): bool
{
return __remove_action(f$id, args);
}
function stop(f: fa_file): bool
{
return __stop(f$id);
}
function data_stream(source: string, data: string)
@ -309,53 +309,36 @@ event bro_init() &priority=5
[$columns=Info, $ev=log_file_analysis]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
info$timedout = T;
set_info(f);
f$info$timedout = T;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-5
event file_hash(f: fa_file, kind: string, hash: string) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_EOF &&
trig != FileAnalysis::TRIGGER_DONE ) return;
set_info(f);
switch ( kind ) {
case "md5":
f$info$md5 = hash;
break;
case "sha1":
f$info$sha1 = hash;
break;
case "sha256":
f$info$sha256 = hash;
break;
}
}
info$conn_uids = set();
if ( info?$conns )
for ( cid in info$conns )
add info$conn_uids[info$conns[cid]$uid];
info$actions_taken = set();
info$extracted_files = set();
for ( act in info$actions )
event file_state_remove(f: fa_file) &priority=5
{
add info$actions_taken[act$act];
local result: FileAnalysis::ActionResults = info$actions[act];
switch ( act$act ) {
case FileAnalysis::ACTION_EXTRACT:
add info$extracted_files[act$extract_filename];
break;
case FileAnalysis::ACTION_MD5:
if ( result?$md5 )
info$md5 = result$md5;
break;
case FileAnalysis::ACTION_SHA1:
if ( result?$sha1 )
info$sha1 = result$sha1;
break;
case FileAnalysis::ACTION_SHA256:
if ( result?$sha256 )
info$sha256 = result$sha256;
break;
case FileAnalysis::ACTION_DATA_EVENT:
# no direct result
break;
}
set_info(f);
}
Log::write(FileAnalysis::LOG, info);
event file_state_remove(f: fa_file) &priority=-5
{
Log::write(FileAnalysis::LOG, f$info);
}

61
scripts/base/init-bare.bro
View file

@ -316,6 +316,67 @@ type connection: record {
tunnel: EncapsulatingConnVector &optional;
};
## A file that Bro is analyzing. This is Bro's type for describing the basic
## internal metadata collected about a "file", which is essentially just a
## byte stream that is e.g. pulled from a network connection or possibly
## some other input source.
type fa_file: record {
## An identifier associated with a single file.
id: string;
## Identifier associated with a container file from which this one was
## extracted as part of the file analysis.
parent_id: string &optional;
## An identification of the source of the file data. E.g. it may be
## a network protocol over which it was transferred, or a local file
## path which was read, or some other input source.
source: string &optional;
## The set of connections over which the file was transferred.
conns: table[conn_id] of connection &optional;
## The time at which the last activity for the file was seen.
last_active: time;
## Number of bytes provided to the file analysis engine for the file.
seen_bytes: count &default=0;
## Total number of bytes that are supposed to comprise the full file.
total_bytes: count &optional;
## The number of bytes in the file stream that were completely missed
## during the process of analysis e.g. due to dropped packets.
missing_bytes: count &default=0;
## The number of not all-in-sequence bytes in the file stream that
## were delivered to file actions/analyzers due to reassembly buffer
## overflow.
overflow_bytes: count &default=0;
## The amount of time between receiving new data for this file that
## the analysis engine will wait before giving up on it.
timeout_interval: interval &default=2mins;
## The number of bytes at the beginning of a file to save for later
## inspection in *bof_buffer* field.
bof_buffer_size: count &default=1024;
## The content of the beginning of a file up to *bof_buffer_size* bytes.
## This is also the buffer that's used for file/mime type detection.
bof_buffer: string &optional;
## A file type provided by libmagic against the *bof_buffer*, or
## in the cases where no buffering of the beginning of file occurs,
## an initial guess of the file type based on the first data seen.
file_type: string &optional;
## A mime type provided by libmagic against the *bof_buffer*, or
## in the cases where no buffering of the beginning of file occurs,
## an initial guess of the mime type based on the first data seen.
mime_type: string &optional;
} &redef;
## Fields of a SYN packet.
##
## .. bro:see:: connection_SYN_packet

57
scripts/base/protocols/ftp/file-extract.bro
View file

@ -24,21 +24,21 @@ redef record Info += {
extract_file: bool &default=F;
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "FTP_DATA" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;
if ( ! f?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in ftp_data_expected ) next;
@ -48,8 +48,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
@ -57,48 +56,44 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "FTP_DATA" ) return;
if ( extract_file_types !in info$mime_type ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;
if ( extract_file_types !in f$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-5
event file_state_remove(f: fa_file) &priority=4
{
if ( trig != FileAnalysis::TRIGGER_EOF &&
trig != FileAnalysis::TRIGGER_DONE ) return;
if ( ! info?$source ) return;
if ( info$source != "FTP_DATA" ) return;
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;
if ( ! f?$info ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT )
for ( filename in f$info$extracted_files )
{
local s: FTP::Info;
s$ts = network_time();
s$tags = set();
s$user = "<ftp-data>";
s$extraction_file = act$extract_filename;
s$extraction_file = filename;
if ( info?$conns )
for ( cid in info$conns )
if ( f?$conns )
for ( cid in f$conns )
{
s$uid = info$conns[cid]$uid;
s$uid = f$conns[cid]$uid;
s$id = cid;
break;
}
Log::write(FTP::LOG, s);

41
scripts/base/protocols/http/file-extract.bro
View file

@ -26,29 +26,29 @@ export {
global extract_count: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( extract_file_types !in info$mime_type ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( extract_file_types !in f$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
@ -56,21 +56,21 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
@ -78,8 +78,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;

58
scripts/base/protocols/http/file-hash.bro
View file

@ -23,67 +23,47 @@ export {
&redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( generate_md5 in info$mime_type )
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
else if ( info?$conns )
if ( generate_md5 in f$mime_type )
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
else if ( f?$conns )
{
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
if ( c$http$calc_md5 )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
return;
}
}
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_state_remove(f: fa_file) &priority=4
{
if ( trig != FileAnalysis::TRIGGER_DONE &&
trig != FileAnalysis::TRIGGER_EOF ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
if ( ! f?$info ) return;
if ( ! f$info?$md5 ) return;
local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5];
if ( act !in info$actions ) return;
local result = info$actions[act];
if ( ! result?$md5 ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
c$http$md5 = result$md5;
c$http$md5 = f$info$md5;
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_GAP ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
FileAnalysis::remove_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
}

34
scripts/base/protocols/http/file-ident.bro
View file

@ -34,24 +34,24 @@ export {
const ignored_incorrect_file_type_urls = /^$/ &redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
c$http$mime_type = info$mime_type;
c$http$mime_type = f$mime_type;
local mime_str: string = split1(info$mime_type, /;/)[1];
local mime_str: string = split1(f$mime_type, /;/)[1];
if ( mime_str !in mime_types_extensions ) next;
if ( ! c$http?$uri ) next;
@ -68,26 +68,26 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW_CONN ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "HTTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
# Spread the mime around (e.g. for partial content, TRIGGER_TYPE only
# happens once for the first connection, but if there's subsequent
# connections to transfer the same file, they'll be lacking the mime_type
# field if we don't do this).
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$http ) next;
c$http$mime_type = info$mime_type;
c$http$mime_type = f$mime_type;
}
}

73
scripts/base/protocols/irc/dcc-send.bro
View file

@ -41,21 +41,21 @@ global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
global extract_count: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "IRC_DATA" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
if ( ! f?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
@ -65,8 +65,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
@ -76,29 +75,29 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
}
function set_dcc_mime(info: FileAnalysis::Info)
function set_dcc_mime(f: fa_file)
{
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
local s = dcc_expected_transfers[cid$resp_h, cid$resp_p];
s$dcc_mime_type = info$mime_type;
s$dcc_mime_type = f$mime_type;
}
}
function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
function set_dcc_extraction_file(f: fa_file, filename: string)
{
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
@ -108,13 +107,13 @@ function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string)
}
}
function log_dcc(info: FileAnalysis::Info)
function log_dcc(f: fa_file)
{
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next;
@ -137,37 +136,37 @@ function log_dcc(info: FileAnalysis::Info)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "IRC_DATA" ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
set_dcc_mime(info);
set_dcc_mime(f);
if ( extract_file_types !in info$mime_type ) return;
if ( extract_file_types !in f$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
set_dcc_extraction_file(info, fname);
set_dcc_extraction_file(f, fname);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$source ) return;
if ( info$source != "IRC_DATA" ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
log_dcc(info);
log_dcc(f);
}
event irc_dcc_message(c: connection, is_orig: bool,

26
scripts/base/protocols/smtp/entities-excerpt.bro
View file

@ -16,34 +16,34 @@ export {
const default_entity_excerpt_len = 0 &redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( default_entity_excerpt_len > info$bof_buffer_size )
info$bof_buffer_size = default_entity_excerpt_len;
if ( default_entity_excerpt_len > f$bof_buffer_size )
f$bof_buffer_size = default_entity_excerpt_len;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_BOF_BUFFER ) return;
if ( ! info?$bof_buffer ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$bof_buffer ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( default_entity_excerpt_len > 0 )
c$smtp$current_entity$excerpt =
info$bof_buffer[0:default_entity_excerpt_len];
f$bof_buffer[0:default_entity_excerpt_len];
}
}

116
scripts/base/protocols/smtp/entities.bro
View file

@ -95,21 +95,21 @@ event mime_begin_entity(c: connection) &priority=10
set_session(c, T);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$conns ) return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
local extracting: bool = F;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
@ -118,8 +118,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
if ( ! extracting )
{
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
extracting = T;
++extract_count;
@ -129,29 +128,28 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
}
if ( c$smtp$current_entity$calc_md5 )
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
}
}
function check_extract_by_type(info: FileAnalysis::Info)
function check_extract_by_type(f: fa_file)
{
if ( extract_file_types !in info$mime_type ) return;
if ( extract_file_types !in f$mime_type ) return;
for ( act in info$actions )
if ( act$act == FileAnalysis::ACTION_EXTRACT ) return;
if ( f?$info && FileAnalysis::ACTION_EXTRACT in f$info$actions_taken )
return;
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id,
local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, f$id,
extract_count);
++extract_count;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=fname]);
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
@ -159,86 +157,56 @@ function check_extract_by_type(info: FileAnalysis::Info)
}
}
function check_md5_by_type(info: FileAnalysis::Info)
function check_md5_by_type(f: fa_file)
{
if ( never_calc_md5 ) return;
if ( generate_md5 !in info$mime_type ) return;
if ( generate_md5 !in f$mime_type ) return;
FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_MD5]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! info?$mime_type ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( info?$conns )
for ( cid in info$conns )
if ( f?$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
c$smtp$current_entity$mime_type = info$mime_type;
c$smtp$current_entity$mime_type = f$mime_type;
}
check_extract_by_type(info);
check_md5_by_type(info);
check_extract_by_type(f);
check_md5_by_type(f);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
event file_state_remove(f: fa_file) &priority=4
{
if ( trig != FileAnalysis::TRIGGER_GAP ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
# Only log if there was some content.
if ( f$seen_bytes == 0 ) next;
FileAnalysis::remove_action(info$file_id,
[$act=FileAnalysis::ACTION_MD5]);
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_EOF &&
trig != FileAnalysis::TRIGGER_DONE ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! info?$conns ) return;
for ( cid in info$conns )
{
local c: connection = info$conns[cid];
if ( ! c?$smtp ) next;
if ( ! c$smtp?$current_entity ) next;
# Only log is there was some content.
if ( info$seen_bytes == 0 ) next;
local act: FileAnalysis::ActionArgs = [$act=FileAnalysis::ACTION_MD5];
if ( act in info$actions )
{
local result = info$actions[act];
if ( result?$md5 )
c$smtp$current_entity$md5 = result$md5;
}
c$smtp$current_entity$content_len = info$seen_bytes;
if ( f?$info && f$info?$md5 )
c$smtp$current_entity$md5 = f$info$md5;
c$smtp$current_entity$content_len = f$seen_bytes;
Log::write(SMTP::ENTITIES_LOG, c$smtp$current_entity);
delete c$smtp$current_entity;
return;

17
scripts/policy/frameworks/intel/smtp-url-extraction.bro
View file

@ -3,13 +3,13 @@
@load base/utils/urls
@load ./where-locations
event intel_mime_data(info: FileAnalysis::Info, data: string)
event intel_mime_data(f: fa_file, data: string)
{
if ( ! info?$conns ) return;
if ( ! f?$conns ) return;
for ( cid in info$conns )
for ( cid in f$conns )
{
local c: connection = info$conns[cid];
local c: connection = f$conns[cid];
local urls = find_all_urls_without_scheme(data);
for ( url in urls )
{
@ -21,14 +21,13 @@ event intel_mime_data(info: FileAnalysis::Info, data: string)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! info?$source ) return;
if ( info$source != "SMTP" ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_DATA_EVENT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$stream_event=intel_mime_data]);
}

4
src/CMakeLists.txt
View file

@ -450,8 +450,8 @@ set(bro_SRCS
input/readers/Binary.cc
file_analysis/Manager.cc
file_analysis/Info.cc
file_analysis/InfoTimer.cc
file_analysis/File.cc
file_analysis/FileTimer.cc
file_analysis/PendingFile.cc
file_analysis/FileID.h
file_analysis/Action.h

1
src/Event.cc
View file

@ -62,6 +62,7 @@ EventMgr::EventMgr()
current_aid = 0;
src_val = 0;
draining = 0;
bypass_queue = false;
}
EventMgr::~EventMgr()

1
src/Event.h
View file

@ -128,6 +128,7 @@ protected:
TimerMgr* current_mgr;
RecordVal* src_val;
bool draining;
bool bypass_queue;
};
extern EventMgr mgr;

2
src/NetVar.cc
View file

@ -9,6 +9,7 @@ RecordType* conn_id;
RecordType* endpoint;
RecordType* endpoint_stats;
RecordType* connection_type;
RecordType* fa_file_type;
RecordType* icmp_conn;
RecordType* icmp_context;
RecordType* SYN_packet;
@ -314,6 +315,7 @@ void init_net_var()
endpoint = internal_type("endpoint")->AsRecordType();
endpoint_stats = internal_type("endpoint_stats")->AsRecordType();
connection_type = internal_type("connection")->AsRecordType();
fa_file_type = internal_type("fa_file")->AsRecordType();
icmp_conn = internal_type("icmp_conn")->AsRecordType();
icmp_context = internal_type("icmp_context")->AsRecordType();
signature_state = internal_type("signature_state")->AsRecordType();

1
src/NetVar.h
View file

@ -12,6 +12,7 @@ extern RecordType* conn_id;
extern RecordType* endpoint;
extern RecordType* endpoint_stats;
extern RecordType* connection_type;
extern RecordType* fa_file_type;
extern RecordType* icmp_conn;
extern RecordType* icmp_context;
extern RecordType* signature_state;

16
src/bro.bif
View file

@ -5538,22 +5538,6 @@ function match_signatures%(c: connection, pattern_type: int, s: string,
return new Val(1, TYPE_BOOL);
%}
## For use within a :bro:see:`get_file_handle` handler to return a unique
## identifier to associate with some buffered input to the file analysis
## framework. The buffered data will then immediately be allowed to pass
## pass through the file analysis framework and execute any policy hooks
## that are available. If an empty string is returned, that signifies that
## the buffered data will be discarded with no further action taken on it.
##
## handle: A string that uniquely identifies a file.
##
## .. bro:see:: get_file_handle FileAnalysis::policy
function return_file_handle%(handle: string%): any
%{
file_mgr->ReceiveHandle(handle->CheckString());
return 0;
%}
# ===========================================================================
#
# Deprecated Functions

18
src/event.bif
View file

@ -6997,6 +6997,24 @@ event bro_script_loaded%(path: string, level: count%);
## .. bro:see:: return_file_handle
event get_file_handle%(tag: count, c: connection, is_orig: bool%);
## This event is generated each time file analysis is ending for a given file.
##
## f: The file.
event file_state_remove%(f: fa_file%);
## This event is generated each time file analysis generates a digest of the
## file contents.
##
## f: The file.
##
## kind: The type of digest algorithm.
##
## hash: The result of the hashing.
##
## .. bro:see:: FileAnalysis::add_action FileAnalysis::ACTION_MD5
## FileAnalysis::ACTION_SHA1 FileAnalysis::ACTION_SHA256
event file_hash%(f: fa_file, kind: string, hash: string%);
## Deprecated. Will be removed.
event stp_create_endp%(c: connection, e: int, is_orig: bool%);

86
src/file_analysis.bif
View file

@ -6,74 +6,6 @@ module FileAnalysis;
#include "file_analysis/Manager.h"
%%}
type Info: record;
type ActionArgs: record;
type ActionResults: record;
## An enumeration of significant things that can occur over the course of
## analyzing files. The :bro:see:`FileAnalysis::policy` hook is called each
## time a trigger occurs.
enum Trigger %{
## Raised when any part of a new file is detected.
TRIGGER_NEW,
## Raised when file is detected being transported over a new network
## connection (other than the first).
TRIGGER_NEW_CONN,
## Raised when file analysis has likely seen a complete file. That
## is when a number of bytes indicated by the *total_bytes* field of
## :bro:see:`FileAnalysis::Info` have been processed.
TRIGGER_DONE,
## Raised when file analysis for a given file is aborted due
## to not seeing any data for it recently. Note that this doesn't
## necessarily mean the full file wasn't seen (e.g. if the
## :bro:see:`FileAnalysis::Info` record indicates the file *total_bytes*
## isn't known). Use :bro:see:`FileAnalysis::postpone_timeout`
## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
## defer the timeout until later.
TRIGGER_TIMEOUT,
## Raised when the beginning of a file is detected.
TRIGGER_BOF,
## Raised when the beginning of a file is available in the *bof_buffer*
## field of :bro:see:`FileAnalysis::Info` and that beginning
## is at least the number of bytes indicated by the *bof_buffer_size* field.
TRIGGER_BOF_BUFFER,
## Raised when an initial guess at the file/mime type of a file is matched.
TRIGGER_TYPE,
## Raised to signal that no more file data is incoming and it couldn't be
## determined whether the full file was actually seen and analyzed.
TRIGGER_EOF,
## Raised when there's a missing chunk of data in the file stream.
TRIGGER_GAP,
%}
## An enumeration of various file analysis actions that can be taken.
enum Action %{
## Extract a file to local filesystem
ACTION_EXTRACT,
## Calculate an MD5 digest of the file's contents.
ACTION_MD5,
## Calculate an SHA1 digest of the file's contents.
ACTION_SHA1,
## Calculate an SHA256 digest of the file's contents.
ACTION_SHA256,
## Deliver the file contents to the script-layer in an event.
ACTION_DATA_EVENT,
%}
## :bro:see:`FileAnalysis::postpone_timeout`.
function FileAnalysis::__postpone_timeout%(file_id: string%): bool
%{
@ -147,3 +79,21 @@ function FileAnalysis::__eof%(source: string%): any
file_mgr->EndOfFile(source->CheckString());
return 0;
%}
module GLOBAL;
## For use within a :bro:see:`get_file_handle` handler to return a unique
## identifier to associate with some buffered input to the file analysis
## framework. The buffered data will then immediately be allowed to pass
## pass through the file analysis framework and execute any policy hooks
## that are available. If an empty string is returned, that signifies that
## the buffered data will be discarded with no further action taken on it.
##
## handle: A string that uniquely identifies a file.
##
## .. bro:see:: get_file_handle FileAnalysis::policy
function return_file_handle%(handle: string%): any
%{
file_mgr->ReceiveHandle(handle->CheckString());
return 0;
%}

16
src/file_analysis/Action.h
View file

@ -8,10 +8,10 @@ namespace file_analysis {
typedef BifEnum::FileAnalysis::Action ActionTag;
class Info;
class File;
/**
* Base class for actions that can be attached to a file_analysis::Info object.
* Base class for actions that can be attached to a file_analysis::File object.
*/
class Action {
public:
@ -67,9 +67,9 @@ public:
RecordVal* Args() const { return args; }
/**
* @return the file_analysis::Info object to which the action is attached.
* @return the file_analysis::File object to which the action is attached.
*/
Info* GetInfo() const { return info; }
File* GetFile() const { return file; }
/**
* @return the action tag equivalent of the 'act' field from the ActionArgs
@ -84,17 +84,17 @@ public:
protected:
Action(RecordVal* arg_args, Info* arg_info)
Action(RecordVal* arg_args, File* arg_file)
: tag(Action::ArgsTag(arg_args)), args(arg_args->Ref()->AsRecordVal()),
info(arg_info)
file(arg_file)
{}
ActionTag tag;
RecordVal* args;
Info* info;
File* file;
};
typedef Action* (*ActionInstantiator)(RecordVal* args, Info* info);
typedef Action* (*ActionInstantiator)(RecordVal* args, File* file);
} // namespace file_analysis

24
src/file_analysis/ActionSet.cc
View file

@ -1,5 +1,5 @@
#include "ActionSet.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
#include "Extract.h"
#include "DataEvent.h"
@ -21,7 +21,7 @@ static void action_del_func(void* v)
delete (Action*) v;
}
ActionSet::ActionSet(Info* arg_info) : info(arg_info)
ActionSet::ActionSet(File* arg_file) : file(arg_file)
{
TypeList* t = new TypeList();
t->Append(BifType::Record::FileAnalysis::ActionArgs->Ref());
@ -50,7 +50,7 @@ bool ActionSet::AddAction(RecordVal* args)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d skipped for file id"
" %s: already exists", Action::ArgsTag(args),
info->GetFileID().c_str());
file->GetID().c_str());
delete key;
return true;
}
@ -90,7 +90,7 @@ bool ActionSet::Add::Perform(ActionSet* set)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d skipped for file id"
" %s: already exists", act->Tag(),
act->GetInfo()->GetFileID().c_str());
act->GetFile()->GetID().c_str());
Abort();
return true;
}
@ -112,12 +112,12 @@ bool ActionSet::RemoveAction(ActionTag tag, HashKey* key)
if ( ! act )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Skip remove action %d for file id %s",
tag, info->GetFileID().c_str());
tag, file->GetID().c_str());
return false;
}
DBG_LOG(DBG_FILE_ANALYSIS, "Remove action %d for file id %s", act->Tag(),
info->GetFileID().c_str());
file->GetID().c_str());
delete act;
return true;
}
@ -147,12 +147,12 @@ HashKey* ActionSet::GetKey(const RecordVal* args) const
Action* ActionSet::InstantiateAction(RecordVal* args) const
{
Action* act = action_factory[Action::ArgsTag(args)](args, info);
Action* act = action_factory[Action::ArgsTag(args)](args, file);
if ( ! act )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Instantiate action %d failed for file id",
" %s", Action::ArgsTag(args), info->GetFileID().c_str());
" %s", Action::ArgsTag(args), file->GetID().c_str());
return 0;
}
@ -162,11 +162,9 @@ Action* ActionSet::InstantiateAction(RecordVal* args) const
void ActionSet::InsertAction(Action* act, HashKey* key)
{
DBG_LOG(DBG_FILE_ANALYSIS, "Add action %d for file id %s", act->Tag(),
info->GetFileID().c_str());
file->GetID().c_str());
action_map.Insert(key, act);
delete key;
info->GetVal()->Lookup(Info::actions_idx)->AsTableVal()->Assign(act->Args(),
new RecordVal(BifType::Record::FileAnalysis::ActionResults));
}
void ActionSet::DrainModifications()
@ -174,7 +172,7 @@ void ActionSet::DrainModifications()
if ( mod_queue.empty() ) return;
DBG_LOG(DBG_FILE_ANALYSIS, "Start flushing action mod queue of file id %s",
info->GetFileID().c_str());
file->GetID().c_str());
do
{
Modification* mod = mod_queue.front();
@ -183,5 +181,5 @@ void ActionSet::DrainModifications()
mod_queue.pop();
} while ( ! mod_queue.empty() );
DBG_LOG(DBG_FILE_ANALYSIS, "End flushing action mod queue of file id %s",
info->GetFileID().c_str());
file->GetID().c_str());
}

6
src/file_analysis/ActionSet.h
View file

@ -10,7 +10,7 @@
namespace file_analysis {
class Info;
class File;
declare(PDict,Action);
/**
@ -21,7 +21,7 @@ declare(PDict,Action);
class ActionSet {
public:
ActionSet(Info* arg_info);
ActionSet(File* arg_file);
~ActionSet();
@ -63,7 +63,7 @@ protected:
void InsertAction(Action* act, HashKey* key);
bool RemoveAction(ActionTag tag, HashKey* key);
Info* info;
File* file;
CompositeHash* action_hash; /**< ActionArgs hashes Action map lookup. */
PDict(Action) action_map; /**< Actions indexed by ActionArgs. */

16
src/file_analysis/DataEvent.cc
View file

@ -7,13 +7,13 @@
using namespace file_analysis;
DataEvent::DataEvent(RecordVal* args, Info* info,
DataEvent::DataEvent(RecordVal* args, File* file,
EventHandlerPtr ce, EventHandlerPtr se)
: Action(args, info), chunk_event(ce), stream_event(se)
: Action(args, file), chunk_event(ce), stream_event(se)
{
}
Action* DataEvent::Instantiate(RecordVal* args, Info* info)
Action* DataEvent::Instantiate(RecordVal* args, File* file)
{
using BifType::Record::FileAnalysis::ActionArgs;
@ -36,7 +36,7 @@ Action* DataEvent::Instantiate(RecordVal* args, Info* info)
if ( stream_val )
stream = event_registry->Lookup(stream_val->AsFunc()->Name());
return new DataEvent(args, info, chunk, stream);
return new DataEvent(args, file, chunk, stream);
}
bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
@ -44,10 +44,10 @@ bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
if ( ! chunk_event ) return true;
val_list* args = new val_list;
args->append(info->GetVal()->Ref());
args->append(file->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
args->append(new Val(offset, TYPE_COUNT));
mgr.QueueEvent(chunk_event, args);
mgr.Dispatch(new Event(chunk_event, args));
return true;
}
@ -57,9 +57,9 @@ bool DataEvent::DeliverStream(const u_char* data, uint64 len)
if ( ! stream_event ) return true;
val_list* args = new val_list;
args->append(info->GetVal()->Ref());
args->append(file->GetVal()->Ref());
args->append(new StringVal(new BroString(data, len, 0)));
mgr.QueueEvent(stream_event, args);
mgr.Dispatch(new Event(stream_event, args));
return true;
}

6
src/file_analysis/DataEvent.h
View file

@ -4,7 +4,7 @@
#include <string>
#include "Val.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
namespace file_analysis {
@ -15,7 +15,7 @@ namespace file_analysis {
class DataEvent : public Action {
public:
static Action* Instantiate(RecordVal* args, Info* info);
static Action* Instantiate(RecordVal* args, File* file);
virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
@ -23,7 +23,7 @@ public:
protected:
DataEvent(RecordVal* args, Info* info,
DataEvent(RecordVal* args, File* file,
EventHandlerPtr ce, EventHandlerPtr se);
EventHandlerPtr chunk_event;

8
src/file_analysis/Extract.cc
View file

@ -5,8 +5,8 @@
using namespace file_analysis;
Extract::Extract(RecordVal* args, Info* info, const string& arg_filename)
: Action(args, info), filename(arg_filename)
Extract::Extract(RecordVal* args, File* file, const string& arg_filename)
: Action(args, file), filename(arg_filename)
{
fd = open(filename.c_str(), O_WRONLY | O_CREAT | O_TRUNC, 0666);
@ -25,7 +25,7 @@ Extract::~Extract()
safe_close(fd);
}
Action* Extract::Instantiate(RecordVal* args, Info* info)
Action* Extract::Instantiate(RecordVal* args, File* file)
{
using BifType::Record::FileAnalysis::ActionArgs;
const char* field = "extract_filename";
@ -33,7 +33,7 @@ Action* Extract::Instantiate(RecordVal* args, Info* info)
if ( ! v ) return 0;
return new Extract(args, info, v->AsString()->CheckString());
return new Extract(args, file, v->AsString()->CheckString());
}
bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)

6
src/file_analysis/Extract.h
View file

@ -4,7 +4,7 @@
#include <string>
#include "Val.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
namespace file_analysis {
@ -15,7 +15,7 @@ namespace file_analysis {
class Extract : public Action {
public:
static Action* Instantiate(RecordVal* args, Info* info);
static Action* Instantiate(RecordVal* args, File* file);
virtual ~Extract();
@ -23,7 +23,7 @@ public:
protected:
Extract(RecordVal* args, Info* info, const string& arg_filename);
Extract(RecordVal* args, File* file, const string& arg_filename);
string filename;
int fd;

135
src/file_analysis/Info.cc → src/file_analysis/File.cc
View file

@ -1,14 +1,15 @@
#include <string>
#include <openssl/md5.h>
#include "Info.h"
#include "InfoTimer.h"
#include "File.h"
#include "FileTimer.h"
#include "FileID.h"
#include "Manager.h"
#include "Reporter.h"
#include "Val.h"
#include "Type.h"
#include "Analyzer.h"
#include "Event.h"
using namespace file_analysis;
@ -32,33 +33,32 @@ static RecordVal* get_conn_id_val(const Connection* conn)
return v;
}
int Info::file_id_idx = -1;
int Info::parent_file_id_idx = -1;
int Info::source_idx = -1;
int Info::conns_idx = -1;
int Info::last_active_idx = -1;
int Info::seen_bytes_idx = -1;
int Info::total_bytes_idx = -1;
int Info::missing_bytes_idx = -1;
int Info::overflow_bytes_idx = -1;
int Info::timeout_interval_idx = -1;
int Info::bof_buffer_size_idx = -1;
int Info::bof_buffer_idx = -1;
int Info::file_type_idx = -1;
int Info::mime_type_idx = -1;
int Info::actions_idx = -1;
int File::id_idx = -1;
int File::parent_id_idx = -1;
int File::source_idx = -1;
int File::conns_idx = -1;
int File::last_active_idx = -1;
int File::seen_bytes_idx = -1;
int File::total_bytes_idx = -1;
int File::missing_bytes_idx = -1;
int File::overflow_bytes_idx = -1;
int File::timeout_interval_idx = -1;
int File::bof_buffer_size_idx = -1;
int File::bof_buffer_idx = -1;
int File::file_type_idx = -1;
int File::mime_type_idx = -1;
magic_t Info::magic = 0;
magic_t Info::magic_mime = 0;
magic_t File::magic = 0;
magic_t File::magic_mime = 0;
string Info::salt;
string File::salt;
void Info::StaticInit()
void File::StaticInit()
{
if ( file_id_idx != -1 ) return;
if ( id_idx != -1 ) return;
file_id_idx = Idx("file_id");
parent_file_id_idx = Idx("parent_file_id");
id_idx = Idx("id");
parent_id_idx = Idx("parent_id");
source_idx = Idx("source");
conns_idx = Idx("conns");
last_active_idx = Idx("last_active");
@ -71,7 +71,6 @@ void Info::StaticInit()
bof_buffer_idx = Idx("bof_buffer");
file_type_idx = Idx("file_type");
mime_type_idx = Idx("mime_type");
actions_idx = Idx("actions");
bro_init_magic(&magic, MAGIC_NONE);
bro_init_magic(&magic_mime, MAGIC_MIME);
@ -79,26 +78,26 @@ void Info::StaticInit()
salt = BifConst::FileAnalysis::salt->CheckString();
}
Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
: file_id(""), unique(unique), val(0), postpone_timeout(false),
File::File(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
: id(""), unique(unique), val(0), postpone_timeout(false),
first_chunk(true), need_type(false), need_reassembly(false), done(false),
actions(this)
{
StaticInit();
char id[20];
char tmp[20];
uint64 hash[2];
string msg(unique + salt);
MD5(reinterpret_cast<const u_char*>(msg.data()), msg.size(),
reinterpret_cast<u_char*>(hash));
uitoa_n(hash[0], id, sizeof(id), 62);
uitoa_n(hash[0], tmp, sizeof(tmp), 62);
DBG_LOG(DBG_FILE_ANALYSIS, "Creating new Info object %s (%s)", id,
DBG_LOG(DBG_FILE_ANALYSIS, "Creating new File object %s (%s)", tmp,
unique.c_str());
val = new RecordVal(BifType::Record::FileAnalysis::Info);
val->Assign(file_id_idx, new StringVal(id));
file_id = FileID(id);
val = new RecordVal(fa_file_type);
val->Assign(id_idx, new StringVal(tmp));
id = FileID(tmp);
if ( conn )
{
@ -113,23 +112,23 @@ Info::Info(const string& unique, Connection* conn, AnalyzerTag::Tag tag)
UpdateLastActivityTime();
}
Info::~Info()
File::~File()
{
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying Info object %s", file_id.c_str());
DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
Unref(val);
}
void Info::UpdateLastActivityTime()
void File::UpdateLastActivityTime()
{
val->Assign(last_active_idx, new Val(network_time, TYPE_TIME));
}
double Info::GetLastActivityTime() const
double File::GetLastActivityTime() const
{
return val->Lookup(last_active_idx)->AsTime();
}
void Info::UpdateConnectionFields(Connection* conn)
void File::UpdateConnectionFields(Connection* conn)
{
if ( ! conn ) return;
@ -155,7 +154,7 @@ void Info::UpdateConnectionFields(Connection* conn)
Unref(idx);
}
uint64 Info::LookupFieldDefaultCount(int idx) const
uint64 File::LookupFieldDefaultCount(int idx) const
{
Val* v = val->LookupWithDefault(idx);
uint64 rval = v->AsCount();
@ -163,7 +162,7 @@ uint64 Info::LookupFieldDefaultCount(int idx) const
return rval;
}
double Info::LookupFieldDefaultInterval(int idx) const
double File::LookupFieldDefaultInterval(int idx) const
{
Val* v = val->LookupWithDefault(idx);
double rval = v->AsInterval();
@ -171,46 +170,31 @@ double Info::LookupFieldDefaultInterval(int idx) const
return rval;
}
int Info::Idx(const string& field)
int File::Idx(const string& field)
{
int rval = BifType::Record::FileAnalysis::Info->FieldOffset(field.c_str());
int rval = fa_file_type->FieldOffset(field.c_str());
if ( rval < 0 )
reporter->InternalError("Unknown FileAnalysis::Info field: %s",
field.c_str());
reporter->InternalError("Unknown fa_file field: %s", field.c_str());
return rval;
}
double Info::GetTimeoutInterval() const
double File::GetTimeoutInterval() const
{
return LookupFieldDefaultInterval(timeout_interval_idx);
}
RecordVal* Info::GetResults(RecordVal* args) const
{
TableVal* actions_table = val->Lookup(actions_idx)->AsTableVal();
RecordVal* rval = actions_table->Lookup(args)->AsRecordVal();
if ( ! rval )
{
rval = new RecordVal(BifType::Record::FileAnalysis::ActionResults);
actions_table->Assign(args, rval);
}
return rval;
}
void Info::IncrementByteCount(uint64 size, int field_idx)
void File::IncrementByteCount(uint64 size, int field_idx)
{
uint64 old = LookupFieldDefaultCount(field_idx);
val->Assign(field_idx, new Val(old + size, TYPE_COUNT));
}
void Info::SetTotalBytes(uint64 size)
void File::SetTotalBytes(uint64 size)
{
val->Assign(total_bytes_idx, new Val(size, TYPE_COUNT));
}
bool Info::IsComplete() const
bool File::IsComplete() const
{
Val* total = val->Lookup(total_bytes_idx);
if ( ! total ) return false;
@ -219,22 +203,22 @@ bool Info::IsComplete() const
return false;
}
void Info::ScheduleInactivityTimer() const
void File::ScheduleInactivityTimer() const
{
timer_mgr->Add(new InfoTimer(network_time, file_id, GetTimeoutInterval()));
timer_mgr->Add(new FileTimer(network_time, id, GetTimeoutInterval()));
}
bool Info::AddAction(RecordVal* args)
bool File::AddAction(RecordVal* args)
{
return done ? false : actions.QueueAddAction(args);
}
bool Info::RemoveAction(const RecordVal* args)
bool File::RemoveAction(const RecordVal* args)
{
return done ? false : actions.QueueRemoveAction(args);
}
bool Info::BufferBOF(const u_char* data, uint64 len)
bool File::BufferBOF(const u_char* data, uint64 len)
{
if ( bof_buffer.full || bof_buffer.replayed ) return false;
@ -267,7 +251,7 @@ bool Info::BufferBOF(const u_char* data, uint64 len)
return true;
}
bool Info::DetectTypes(const u_char* data, uint64 len)
bool File::DetectTypes(const u_char* data, uint64 len)
{
const char* desc = bro_magic_buffer(magic, data, len);
const char* mime = bro_magic_buffer(magic_mime, data, len);
@ -281,7 +265,7 @@ bool Info::DetectTypes(const u_char* data, uint64 len)
return desc || mime;
}
void Info::ReplayBOF()
void File::ReplayBOF()
{
if ( bof_buffer.replayed ) return;
bof_buffer.replayed = true;
@ -307,7 +291,7 @@ void Info::ReplayBOF()
DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
}
void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
void File::DataIn(const u_char* data, uint64 len, uint64 offset)
{
actions.DrainModifications();
@ -344,7 +328,7 @@ void Info::DataIn(const u_char* data, uint64 len, uint64 offset)
IncrementByteCount(len, seen_bytes_idx);
}
void Info::DataIn(const u_char* data, uint64 len)
void File::DataIn(const u_char* data, uint64 len)
{
actions.DrainModifications();
@ -383,7 +367,7 @@ void Info::DataIn(const u_char* data, uint64 len)
IncrementByteCount(len, seen_bytes_idx);
}
void Info::EndOfFile()
void File::EndOfFile()
{
if ( done ) return;
@ -403,15 +387,12 @@ void Info::EndOfFile()
actions.QueueRemoveAction(act->Args());
}
if ( IsComplete() )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_DONE, this);
else
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_EOF, this);
file_mgr->FileEvent(file_state_remove, this);
actions.DrainModifications();
}
void Info::Gap(uint64 offset, uint64 len)
void File::Gap(uint64 offset, uint64 len)
{
actions.DrainModifications();

37
src/file_analysis/Info.h → src/file_analysis/File.h
View file

@ -1,5 +1,5 @@
#ifndef FILE_ANALYSIS_INFO_H
#define FILE_ANALYSIS_INFO_H
#ifndef FILE_ANALYSIS_FILE_H
#define FILE_ANALYSIS_FILE_H
#include <string>
#include <vector>
@ -15,12 +15,14 @@
namespace file_analysis {
/**
* Wrapper class around \c FileAnalysis::Info record values from script layer.
* Wrapper class around \c fa_file record values from script layer.
*/
class Info {
class File {
friend class Manager;
public:
~Info();
~File();
/**
* @return the #val record.
@ -33,16 +35,9 @@ public:
double GetTimeoutInterval() const;
/**
* @return value of the "file_id" field from #val record.
* @return value of the "id" field from #val record.
*/
FileID GetFileID() const { return file_id; }
/**
* @return looks up the value of the "actions" field in the #val record at
* the index corresponding to \a args. If there was no value at
* the index, it is created.
*/
RecordVal* GetResults(RecordVal* args) const;
FileID GetID() const { return id; }
/**
* @return the string which uniquely identifies the file.
@ -113,12 +108,10 @@ public:
protected:
friend class Manager;
/**
* Constructor; only file_analysis::Manager should be creating these.
*/
Info(const string& unique, Connection* conn = 0,
File(const string& unique, Connection* conn = 0,
AnalyzerTag::Tag tag = AnalyzerTag::Error);
/**
@ -162,9 +155,9 @@ protected:
*/
bool DetectTypes(const u_char* data, uint64 len);
FileID file_id; /**< A pretty hash that likely identifies file*/
FileID id; /**< A pretty hash that likely identifies file*/
string unique; /**< A string that uniquely identifies file */
RecordVal* val; /**< \c FileAnalysis::Info from script layer. */
RecordVal* val; /**< \c fa_file from script layer. */
bool postpone_timeout; /**< Whether postponing timeout is requested. */
bool first_chunk; /**< Track first non-linear chunk. */
bool need_type; /**< Flags next data input to be magic typed. */
@ -198,9 +191,8 @@ protected:
static string salt;
public:
static int file_id_idx;
static int parent_file_id_idx;
static int id_idx;
static int parent_id_idx;
static int source_idx;
static int conns_idx;
static int last_active_idx;
@ -213,7 +205,6 @@ public:
static int bof_buffer_idx;
static int file_type_idx;
static int mime_type_idx;
static int actions_idx;
};
} // namespace file_analysis

20
src/file_analysis/InfoTimer.cc → src/file_analysis/FileTimer.cc
View file

@ -1,23 +1,23 @@
#include "Manager.h"
#include "Info.h"
#include "File.h"
using namespace file_analysis;
InfoTimer::InfoTimer(double t, const FileID& id, double interval)
FileTimer::FileTimer(double t, const FileID& id, double interval)
: Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id)
{
DBG_LOG(DBG_FILE_ANALYSIS, "New %f second timeout timer for %s",
file_id.c_str(), interval);
}
void InfoTimer::Dispatch(double t, int is_expire)
void FileTimer::Dispatch(double t, int is_expire)
{
Info* info = file_mgr->Lookup(file_id);
File* file = file_mgr->Lookup(file_id);
if ( ! info ) return;
if ( ! file ) return;
double last_active = info->GetLastActivityTime();
double last_active = file->GetLastActivityTime();
double inactive_time = t > last_active ? t - last_active : 0.0;
DBG_LOG(DBG_FILE_ANALYSIS, "Checking inactivity for %s, last active at %f, "
@ -26,13 +26,13 @@ void InfoTimer::Dispatch(double t, int is_expire)
if ( last_active == 0.0 )
{
// was created when network_time was zero, so re-schedule w/ valid time
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
file->UpdateLastActivityTime();
file->ScheduleInactivityTimer();
return;
}
if ( inactive_time >= info->GetTimeoutInterval() )
if ( inactive_time >= file->GetTimeoutInterval() )
file_mgr->Timeout(file_id);
else if ( ! is_expire )
info->ScheduleInactivityTimer();
file->ScheduleInactivityTimer();
}

10
src/file_analysis/InfoTimer.h → src/file_analysis/FileTimer.h
View file

@ -1,5 +1,5 @@
#ifndef FILE_ANALYSIS_INFOTIMER_H
#define FILE_ANALYSIS_INFOTIMER_H
#ifndef FILE_ANALYSIS_FILETIMER_H
#define FILE_ANALYSIS_FILETIMER_H
#include <string>
#include "Timer.h"
@ -10,13 +10,13 @@ namespace file_analysis {
/**
* Timer to periodically check if file analysis for a given file is inactive.
*/
class InfoTimer : public Timer {
class FileTimer : public Timer {
public:
InfoTimer(double t, const FileID& id, double interval);
FileTimer(double t, const FileID& id, double interval);
/**
* Check inactivity of file_analysis::Info corresponding to #file_id,
* Check inactivity of file_analysis::File corresponding to #file_id,
* reschedule if active, else call file_analysis::Manager::Timeout.
*/
void Dispatch(double t, int is_expire);

16
src/file_analysis/Hash.cc
View file

@ -2,15 +2,13 @@
#include "Hash.h"
#include "util.h"
#include "Event.h"
using namespace file_analysis;
Hash::Hash(RecordVal* args, Info* info, HashVal* hv, const char* field)
: Action(args, info), hash(hv), fed(false)
Hash::Hash(RecordVal* args, File* file, HashVal* hv, const char* arg_kind)
: Action(args, file), hash(hv), fed(false), kind(arg_kind)
{
using BifType::Record::FileAnalysis::ActionResults;
if ( (result_field_idx = ActionResults->FieldOffset(field)) < 0 )
reporter->InternalError("Missing ActionResults field: %s", field);
hash->Init();
}
@ -45,6 +43,10 @@ void Hash::Finalize()
{
if ( ! hash->IsValid() || ! fed ) return;
StringVal* sv = hash->Get();
info->GetResults(args)->Assign(result_field_idx, sv);
val_list* vl = new val_list();
vl->append(file->GetVal()->Ref());
vl->append(new StringVal(kind));
vl->append(hash->Get());
mgr.Dispatch(new Event(file_hash, vl));
}

30
src/file_analysis/Hash.h
View file

@ -5,7 +5,7 @@
#include "Val.h"
#include "OpaqueVal.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
namespace file_analysis {
@ -26,51 +26,51 @@ public:
protected:
Hash(RecordVal* args, Info* info, HashVal* hv, const char* field);
Hash(RecordVal* args, File* file, HashVal* hv, const char* kind);
void Finalize();
HashVal* hash;
bool fed;
int result_field_idx;
const char* kind;
};
class MD5 : public Hash {
public:
static Action* Instantiate(RecordVal* args, Info* info)
{ return new MD5(args, info); }
static Action* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new MD5(args, file) : 0; }
protected:
MD5(RecordVal* args, Info* info)
: Hash(args, info, new MD5Val(), "md5")
MD5(RecordVal* args, File* file)
: Hash(args, file, new MD5Val(), "md5")
{}
};
class SHA1 : public Hash {
public:
static Action* Instantiate(RecordVal* args, Info* info)
{ return new SHA1(args, info); }
static Action* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new SHA1(args, file) : 0; }
protected:
SHA1(RecordVal* args, Info* info)
: Hash(args, info, new SHA1Val(), "sha1")
SHA1(RecordVal* args, File* file)
: Hash(args, file, new SHA1Val(), "sha1")
{}
};
class SHA256 : public Hash {
public:
static Action* Instantiate(RecordVal* args, Info* info)
{ return new SHA256(args, info); }
static Action* Instantiate(RecordVal* args, File* file)
{ return file_hash ? new SHA256(args, file) : 0; }
protected:
SHA256(RecordVal* args, Info* info)
: Hash(args, info, new SHA256Val(), "sha256")
SHA256(RecordVal* args, File* file)
: Hash(args, file, new SHA256Val(), "sha256")
{}
};

114
src/file_analysis/Manager.cc
View file

@ -2,7 +2,7 @@
#include <string>
#include "Manager.h"
#include "Info.h"
#include "File.h"
#include "Action.h"
#include "Var.h"
#include "Event.h"
@ -66,18 +66,18 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
const string& unique)
{
DataIn(data, len, offset, GetInfo(unique));
DataIn(data, len, offset, GetFile(unique));
}
void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
Info* info)
File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->DataIn(data, len, offset);
file->DataIn(data, len, offset);
if ( info->IsComplete() )
RemoveFile(info->GetUnique());
if ( file->IsComplete() )
RemoveFile(file->GetUnique());
}
void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
@ -90,17 +90,17 @@ void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
{
DataIn(data, len, GetInfo(unique));
DataIn(data, len, GetFile(unique));
}
void Manager::DataIn(const u_char* data, uint64 len, Info* info)
void Manager::DataIn(const u_char* data, uint64 len, File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->DataIn(data, len);
file->DataIn(data, len);
if ( info->IsComplete() )
RemoveFile(info->GetUnique());
if ( file->IsComplete() )
RemoveFile(file->GetUnique());
}
void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
@ -131,14 +131,14 @@ void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
void Manager::Gap(uint64 offset, uint64 len, const string& unique)
{
Gap(offset, len, GetInfo(unique));
Gap(offset, len, GetFile(unique));
}
void Manager::Gap(uint64 offset, uint64 len, Info* info)
void Manager::Gap(uint64 offset, uint64 len, File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->Gap(offset, len);
file->Gap(offset, len);
}
void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
@ -151,22 +151,32 @@ void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
void Manager::SetSize(uint64 size, const string& unique)
{
SetSize(size, GetInfo(unique));
SetSize(size, GetFile(unique));
}
void Manager::SetSize(uint64 size, Info* info)
void Manager::SetSize(uint64 size, File* file)
{
if ( ! info ) return;
if ( ! file ) return;
info->SetTotalBytes(size);
file->SetTotalBytes(size);
if ( info->IsComplete() )
RemoveFile(info->GetUnique());
if ( file->IsComplete() )
RemoveFile(file->GetUnique());
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
void Manager::FileEvent(EventHandlerPtr h, File* file)
{
if ( IsIgnored(info->GetUnique()) ) return;
if ( IsIgnored(file->GetUnique()) ) return;
if ( ! h ) return;
val_list * vl = new val_list();
vl->append(file->GetVal()->Ref());
mgr.Dispatch(new Event(h, vl));
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, File* file)
{
if ( IsIgnored(file->GetUnique()) ) return;
const ID* id = global_scope()->Lookup("FileAnalysis::policy");
assert(id);
@ -174,9 +184,7 @@ void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
val_list vl(2);
vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger));
vl.append(info->val->Ref());
info->postpone_timeout = false;
vl.append(file->val->Ref());
Val* result = hook->Call(&vl);
Unref(result);
@ -184,43 +192,43 @@ void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info)
bool Manager::PostponeTimeout(const FileID& file_id) const
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return false;
if ( ! file ) return false;
info->postpone_timeout = true;
file->postpone_timeout = true;
return true;
}
bool Manager::AddAction(const FileID& file_id, RecordVal* args) const
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return false;
if ( ! file ) return false;
return info->AddAction(args);
return file->AddAction(args);
}
bool Manager::RemoveAction(const FileID& file_id, const RecordVal* args) const
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return false;
if ( ! file ) return false;
return info->RemoveAction(args);
return file->RemoveAction(args);
}
Info* Manager::GetInfo(const string& unique, Connection* conn,
File* Manager::GetFile(const string& unique, Connection* conn,
AnalyzerTag::Tag tag)
{
if ( IsIgnored(unique) ) return 0;
Info* rval = str_map[unique];
File* rval = str_map[unique];
if ( ! rval )
{
rval = str_map[unique] = new Info(unique, conn, tag);
FileID id = rval->GetFileID();
rval = str_map[unique] = new File(unique, conn, tag);
FileID id = rval->GetID();
if ( id_map[id] )
{
@ -242,7 +250,7 @@ Info* Manager::GetInfo(const string& unique, Connection* conn,
return rval;
}
Info* Manager::Lookup(const FileID& file_id) const
File* Manager::Lookup(const FileID& file_id) const
{
IDMap::const_iterator it = id_map.find(file_id);
@ -253,25 +261,27 @@ Info* Manager::Lookup(const FileID& file_id) const
void Manager::Timeout(const FileID& file_id, bool is_terminating)
{
Info* info = Lookup(file_id);
File* file = Lookup(file_id);
if ( ! info ) return;
if ( ! file ) return;
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, info);
file->postpone_timeout = false;
if ( info->postpone_timeout && ! is_terminating )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, file);
if ( file->postpone_timeout && ! is_terminating )
{
DBG_LOG(DBG_FILE_ANALYSIS, "Postpone file analysis timeout for %s",
info->GetFileID().c_str());
info->UpdateLastActivityTime();
info->ScheduleInactivityTimer();
file->GetID().c_str());
file->UpdateLastActivityTime();
file->ScheduleInactivityTimer();
return;
}
DBG_LOG(DBG_FILE_ANALYSIS, "File analysis timeout for %s",
info->GetFileID().c_str());
file->GetID().c_str());
RemoveFile(info->GetUnique());
RemoveFile(file->GetUnique());
}
bool Manager::IgnoreFile(const FileID& file_id)
@ -295,7 +305,7 @@ bool Manager::RemoveFile(const string& unique)
it->second->EndOfFile();
FileID id = it->second->GetFileID();
FileID id = it->second->GetID();
DBG_LOG(DBG_FILE_ANALYSIS, "Remove FileID %s", id.c_str());

48
src/file_analysis/Manager.h
View file

@ -12,9 +12,10 @@
#include "Val.h"
#include "Analyzer.h"
#include "Timer.h"
#include "EventHandler.h"
#include "Info.h"
#include "InfoTimer.h"
#include "File.h"
#include "FileTimer.h"
#include "FileID.h"
#include "PendingFile.h"
@ -24,6 +25,9 @@ namespace file_analysis {
* Main entry point for interacting with file analysis.
*/
class Manager {
friend class FileTimer;
friend class PendingFile;
public:
Manager();
@ -56,7 +60,7 @@ public:
void DataIn(const u_char* data, uint64 len, uint64 offset,
const string& unique);
void DataIn(const u_char* data, uint64 len, uint64 offset,
Info* info);
File* file);
/**
* Pass in sequential file data.
@ -64,7 +68,7 @@ public:
void DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
Connection* conn, bool is_orig);
void DataIn(const u_char* data, uint64 len, const string& unique);
void DataIn(const u_char* data, uint64 len, Info* info);
void DataIn(const u_char* data, uint64 len, File* file);
/**
* Signal the end of file data.
@ -79,7 +83,7 @@ public:
void Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
void Gap(uint64 offset, uint64 len, const string& unique);
void Gap(uint64 offset, uint64 len, Info* info);
void Gap(uint64 offset, uint64 len, File* file);
/**
* Provide the expected number of bytes that comprise a file.
@ -87,7 +91,7 @@ public:
void SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
void SetSize(uint64 size, const string& unique);
void SetSize(uint64 size, Info* info);
void SetSize(uint64 size, File* file);
/**
* Starts ignoring a file, which will finally be removed from internal
@ -119,42 +123,44 @@ public:
/**
* Calls the \c FileAnalysis::policy hook.
*/
void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, Info* info);
void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, File* file);
/**
* Dispatches an event related to the file's life-cycle.
*/
void FileEvent(EventHandlerPtr h, File* file);
protected:
friend class InfoTimer;
friend class PendingFile;
typedef map<string, Info*> StrMap;
typedef map<string, File*> StrMap;
typedef set<string> StrSet;
typedef map<FileID, Info*> IDMap;
typedef map<FileID, File*> IDMap;
typedef queue<PendingFile*> PendingQueue;
/**
* @return the Info object mapped to \a unique or a null pointer if analysis
* is being ignored for the associated file. An Info object may be
* @return the File object mapped to \a unique or a null pointer if analysis
* is being ignored for the associated file. An File object may be
* created if a mapping doesn't exist, and if it did exist, the
* activity time is refreshed along with any connection-related
* fields.
*/
Info* GetInfo(const string& unique, Connection* conn = 0,
File* GetFile(const string& unique, Connection* conn = 0,
AnalyzerTag::Tag tag = AnalyzerTag::Error);
/**
* @return the Info object mapped to \a file_id, or a null pointer if no
* @return the File object mapped to \a file_id, or a null pointer if no
* mapping exists.
*/
Info* Lookup(const FileID& file_id) const;
File* Lookup(const FileID& file_id) const;
/**
* Evaluate timeout policy for a file and remove the Info object mapped to
* Evaluate timeout policy for a file and remove the File object mapped to
* \a file_id if needed.
*/
void Timeout(const FileID& file_id, bool is_terminating = ::terminating);
/**
* Immediately remove file_analysis::Info object associated with \a unique.
* Immediately remove file_analysis::File object associated with \a unique.
* @return false if file string did not map to anything, else true.
*/
bool RemoveFile(const string& unique);
@ -176,8 +182,8 @@ protected:
static bool QueueHandleEvent(AnalyzerTag::Tag tag, Connection* conn,
bool is_orig);
StrMap str_map; /**< Map unique strings to \c FileAnalysis::Info records. */
IDMap id_map; /**< Map file IDs to \c FileAnalysis::Info records. */
StrMap str_map; /**< Map unique string to file_analysis::File. */
IDMap id_map; /**< Map file ID to file_analysis::File records. */
StrSet ignored; /**< Ignored files. Will be finally removed on EOF. */
PendingQueue pending; /**< Files awaiting a unique handle. */

12
src/file_analysis/PendingFile.cc
View file

@ -34,9 +34,9 @@ PendingFile::~PendingFile()
conn_str(conn).c_str());
}
Info* PendingFile::GetInfo(const string& handle) const
File* PendingFile::GetFile(const string& handle) const
{
return file_mgr->GetInfo(handle, conn, tag);
return file_mgr->GetFile(handle, conn, tag);
}
PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
@ -51,7 +51,7 @@ PendingDataInChunk::PendingDataInChunk(const u_char* arg_data, uint64 arg_len,
void PendingDataInChunk::Finish(const string& handle) const
{
file_mgr->DataIn(data, len, offset, GetInfo(handle));
file_mgr->DataIn(data, len, offset, GetFile(handle));
}
PendingDataInChunk::~PendingDataInChunk()
@ -69,7 +69,7 @@ PendingDataInStream::PendingDataInStream(const u_char* arg_data, uint64 arg_len,
void PendingDataInStream::Finish(const string& handle) const
{
file_mgr->DataIn(data, len, GetInfo(handle));
file_mgr->DataIn(data, len, GetFile(handle));
}
PendingDataInStream::~PendingDataInStream()
@ -86,7 +86,7 @@ PendingGap::PendingGap(uint64 arg_offset, uint64 arg_len,
void PendingGap::Finish(const string& handle) const
{
file_mgr->Gap(offset, len, GetInfo(handle));
file_mgr->Gap(offset, len, GetFile(handle));
}
PendingEOF::PendingEOF(AnalyzerTag::Tag arg_tag, Connection* arg_conn)
@ -107,5 +107,5 @@ PendingSize::PendingSize(uint64 arg_size, AnalyzerTag::Tag arg_tag,
void PendingSize::Finish(const string& handle) const
{
file_mgr->SetSize(size, GetInfo(handle));
file_mgr->SetSize(size, GetFile(handle));
}

4
src/file_analysis/PendingFile.h
View file

@ -3,7 +3,7 @@
#include "AnalyzerTags.h"
#include "Conn.h"
#include "Info.h"
#include "File.h"
namespace file_analysis {
@ -23,7 +23,7 @@ protected:
PendingFile(Connection* arg_conn,
AnalyzerTag::Tag arg_tag = AnalyzerTag::Error);
Info* GetInfo(const string& handle) const;
File* GetFile(const string& handle) const;
Connection* conn;
AnalyzerTag::Tag tag;

61
src/types.bif
View file

@ -226,3 +226,64 @@ type gtp_rai: record;
type gtp_qos_profile: record;
type gtp_private_extension: record;
type gtp_gsn_addr: record;
module FileAnalysis;
type ActionArgs: record;
## An enumeration of significant things that can occur over the course of
## analyzing files. The :bro:see:`FileAnalysis::policy` hook is called each
## time a trigger occurs.
enum Trigger %{
## Raised when any part of a new file is detected.
TRIGGER_NEW,
## Raised when file is detected being transported over a new network
## connection (other than the first).
TRIGGER_NEW_CONN,
## Raised when file analysis for a given file is aborted due
## to not seeing any data for it recently. Note that this doesn't
## necessarily mean the full file wasn't seen (e.g. if the
## :bro:see:`fa_file` record indicates the file *total_bytes*
## isn't known). Use :bro:see:`FileAnalysis::postpone_timeout`
## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
## defer the timeout until later.
TRIGGER_TIMEOUT,
## Raised when the beginning of a file is detected.
TRIGGER_BOF,
## Raised when the beginning of a file is available in the *bof_buffer*
## field of :bro:see:`fa_file` and that beginning
## is at least the number of bytes indicated by the *bof_buffer_size* field.
TRIGGER_BOF_BUFFER,
## Raised when an initial guess at the file/mime type of a file is matched.
TRIGGER_TYPE,
## Raised when there's a missing chunk of data in the file stream.
TRIGGER_GAP,
%}
## An enumeration of various file analysis actions that can be taken.
enum Action %{
## Extract a file to local filesystem
ACTION_EXTRACT,
## Calculate an MD5 digest of the file's contents.
ACTION_MD5,
## Calculate an SHA1 digest of the file's contents.
ACTION_SHA1,
## Calculate an SHA256 digest of the file's contents.
ACTION_SHA256,
## Deliver the file contents to the script-layer in an event.
ACTION_DATA_EVENT,
%}
module GLOBAL;

16
testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out
View file

@ -14,15 +14,15 @@ file_stream, Cx92a0ym5R8, 476, ormat for RFC2253^J compliance. (Jon Siwek)^J^
file_chunk, Cx92a0ym5R8, 476, 2524, ormat for RFC2253^J compliance. (Jon Siwek)^J^J * New tool devel-tools/check-release to run before making releases.^J (Robin Sommer)^J^J * devel-tools/update-changes gets a new option -a to amend to^J previous commit if possible. Default is now not to (used to be the^J opposite). (Robin Sommer)^J^J * Change Mozilla trust root generation to index certs by subject DN. (Jon Siwek)^J^J * Change distclean to only remove build dir. (Jon Siwek)^J^J * Make dist now cleans the
file_stream, Cx92a0ym5R8, 1024, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP
file_chunk, Cx92a0ym5R8, 1024, 3000, copied source (Jon Siwek)^J^J * Small tweak to make-release for forced git-clean. (Jon Siwek)^J^J * Fix to not let updates scripts loose their executable permissions.^J (Robin Sommer)^J^J * devel-tools/update-changes now looks for a 'release' tag to^J idenfify the stable version, and 'beta' for the beta versions.^J (Robin Sommer).^J^J * Distribution cleanup. (Robin Sommer)^J^J * New script devel-tools/make-release to create source tar balls.^J (Robin Sommer)^J^J * Removing bdcat. With the new log format, this isn't very useful^J anymore. (Robin Sommer)^J^J * Adding script that shows all pending git fastpath commits. (Robin^J Sommer)^J^J * Script to measure CPU time by loading an increasing set of^J scripts. (Robin Sommer)^J^J * extract-conn script now deals wit *.gz files. (Robin Sommer)^J^J * Tiny update to output a valid CA list file for SSL cert^J validation. (Seth Hall)^J^J * Adding "install-aux" target. Addresses #622. (Jon Siwek)^J^J * Distribution cleanup. (Jon Siwek and Robin Sommer)^J^J * FindPCAP
FileAnalysis::TRIGGER_DONE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705
source: HTTP
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
MD5: 397168fd09991a0e712254df7bc639ac
SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
file_stream, Cx92a0ym5R8, 476, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J
file_chunk, Cx92a0ym5R8, 476, 4024, now links against thread library when necessary (e.g.^J PF_RING's libpcap) (Jon Siwek)^J^J * Install binaries with an RPATH (Jon Siwek)^J^J * Workaround for FreeBSD CMake port missing debug flags (Jon Siwek)^J^J * Rewrite of the update-changes script. (Robin Sommer)^J^J0.1-1 | 2011-06-14 21:12:41 -0700^J^J * Add a script for generating Mozilla's CA list for the SSL analyzer.^J (Seth Hall)^J^J0.1 | 2011-04-01 16:28:22 -0700^J^J * Converting build process to CMake. (Jon Siwek)^J
file_stream, Cx92a0ym5R8, 205, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J
file_chunk, Cx92a0ym5R8, 205, 4500, ^J * Removing cf/hf/ca-* from distribution. The README has a note where^J to find them now. (Robin Sommer)^J^J * General cleanup. (Robin Sommer)^J^J * Initial import of bro/aux from SVN r7088. (Jon Siwek)^J
FILE_STATE_REMOVE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705
source: HTTP
MD5: 397168fd09991a0e712254df7bc639ac
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout
View file

@ -3,7 +3,7 @@ oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
@ -15,7 +15,7 @@ file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]
total bytes: 1022920

2
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out
View file

@ -6,7 +6,7 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out
View file

@ -6,10 +6,10 @@ The Nationa
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
sidhzrR4IT8, 16557, 0
[orig_h=141.142.228.5, orig_p=50737/tcp, resp_h=141.142.192.162, resp_p=38141/tcp]
source: FTP_DATA
SHA1: 44586aed07cfe19cad25076af98f535585cd5797
MD5: 7192a8075196267203adb3dfaa5c908d
SHA1: 44586aed07cfe19cad25076af98f535585cd5797
SHA256: 202674eba48e832690a4475113acf8b16a3f6c82c04c94b36bb2c7ce457ac8d2

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out
View file

@ -6,10 +6,10 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
kg59rqyYxN, 197, 0
[orig_h=141.142.228.5, orig_p=50153/tcp, resp_h=54.243.118.187, resp_p=80/tcp]
source: HTTP
SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31
MD5: 5baba7eea57bc8a42a92c817ed566d72
SHA1: e351b8c693c3353716787c02e2923f4d12ebbb31
SHA256: 202b775be087f5af98e95120e42769a9b3488f84c5aa79c4f4c1093d348f849c

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out
View file

@ -6,11 +6,11 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
Cx92a0ym5R8, 4705, 0
[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
total bytes: 4705
source: HTTP
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
MD5: 397168fd09991a0e712254df7bc639ac
SHA1: 1dd7ac0398df6cbc0696445a91ec681facf4dc47
SHA256: 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18

2
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out
View file

@ -4,7 +4,7 @@ FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
7gZBKVUgy4l, 555523, 0
[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]
[orig_h=10.101.84.70, orig_p=10977/tcp, resp_h=129.174.93.161, resp_p=80/tcp]

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out
View file

@ -3,7 +3,7 @@ oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
@ -14,7 +14,7 @@ FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]
total bytes: 1022920

2
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out
View file

@ -4,7 +4,7 @@ FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
uHS14uhRKGe, 498702, 0
[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]
[orig_h=10.45.179.94, orig_p=19953/tcp, resp_h=129.174.93.170, resp_p=80/tcp]

20
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out
View file

@ -6,12 +6,12 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
aFQKI8SPOL2, 2675, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
source: HTTP
SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
MD5: b932c3310ce47e158d1a5a42e0b01279
SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42
FileAnalysis::TRIGGER_NEW
CCU3vUEr06l, 0, 0
@ -21,12 +21,12 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
CCU3vUEr06l, 21421, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
source: HTTP
SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
MD5: e732f7bf1d7cb4eedcb1661697d7bc8c
SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da
FileAnalysis::TRIGGER_NEW
HCzA0dVwDPj, 0, 0
@ -36,13 +36,13 @@ GIF89a^D\0^D\0\xb3
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
HCzA0dVwDPj, 94, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
total bytes: 94
source: HTTP
SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
MD5: d903de7e30db1691d3130ba5eae6b9a7
SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315
FileAnalysis::TRIGGER_NEW
a1Zu1fteVEf, 0, 0
@ -52,13 +52,13 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
a1Zu1fteVEf, 2349, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
total bytes: 2349
source: HTTP
SHA1: 560eab5a0177246827a94042dd103916d8765ac7
MD5: e0029eea80812e9a8e57b8d05d52938a
SHA1: 560eab5a0177246827a94042dd103916d8765ac7
SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4
FileAnalysis::TRIGGER_NEW
xXlF7wFdsR, 0, 0
@ -68,11 +68,11 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
xXlF7wFdsR, 27579, 0
[orig_h=192.168.1.104, orig_p=1673/tcp, resp_h=63.245.209.11, resp_p=80/tcp]
total bytes: 27579
source: HTTP
SHA1: ee2b41bdef85de14ef332da14fc392f110b84249
MD5: 30aa926344f58019d047e85ba049ca1e
SHA1: ee2b41bdef85de14ef332da14fc392f110b84249
SHA256: eb482bda230a215b90aedbfe1eee72b8193608df76a319aaf11fb85511579a1e

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out
View file

@ -6,13 +6,13 @@ hello world
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
v5HLI7MxPQh, 11, 0
[orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp]
total bytes: 11
source: HTTP
SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3
SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
FileAnalysis::TRIGGER_NEW
PZS1XGHkIf1, 0, 0
@ -22,11 +22,11 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_DONE
FILE_STATE_REMOVE
PZS1XGHkIf1, 366, 0
[orig_h=141.142.228.5, orig_p=53595/tcp, resp_h=54.243.55.129, resp_p=80/tcp]
total bytes: 366
source: HTTP
SHA1: 6a1582672c203210c6d18d700322060b676365e7
MD5: c9337794df612aeaa901dcf9fa446bca
SHA1: 6a1582672c203210c6d18d700322060b676365e7
SHA256: 8eb24c16df7cb45cb6a1790b0d26ad2571f754228d0ac111b3ac59adbfecbeb8

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout
View file

@ -6,9 +6,9 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
nYgPNGLrZf9, 311, 0
source: ../input.log
SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1
MD5: bf4dfa6169b74146da5236e918743599
SHA1: 0a0f20de89c86d7bce1301af6548d6e9ae87b0f1
SHA256: 4e573192c5ea75da72494812fe24dae53a577837b2079df012fd464903d68a6f

4
testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out
View file

@ -6,10 +6,10 @@ PK^C^D^T\0\0\0^H\0\xae
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
wqKMAamJVSb, 42208, 0
[orig_h=192.168.1.77, orig_p=57655/tcp, resp_h=209.197.168.151, resp_p=1024/tcp]
source: IRC_DATA
SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218
MD5: 8c0803242f549c2780cb88b9a9215c65
SHA1: 8abe0239263fd7326eb803d4465cf494f8bea218
SHA256: e4f0b0b9d7580e7a22dc1093c8db4df7d0115a4f3b03cc2875cc69705f0d0204

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/file_analysis.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path file_analysis
#open 2013-03-29-18-28-57
#fields file_id parent_file_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type timedout conn_uids actions_taken extracted_files md5 sha1 sha256
#open 2013-04-04-21-22-26
#fields id parent_id source last_active seen_bytes total_bytes missing_bytes overflow_bytes timeout_interval bof_buffer_size file_type mime_type timedout conn_uids actions_taken extracted_files md5 sha1 sha256
#types string string string time count count count count interval count string string bool table[string] table[enum] table[string] string string string
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set F UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
#close 2013-03-29-18-28-57
Cx92a0ym5R8 - HTTP 1362692527.009775 4705 4705 0 0 120.000000 1024 set set F UWkUyAuUGXf FileAnalysis::ACTION_SHA1,FileAnalysis::ACTION_EXTRACT,FileAnalysis::ACTION_DATA_EVENT,FileAnalysis::ACTION_MD5,FileAnalysis::ACTION_SHA256 Cx92a0ym5R8-file 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18
#close 2013-04-04-21-22-26

12
testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out
View file

@ -6,12 +6,12 @@ Hello^M^J^M^J ^M
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
cwR7l6Zctxb, 79, 0
[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
source: SMTP
SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
MD5: 92bca2e6cdcde73647125da7dccbdd07
SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
FileAnalysis::TRIGGER_NEW
ZAOEQmRyxv1, 0, 0
@ -21,12 +21,12 @@ FileAnalysis::TRIGGER_BOF_BUFFER
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
ZAOEQmRyxv1, 1918, 0
[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
source: SMTP
SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
MD5: d194c6359c85bb88b54caee18b1e9b44
SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
SHA256: b9556e92ddbe52379b64804136f830d111cafe7fcd78e54817fe40f3bc24268d
FileAnalysis::TRIGGER_NEW
Ltd7QO7jEv3, 0, 0
@ -36,10 +36,10 @@ Version 4.9
FileAnalysis::TRIGGER_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_EOF
FILE_STATE_REMOVE
Ltd7QO7jEv3, 10823, 0
[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]
source: SMTP
SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665
MD5: a968bb0f9f9d95835b2e74c845877e87
SHA1: 43bf1cea1cd4b7d15243e15611859aa49d515665
SHA256: d5c4e7248840932b9d74ea2f3b3ae142c723a863abf5fd0599f9dd1171697e12

4
testing/btest/core/leaks/file-analysis-http-get.bro
View file

@ -8,7 +8,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

4
testing/btest/istate/events-ssl.bro
View file

@ -43,10 +43,10 @@ redef ssl_passphrase = "my-password";
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=10
{
FileAnalysis::stop(info$file_id);
FileAnalysis::stop(f);
}
@TEST-END-FILE

4
testing/btest/istate/events.bro
View file

@ -38,10 +38,10 @@ redef tcp_close_delay = 0secs;
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=10
{
FileAnalysis::stop(info$file_id);
FileAnalysis::stop(f);
}
@TEST-END-FILE

17
testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro
View file

@ -5,25 +5,30 @@
global cnt: count = 0;
global timeout_cnt: count = 0;
redef FileAnalysis::default_timeout_interval=2sec;
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
local rval: string = fmt("%s-file%d", info$file_id, cnt);
local rval: string = fmt("%s-file%d", f$id, cnt);
++cnt;
return rval;
};
redef exit_only_after_terminate = T;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
f$timeout_interval=2sec;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
if ( timeout_cnt < 1 )
FileAnalysis::postpone_timeout(info$file_id);
FileAnalysis::postpone_timeout(f);
else
terminate();
++timeout_cnt;

13
testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro
View file

@ -3,18 +3,17 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
for ( act in test_file_actions )
FileAnalysis::remove_action(info$file_id, act);
local filename = test_get_file_name(info);
FileAnalysis::remove_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::remove_action(f, act);
local filename = test_get_file_name(f);
FileAnalysis::remove_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}

4
testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro
View file

@ -2,8 +2,8 @@
# @TEST-EXEC: btest-diff get.out
# @TEST-EXEC: test ! -s Cx92a0ym5R8-file
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
FileAnalysis::stop(info$file_id);
FileAnalysis::stop(f);
}

2
testing/btest/scripts/base/frameworks/file-analysis/ftp.bro
View file

@ -4,7 +4,7 @@
redef test_file_analysis_source = "FTP_DATA";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return "thefile";
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/get.bro
View file

@ -7,7 +7,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro
View file

@ -17,9 +17,9 @@ global cnt: count = 0;
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
local rval: string = fmt("%s-file%d", info$file_id, cnt);
local rval: string = fmt("%s-file%d", f$id, cnt);
++cnt;
return rval;
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro
View file

@ -8,7 +8,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

4
testing/btest/scripts/base/frameworks/file-analysis/http/post.bro
View file

@ -5,7 +5,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

9
testing/btest/scripts/base/frameworks/file-analysis/input/basic.bro
View file

@ -5,9 +5,9 @@
redef exit_only_after_terminate = T;
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};
@TEST-START-FILE input.log
@ -42,10 +42,7 @@ event bro_init()
Input::remove("input");
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
&priority=-10
event file_state_remove(f: fa_file) &priority=-10
{
if ( trig != FileAnalysis::TRIGGER_EOF ) return;
terminate();
}

2
testing/btest/scripts/base/frameworks/file-analysis/irc.bro
View file

@ -4,7 +4,7 @@
redef test_file_analysis_source = "IRC_DATA";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return "thefile";
};

4
testing/btest/scripts/base/frameworks/file-analysis/logging.bro
View file

@ -3,7 +3,7 @@
redef test_file_analysis_source = "HTTP";
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
return fmt("%s-file", info$file_id);
return fmt("%s-file", f$id);
};

2
testing/btest/scripts/base/frameworks/file-analysis/smtp.bro
View file

@ -8,7 +8,7 @@ redef test_file_analysis_source = "SMTP";
global mycnt: count = 0;
redef test_get_file_name = function(info: FileAnalysis::Info): string
redef test_get_file_name = function(f: fa_file): string
{
local rval: string = fmt("thefile%d", mycnt);
++mycnt;

2
testing/external/subdir-btest.cfg vendored
View file

@ -17,4 +17,4 @@ TRACES=%(testbase)s/Traces
SCRIPTS=%(testbase)s/../scripts
DIST=%(testbase)s/../../..
BUILD=%(testbase)s/../../../build
BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX
#BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX

91
testing/scripts/file-analysis-test.bro
View file

@ -3,43 +3,43 @@ global test_file_analysis_source: string = "" &redef;
global test_file_actions: set[FileAnalysis::ActionArgs];
global test_get_file_name: function(info: FileAnalysis::Info): string =
function(info: FileAnalysis::Info): string { return ""; } &redef;
global test_get_file_name: function(f: fa_file): string =
function(f: fa_file): string { return ""; } &redef;
global test_print_file_data_events: bool = F &redef;
event file_chunk(info: FileAnalysis::Info, data: string, off: count)
event file_chunk(f: fa_file, data: string, off: count)
{
if ( test_print_file_data_events )
print "file_chunk", info$file_id, |data|, off, data;
print "file_chunk", f$id, |data|, off, data;
}
event file_stream(info: FileAnalysis::Info, data: string)
event file_stream(f: fa_file, data: string)
{
if ( test_print_file_data_events )
print "file_stream", info$file_id, |data|, data;
print "file_stream", f$id, |data|, data;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
print f$id, f$seen_bytes, f$missing_bytes;
if ( test_file_analysis_source == "" ||
info$source == test_file_analysis_source )
f$source == test_file_analysis_source )
{
for ( act in test_file_actions )
FileAnalysis::add_action(info$file_id, act);
FileAnalysis::add_action(f, act);
local filename: string = test_get_file_name(info);
local filename: string = test_get_file_name(f);
if ( filename != "" )
FileAnalysis::add_action(info$file_id,
FileAnalysis::add_action(f,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_action(info$file_id,
FileAnalysis::add_action(f,
[$act=FileAnalysis::ACTION_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
@ -48,60 +48,53 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
if ( f?$bof_buffer )
print f$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
if ( f?$file_type )
print "file type is set";
if ( info?$mime_type )
if ( f?$mime_type )
print "mime type is set";
break;
}
}
case FileAnalysis::TRIGGER_EOF:
fallthrough;
case FileAnalysis::TRIGGER_DONE:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info?$conns )
for ( cid in info$conns )
event file_state_remove(f: fa_file)
{
print "FILE_STATE_REMOVE";
print f$id, f$seen_bytes, f$missing_bytes;
if ( f?$conns )
for ( cid in f$conns )
print cid;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
if ( f?$total_bytes )
print "total bytes: " + fmt("%s", f$total_bytes);
if ( f?$source )
print "source: " + f$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
if ( info$actions[act]?$md5 )
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
if ( info$actions[act]?$sha1 )
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
if ( info$actions[act]?$sha256 )
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
if ( ! f?$info ) return;
if ( f$info?$md5 )
print fmt("MD5: %s", f$info$md5);
if ( f$info?$sha1 )
print fmt("SHA1: %s", f$info$sha1);
if ( f$info?$sha256 )
print fmt("SHA256: %s", f$info$sha256);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
# avoids libmagic variances across systems
if ( info?$mime_type )
info$mime_type = "set";
if ( info?$file_type )
info$file_type = "set";
if ( f?$mime_type )
f$mime_type = "set";
if ( f?$file_type )
f$file_type = "set";
}
event bro_init()