Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-02 14:48:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

FileAnalysis: finish switching hooks to events.

Browse source
This commit is contained in:
Jon Siwek 2013-04-10 11:13:43 -05:00
parent 641154f8e8
commit a2d9b47bcd
39 changed files with 219 additions and 305 deletions
Download patch file Download diff file Expand all files Collapse all files

17
scripts/base/frameworks/file-analysis/main.bro
View file

@ -104,12 +104,6 @@ export {
sha256: string &log &optional;
} &redef;
## Evaluated every time a significant event occurs during the course of
## file analysis. Fields of the *info* argument may be modified or
## other actions may be added or removed inside the body of any handlers
## of this hook.
global policy: hook(trig: Trigger, f: fa_file);
## A table that can be used to disable file analysis completely for
## any files transferred over given network protocol analyzers.
const disable: table[AnalyzerTag] of bool = table() &redef;
@ -127,10 +121,9 @@ export {
const salt = "I recommend changing this." &redef;
## Postpones the timeout of file analysis for a given file.
## When used within a :bro:see:`FileAnalysis::policy` handler for
## :bro:see:`FileAnalysis::TRIGGER_TIMEOUT`, the analysis will delay
## timing out for the period of time indicated by the *timeout_interval*
## field of :bro:see:`fa_file`.
## When used within a :bro:see:`file_timeout` handler for, the analysis
## the analysis will delay timing out for the period of time indicated by
## the *timeout_interval* field of :bro:see:`fa_file`.
##
## f: the file.
##
@ -309,10 +302,8 @@ event bro_init() &priority=5
[$columns=Info, $ev=log_file_analysis]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_timeout(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
set_info(f);
f$info$timedout = T;
}

8
scripts/base/protocols/ftp/file-extract.bro
View file

@ -24,10 +24,8 @@ redef record Info += {
extract_file: bool &default=F;
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;
if ( ! f?$conns ) return;
@ -56,10 +54,8 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_type(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "FTP_DATA" ) return;

11
scripts/base/protocols/http/file-extract.bro
View file

@ -18,18 +18,15 @@ export {
extraction_file: string &log &optional;
## Indicates if the response body is to be extracted or not. Must be
## set before or by the first :bro:enum:`FileAnalysis::TRIGGER_NEW`
## for the file content.
## set before or by the first :bro:see:`file_new` for the file content.
extract_file: bool &default=F;
};
}
global extract_count: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_type(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
@ -56,10 +53,8 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;

4
scripts/base/protocols/http/file-hash.bro
View file

@ -23,10 +23,8 @@ export {
&redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_type(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;

10
scripts/base/protocols/http/file-ident.bro
View file

@ -34,10 +34,8 @@ export {
const ignored_incorrect_file_type_urls = /^$/ &redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_type(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
@ -68,16 +66,14 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_over_new_connection(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW_CONN ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "HTTP" ) return;
if ( ! f?$conns ) return;
# Spread the mime around (e.g. for partial content, TRIGGER_TYPE only
# Spread the mime around (e.g. for partial content, file_type event only
# happens once for the first connection, but if there's subsequent
# connections to transfer the same file, they'll be lacking the mime_type
# field if we don't do this).

12
scripts/base/protocols/irc/dcc-send.bro
View file

@ -41,10 +41,8 @@ global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins;
global extract_count: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
if ( ! f?$conns ) return;
@ -136,10 +134,8 @@ function log_dcc(f: fa_file)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_type(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;
@ -159,10 +155,8 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
set_dcc_extraction_file(f, fname);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=-5
event file_type(f: fa_file) &priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$source ) return;
if ( f$source != "IRC_DATA" ) return;

8
scripts/base/protocols/smtp/entities-excerpt.bro
View file

@ -16,10 +16,8 @@ export {
const default_entity_excerpt_len = 0 &redef;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
@ -27,10 +25,8 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
f$bof_buffer_size = default_entity_excerpt_len;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_bof_buffer(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_BOF_BUFFER ) return;
if ( ! f?$bof_buffer ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;

8
scripts/base/protocols/smtp/entities.bro
View file

@ -95,10 +95,8 @@ event mime_begin_entity(c: connection) &priority=10
set_session(c, T);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;
if ( ! f?$conns ) return;
@ -165,10 +163,8 @@ function check_md5_by_type(f: fa_file)
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_MD5]);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_type(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
if ( ! f?$mime_type ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;

4
scripts/policy/frameworks/intel/smtp-url-extraction.bro
View file

@ -21,10 +21,8 @@ event intel_mime_data(f: fa_file, data: string)
}
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=5
event file_new(f: fa_file) &priority=5
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
if ( ! f?$source ) return;
if ( f$source != "SMTP" ) return;

1
src/Event.cc
View file

@ -62,7 +62,6 @@ EventMgr::EventMgr()
current_aid = 0;
src_val = 0;
draining = 0;
bypass_queue = false;
}
EventMgr::~EventMgr()

1
src/Event.h
View file

@ -128,7 +128,6 @@ protected:
TimerMgr* current_mgr;
RecordVal* src_val;
bool draining;
bool bypass_queue;
};
extern EventMgr mgr;

13
src/event.bif
View file

@ -6997,6 +6997,19 @@ event bro_script_loaded%(path: string, level: count%);
## .. bro:see:: return_file_handle
event get_file_handle%(tag: count, c: connection, is_orig: bool%);
# TODO: document
event file_new%(f: fa_file%);
# TODO: give the new connection
event file_over_new_connection%(f: fa_file%);
event file_timeout%(f: fa_file%);
event file_bof%(f: fa_file%);
# TODO: give buffer? (unless we remove the event completely)
event file_bof_buffer%(f: fa_file%);
# TODO: give mime type? (unless we remove the event completely)
event file_type%(f: fa_file%);
# TODO: give size of gap
event file_gap%(f: fa_file%);
## This event is generated each time file analysis is ending for a given file.
##
## f: The file.

28
src/file_analysis/File.cc
View file

@ -147,8 +147,7 @@ void File::UpdateConnectionFields(Connection* conn)
{
conns->AsTableVal()->Assign(idx, conn->BuildConnVal());
if ( ! is_first )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW_CONN,
this);
file_mgr->FileEvent(file_over_new_connection, this);
}
Unref(idx);
@ -223,22 +222,10 @@ bool File::BufferBOF(const u_char* data, uint64 len)
if ( bof_buffer.full || bof_buffer.replayed ) return false;
if ( bof_buffer.chunks.size() == 0 )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_BOF, this);
file_mgr->FileEvent(file_bof, this);
uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
/* Leaving out this optimization (I think) for now to keep things simpler.
// If first chunk satisfies desired size, do everything now without copying.
if ( bof_buffer.chunks.empty() && len >= desired_size )
{
bof_buffer.full = bof_buffer.replayed = true;
val->Assign(bof_buffer_idx, new StringVal(new BroString(data, len, 0)));
file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
// TODO: libmagic stuff
return false;
}
*/
bof_buffer.chunks.push_back(new BroString(data, len, 0));
bof_buffer.size += len;
@ -281,11 +268,10 @@ void File::ReplayBOF()
val->Assign(bof_buffer_idx, new StringVal(bs));
bool have_type = DetectTypes(bs->Bytes(), bs->Len());
using BifEnum::FileAnalysis::TRIGGER_BOF_BUFFER;
file_mgr->EvaluatePolicy(TRIGGER_BOF_BUFFER, this);
file_mgr->FileEvent(file_bof_buffer, this);
if ( have_type )
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
file_mgr->FileEvent(file_type, this);
for ( size_t i = 0; i < bof_buffer.chunks.size(); ++i )
DataIn(bof_buffer.chunks[i]->Bytes(), bof_buffer.chunks[i]->Len());
@ -299,7 +285,7 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset)
{
if ( DetectTypes(data, len) )
{
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
file_mgr->FileEvent(file_type, this);
actions.DrainModifications();
}
@ -338,7 +324,7 @@ void File::DataIn(const u_char* data, uint64 len)
{
if ( DetectTypes(data, len) )
{
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TYPE, this);
file_mgr->FileEvent(file_type, this);
actions.DrainModifications();
}
@ -409,7 +395,7 @@ void File::Gap(uint64 offset, uint64 len)
actions.QueueRemoveAction(act->Args());
}
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_GAP, this);
file_mgr->FileEvent(file_gap, this);
actions.DrainModifications();
IncrementByteCount(len, missing_bytes_idx);

20
src/file_analysis/Manager.cc
View file

@ -174,22 +174,6 @@ void Manager::FileEvent(EventHandlerPtr h, File* file)
mgr.Dispatch(new Event(h, vl));
}
void Manager::EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, File* file)
{
if ( IsIgnored(file->GetUnique()) ) return;
const ID* id = global_scope()->Lookup("FileAnalysis::policy");
assert(id);
const Func* hook = id->ID_Val()->AsFunc();
val_list vl(2);
vl.append(new EnumVal(t, BifType::Enum::FileAnalysis::Trigger));
vl.append(file->val->Ref());
Val* result = hook->Call(&vl);
Unref(result);
}
bool Manager::PostponeTimeout(const FileID& file_id) const
{
File* file = Lookup(file_id);
@ -237,7 +221,7 @@ File* Manager::GetFile(const string& unique, Connection* conn,
}
id_map[id] = rval;
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_NEW, rval);
FileEvent(file_new, rval);
rval->ScheduleInactivityTimer();
if ( IsIgnored(unique) ) return 0;
}
@ -267,7 +251,7 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating)
file->postpone_timeout = false;
file_mgr->EvaluatePolicy(BifEnum::FileAnalysis::TRIGGER_TIMEOUT, file);
FileEvent(file_timeout, file);
if ( file->postpone_timeout && ! is_terminating )
{

9
src/file_analysis/Manager.h
View file

@ -101,8 +101,8 @@ public:
bool IgnoreFile(const FileID& file_id);
/**
* If called during \c FileAnalysis::policy evaluation for a
* \c FileAnalysis::TRIGGER_TIMEOUT, requests deferral of analysis timeout.
* If called during a \c file_timeout event handler, requests deferral of
* analysis timeout.
*/
bool PostponeTimeout(const FileID& file_id) const;
@ -120,11 +120,6 @@ public:
*/
bool RemoveAction(const FileID& file_id, const RecordVal* args) const;
/**
* Calls the \c FileAnalysis::policy hook.
*/
void EvaluatePolicy(BifEnum::FileAnalysis::Trigger t, File* file);
/**
* Dispatches an event related to the file's life-cycle.
*/

36
src/types.bif
View file

@ -231,42 +231,6 @@ module FileAnalysis;
type ActionArgs: record;
## An enumeration of significant things that can occur over the course of
## analyzing files. The :bro:see:`FileAnalysis::policy` hook is called each
## time a trigger occurs.
enum Trigger %{
## Raised when any part of a new file is detected.
TRIGGER_NEW,
## Raised when file is detected being transported over a new network
## connection (other than the first).
TRIGGER_NEW_CONN,
## Raised when file analysis for a given file is aborted due
## to not seeing any data for it recently. Note that this doesn't
## necessarily mean the full file wasn't seen (e.g. if the
## :bro:see:`fa_file` record indicates the file *total_bytes*
## isn't known). Use :bro:see:`FileAnalysis::postpone_timeout`
## during a :bro:see:`FileAnalysis::policy` handler for this trigger to
## defer the timeout until later.
TRIGGER_TIMEOUT,
## Raised when the beginning of a file is detected.
TRIGGER_BOF,
## Raised when the beginning of a file is available in the *bof_buffer*
## field of :bro:see:`fa_file` and that beginning
## is at least the number of bytes indicated by the *bof_buffer_size* field.
TRIGGER_BOF_BUFFER,
## Raised when an initial guess at the file/mime type of a file is matched.
TRIGGER_TYPE,
## Raised when there's a missing chunk of data in the file stream.
TRIGGER_GAP,
%}
## An enumeration of various file analysis actions that can be taken.
enum Action %{

6
testing/btest/Baseline/istate.events-ssl/sender.http.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#open 2013-03-22-21-05-55
#open 2013-04-10-15-49-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-03-22-21-05-56
1365608977.146651 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-04-10-15-49-38

6
testing/btest/Baseline/istate.events/sender.http.log
View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#open 2013-03-22-21-03-17
#open 2013-04-10-15-48-08
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-03-22-21-03-18
1365608887.935644 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-04-10-15-48-09

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.actions.data_event/out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
file_stream, Cx92a0ym5R8, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea

12
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.postpone_timeout/bro..stdout
View file

@ -1,6 +1,6 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -8,13 +8,13 @@ oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
source: HTTP
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_TIMEOUT
FILE_TIMEOUT
FILE_TIMEOUT
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.remove_action/get.out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

2
testing/btest/Baseline/scripts.base.frameworks.file-analysis.bifs.stop/get.out
View file

@ -1,2 +1,2 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.ftp/out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
sidhzrR4IT8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
The Nationa
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get-gzip.out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
kg59rqyYxN, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
{^J "origin
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.get/get.out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

6
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/a.out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
7gZBKVUgy4l, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FILE_OVER_NEW_CONNECTION
FILE_STATE_REMOVE
7gZBKVUgy4l, 555523, 0
[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]

10
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/b.out
View file

@ -1,6 +1,6 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -8,12 +8,12 @@ oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
source: HTTP
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FILE_TIMEOUT
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]

6
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.partial-content/c.out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
uHS14uhRKGe, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FILE_OVER_NEW_CONNECTION
FILE_STATE_REMOVE
uHS14uhRKGe, 498702, 0
[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]

40
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.pipeline/out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
aFQKI8SPOL2, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
/*^J********
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -13,12 +13,12 @@ source: HTTP
MD5: b932c3310ce47e158d1a5a42e0b01279
SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42
FileAnalysis::TRIGGER_NEW
FILE_NEW
CCU3vUEr06l, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
//-- Google
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -28,12 +28,12 @@ source: HTTP
MD5: e732f7bf1d7cb4eedcb1661697d7bc8c
SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da
FileAnalysis::TRIGGER_NEW
FILE_NEW
HCzA0dVwDPj, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
GIF89a^D\0^D\0\xb3
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -44,12 +44,12 @@ source: HTTP
MD5: d903de7e30db1691d3130ba5eae6b9a7
SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315
FileAnalysis::TRIGGER_NEW
FILE_NEW
a1Zu1fteVEf, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
\x89PNG^M^J^Z^J\0\0\0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -60,12 +60,12 @@ source: HTTP
MD5: e0029eea80812e9a8e57b8d05d52938a
SHA1: 560eab5a0177246827a94042dd103916d8765ac7
SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4
FileAnalysis::TRIGGER_NEW
FILE_NEW
xXlF7wFdsR, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
\x89PNG^M^J^Z^J\0\0\0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

16
testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.post/out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
v5HLI7MxPQh, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
hello world
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -14,12 +14,12 @@ source: HTTP
MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3
SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
FileAnalysis::TRIGGER_NEW
FILE_NEW
PZS1XGHkIf1, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
{^J "origin
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.input.basic/bro..stdout
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
nYgPNGLrZf9, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
#separator
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

8
testing/btest/Baseline/scripts.base.frameworks.file-analysis.irc/out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
wqKMAamJVSb, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
PK^C^D^T\0\0\0^H\0\xae
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

24
testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out
View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
cwR7l6Zctxb, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
Hello^M^J^M^J ^M
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -13,12 +13,12 @@ source: SMTP
MD5: 92bca2e6cdcde73647125da7dccbdd07
SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
FileAnalysis::TRIGGER_NEW
FILE_NEW
ZAOEQmRyxv1, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
<html xmlns
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -28,12 +28,12 @@ source: SMTP
MD5: d194c6359c85bb88b54caee18b1e9b44
SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
SHA256: b9556e92ddbe52379b64804136f830d111cafe7fcd78e54817fe40f3bc24268d
FileAnalysis::TRIGGER_NEW
FILE_NEW
Ltd7QO7jEv3, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
Version 4.9
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

13
testing/btest/istate/events-ssl.bro
View file

@ -41,10 +41,13 @@ redef ssl_ca_certificate = "../ca_cert.pem";
redef ssl_private_key = "../bro.pem";
redef ssl_passphrase = "my-password";
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=10
# File-analysis fields in http.log won't get set on receiver side correctly,
# one problem is with the way serialization may send a unique ID in place
# of a full value and expect the remote side to associate that unique ID with
# a value it received at an earlier time. So sometimes modifications the sender
# makes to the value aren't seen on the receiver (in this case, the mime_type
# field).
event file_new(f: fa_file)
{
FileAnalysis::stop(f);
}
@ -63,7 +66,7 @@ event bro_init()
redef peer_description = "events-rcv";
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T, $retry=1sec]
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $ssl=T, $retry=1sec]
};
redef ssl_ca_certificate = "../ca_cert.pem";

12
testing/btest/istate/events.bro
View file

@ -36,10 +36,12 @@ redef peer_description = "events-send";
# it gets propagated but that's ok.)
redef tcp_close_delay = 0secs;
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=10
# File-analysis fields in http.log won't get set on receiver side correctly,
# one problem is with the way serialization may send a unique ID in place
# of a full value and expect the remote side to associate that unique ID with
# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type
# field).
event file_new(f: fa_file)
{
FileAnalysis::stop(f);
}
@ -58,7 +60,7 @@ event bro_init()
redef peer_description = "events-rcv";
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $retry=1sec]
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $retry=1sec]
};
event remote_connection_closed(p: event_peer)

10
testing/btest/scripts/base/frameworks/file-analysis/bifs/postpone_timeout.bro
View file

@ -16,17 +16,13 @@ redef test_get_file_name = function(f: fa_file): string
redef exit_only_after_terminate = T;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_new(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
f$timeout_interval=2sec;
f$timeout_interval = 2sec;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_timeout(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
if ( timeout_cnt < 1 )
FileAnalysis::postpone_timeout(f);
else

3
testing/btest/scripts/base/frameworks/file-analysis/bifs/remove_action.bro
View file

@ -8,9 +8,8 @@ redef test_get_file_name = function(f: fa_file): string
return fmt("%s-file", f$id);
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_type(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
for ( act in test_file_actions )
FileAnalysis::remove_action(f, act);
local filename = test_get_file_name(f);

3
testing/btest/scripts/base/frameworks/file-analysis/bifs/stop.bro
View file

@ -2,8 +2,7 @@
# @TEST-EXEC: btest-diff get.out
# @TEST-EXEC: test ! -s Cx92a0ym5R8-file
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_new(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
FileAnalysis::stop(f);
}

71
testing/scripts/file-analysis-test.bro
View file

@ -20,12 +20,10 @@ event file_stream(f: fa_file, data: string)
print "file_stream", f$id, |data|, data;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_new(f: fa_file)
{
print trig;
print "FILE_NEW";
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print f$id, f$seen_bytes, f$missing_bytes;
if ( test_file_analysis_source == "" ||
@ -36,30 +34,59 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
local filename: string = test_get_file_name(f);
if ( filename != "" )
FileAnalysis::add_action(f,
[$act=FileAnalysis::ACTION_EXTRACT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_action(f,
[$act=FileAnalysis::ACTION_DATA_EVENT,
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
}
break;
}
case FileAnalysis::TRIGGER_BOF_BUFFER:
event file_over_new_connection(f: fa_file)
{
print "FILE_OVER_NEW_CONNECTION";
}
event file_timeout(f: fa_file)
{
print "FILE_TIMEOUT";
}
event file_bof(f: fa_file)
{
print "FILE_BOF";
}
event file_bof_buffer(f: fa_file)
{
print "FILE_BOF_BUFFER";
if ( f?$bof_buffer )
print f$bof_buffer[0:10];
break;
}
case FileAnalysis::TRIGGER_TYPE:
event file_type(f: fa_file) &priority=-5
{
}
event file_type(f: fa_file)
{
print "FILE_TYPE";
# not actually printing the values due to libmagic variances
if ( f?$file_type )
{
print "file type is set";
if ( f?$mime_type )
print "mime type is set";
break;
f$file_type = "set";
}
if ( f?$mime_type )
{
print "mime type is set";
f$mime_type = "set";
}
}
event file_gap(f: fa_file)
{
print "FILE_GAP";
}
event file_state_remove(f: fa_file)
@ -85,18 +112,6 @@ event file_state_remove(f: fa_file)
print fmt("SHA256: %s", f$info$sha256);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
# avoids libmagic variances across systems
if ( f?$mime_type )
f$mime_type = "set";
if ( f?$file_type )
f$file_type = "set";
}
event bro_init()
{
add test_file_actions[[$act=FileAnalysis::ACTION_MD5]];