FileAnalysis: finish switching hooks to events.

This commit is contained in:
Jon Siwek 2013-04-10 11:13:43 -05:00
parent 641154f8e8
commit a2d9b47bcd
39 changed files with 219 additions and 305 deletions

View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#open 2013-03-22-21-05-55
#open 2013-04-10-15-49-37
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
1363986354.505533 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-03-22-21-05-56
1365608977.146651 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-04-10-15-49-38

View file

@ -3,8 +3,8 @@
#empty_field (empty)
#unset_field -
#path http
#open 2013-03-22-21-03-17
#open 2013-04-10-15-48-08
#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extraction_file
#types time string addr port addr port count string string string string string count count count string count string string table[enum] string string table[string] string string string
1363986197.076696 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-03-22-21-03-18
1365608887.935644 arKYeMETxOg 141.42.64.125 56730 125.190.109.199 80 1 GET www.icir.org / - Wget/1.10 0 9130 200 OK - - - (empty) - - - - - -
#close 2013-04-10-15-48-09

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
file_stream, Cx92a0ym5R8, 1500, ^J0.26 | 2012-08-24 15:10:04 -0700^J^J * Fixing update-changes, which could pick the wrong control file. (Robin Sommer)^J^J * Fixing GPG signing script. (Robin Sommer)^J^J0.25 | 2012-08-01 13:55:46 -0500^J^J * Fix configure script to exit with non-zero status on error (Jon Siwek)^J^J0.24 | 2012-07-05 12:50:43 -0700^J^J * Raise minimum required CMake version to 2.6.3 (Jon Siwek)^J^J * Adding script to delete old fully-merged branches. (Robin Sommer)^J^J0.23-2 | 2012-01-25 13:24:01 -0800^J^J * Fix a bro-cut error message. (Daniel Thayer)^J^J0.23 | 2012-01-11 12:16:11 -0800^J^J * Tweaks to release scripts, plus a new one for signing files.^J (Robin Sommer)^J^J0.22 | 2012-01-10 16:45:19 -0800^J^J * Tweaks for OpenBSD support. (Jon Siwek)^J^J * bro-cut extensions and fixes. (Robin Sommer)^J ^J - If no field names are given on the command line, we now pass through^J all fields. Adresses #657.^J^J - Removing some GNUism from awk script. Addresses #653.^J^J - Added option for time output in UTC. Addresses #668.^J^J - Added output field separator option -F. Addresses #649.^J^J - Fixing option -c: only some header lines were passed through^J rather than all. (Robin Sommer)^J^J * Fix parallel make portability. (Jon Siwek)^J^J0.21-9 | 2011-11-07 05:44:14 -0800^J^J * Fixing compiler warnings. Addresses #388. (Jon Siwek)^J^J0.21-2 | 2011-11-02 18:12:13 -0700^J^J * Fix for misnaming temp file in update-changes script. (Robin Sommer)^J^J0.21-1 | 2011-11-02 18:10:39 -0700^J^J * Little fix for make-relea

View file

@ -1,6 +1,6 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -8,13 +8,13 @@ oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
source: HTTP
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FileAnalysis::TRIGGER_TIMEOUT
FILE_TIMEOUT
FILE_TIMEOUT
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,2 +1,2 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
sidhzrR4IT8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
The Nationa
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
kg59rqyYxN, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
{^J "origin
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
Cx92a0ym5R8, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
^J0.26 | 201
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
7gZBKVUgy4l, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FILE_OVER_NEW_CONNECTION
FILE_STATE_REMOVE
7gZBKVUgy4l, 555523, 0
[orig_h=10.101.84.70, orig_p=10978/tcp, resp_h=129.174.93.161, resp_p=80/tcp]

View file

@ -1,6 +1,6 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -8,12 +8,12 @@ oDwT1BbzjM1, 1022920, 0
[orig_h=192.168.72.14, orig_p=3254/tcp, resp_h=65.54.95.206, resp_p=80/tcp]
total bytes: 1022920
source: HTTP
FileAnalysis::TRIGGER_NEW
FILE_NEW
oDwT1BbzjM1, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_TIMEOUT
FILE_TIMEOUT
FILE_STATE_REMOVE
oDwT1BbzjM1, 206024, 0
[orig_h=192.168.72.14, orig_p=3257/tcp, resp_h=65.54.95.14, resp_p=80/tcp]

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
uHS14uhRKGe, 0, 0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FileAnalysis::TRIGGER_NEW_CONN
FILE_OVER_NEW_CONNECTION
FILE_STATE_REMOVE
uHS14uhRKGe, 498702, 0
[orig_h=10.45.179.94, orig_p=19950/tcp, resp_h=129.174.93.170, resp_p=80/tcp]

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
aFQKI8SPOL2, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
/*^J********
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -13,12 +13,12 @@ source: HTTP
MD5: b932c3310ce47e158d1a5a42e0b01279
SHA1: 0e42ae17eea9b074981bd3a34535ad3a22d02706
SHA256: 5b037a2c5e36f56e63a3012c73e46a04b27741d8ff8f8b62c832fb681fc60f42
FileAnalysis::TRIGGER_NEW
FILE_NEW
CCU3vUEr06l, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
//-- Google
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -28,12 +28,12 @@ source: HTTP
MD5: e732f7bf1d7cb4eedcb1661697d7bc8c
SHA1: 8f241117afaa8ca5f41dc059e66d75c283dcc983
SHA256: 6a509fd05aa7c8fa05080198894bb19e638554ffcee0e0b3d7bc8ff54afee1da
FileAnalysis::TRIGGER_NEW
FILE_NEW
HCzA0dVwDPj, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
GIF89a^D\0^D\0\xb3
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -44,12 +44,12 @@ source: HTTP
MD5: d903de7e30db1691d3130ba5eae6b9a7
SHA1: 81f5f056ce5e97d940854bb0c48017b45dd9f15e
SHA256: 6fb22aa9d780ea63bd7a2e12b92b16fcbf1c4874f1d3e11309a5ba984433c315
FileAnalysis::TRIGGER_NEW
FILE_NEW
a1Zu1fteVEf, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
\x89PNG^M^J^Z^J\0\0\0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -60,12 +60,12 @@ source: HTTP
MD5: e0029eea80812e9a8e57b8d05d52938a
SHA1: 560eab5a0177246827a94042dd103916d8765ac7
SHA256: e0b4500c1fd1d675da4137461cbe64d3c8489f4180d194e47683b20e7fb876f4
FileAnalysis::TRIGGER_NEW
FILE_NEW
xXlF7wFdsR, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
\x89PNG^M^J^Z^J\0\0\0
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
v5HLI7MxPQh, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
hello world
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -14,12 +14,12 @@ source: HTTP
MD5: 5eb63bbbe01eeed093cb22bb8f5acdc3
SHA1: 2aae6c35c94fcfb415dbe95f408b9ce91ee846ed
SHA256: b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9
FileAnalysis::TRIGGER_NEW
FILE_NEW
PZS1XGHkIf1, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
{^J "origin
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
nYgPNGLrZf9, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
#separator
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
wqKMAamJVSb, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
PK^C^D^T\0\0\0^H\0\xae
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -1,9 +1,9 @@
FileAnalysis::TRIGGER_NEW
FILE_NEW
cwR7l6Zctxb, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
Hello^M^J^M^J ^M
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -13,12 +13,12 @@ source: SMTP
MD5: 92bca2e6cdcde73647125da7dccbdd07
SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13
SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5
FileAnalysis::TRIGGER_NEW
FILE_NEW
ZAOEQmRyxv1, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
<html xmlns
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE
@ -28,12 +28,12 @@ source: SMTP
MD5: d194c6359c85bb88b54caee18b1e9b44
SHA1: e54af6c6616525611364b80bd6557a7ea21dae94
SHA256: b9556e92ddbe52379b64804136f830d111cafe7fcd78e54817fe40f3bc24268d
FileAnalysis::TRIGGER_NEW
FILE_NEW
Ltd7QO7jEv3, 0, 0
FileAnalysis::TRIGGER_BOF
FileAnalysis::TRIGGER_BOF_BUFFER
FILE_BOF
FILE_BOF_BUFFER
Version 4.9
FileAnalysis::TRIGGER_TYPE
FILE_TYPE
file type is set
mime type is set
FILE_STATE_REMOVE

View file

@ -41,10 +41,13 @@ redef ssl_ca_certificate = "../ca_cert.pem";
redef ssl_private_key = "../bro.pem";
redef ssl_passphrase = "my-password";
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=10
# File-analysis fields in http.log won't get set on receiver side correctly,
# one problem is with the way serialization may send a unique ID in place
# of a full value and expect the remote side to associate that unique ID with
# a value it received at an earlier time. So sometimes modifications the sender
# makes to the value aren't seen on the receiver (in this case, the mime_type
# field).
event file_new(f: fa_file)
{
FileAnalysis::stop(f);
}
@ -63,7 +66,7 @@ event bro_init()
redef peer_description = "events-rcv";
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T, $retry=1sec]
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $ssl=T, $retry=1sec]
};
redef ssl_ca_certificate = "../ca_cert.pem";

View file

@ -36,10 +36,12 @@ redef peer_description = "events-send";
# it gets propagated but that's ok.)
redef tcp_close_delay = 0secs;
# File analysis that populates fields in the http.log would make the sender's
# log differ from the receiver's since hooks don't get sent to peers.
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=10
# File-analysis fields in http.log won't get set on receiver side correctly,
# one problem is with the way serialization may send a unique ID in place
# of a full value and expect the remote side to associate that unique ID with
# a value it received at an earlier time. So sometimes modifications the sender# makes to the value aren't seen on the receiver (in this case, the mime_type
# field).
event file_new(f: fa_file)
{
FileAnalysis::stop(f);
}
@ -58,7 +60,7 @@ event bro_init()
redef peer_description = "events-rcv";
redef Communication::nodes += {
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $retry=1sec]
["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match|file_.*/, $connect=T, $retry=1sec]
};
event remote_connection_closed(p: event_peer)

View file

@ -16,17 +16,13 @@ redef test_get_file_name = function(f: fa_file): string
redef exit_only_after_terminate = T;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_new(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
f$timeout_interval=2sec;
f$timeout_interval = 2sec;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_timeout(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TIMEOUT ) return;
if ( timeout_cnt < 1 )
FileAnalysis::postpone_timeout(f);
else

View file

@ -8,9 +8,8 @@ redef test_get_file_name = function(f: fa_file): string
return fmt("%s-file", f$id);
};
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_type(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
for ( act in test_file_actions )
FileAnalysis::remove_action(f, act);
local filename = test_get_file_name(f);

View file

@ -2,8 +2,7 @@
# @TEST-EXEC: btest-diff get.out
# @TEST-EXEC: test ! -s Cx92a0ym5R8-file
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_new(f: fa_file)
{
if ( trig != FileAnalysis::TRIGGER_NEW ) return;
FileAnalysis::stop(f);
}

View file

@ -20,46 +20,73 @@ event file_stream(f: fa_file, data: string)
print "file_stream", f$id, |data|, data;
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
event file_new(f: fa_file)
{
print trig;
print "FILE_NEW";
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print f$id, f$seen_bytes, f$missing_bytes;
print f$id, f$seen_bytes, f$missing_bytes;
if ( test_file_analysis_source == "" ||
f$source == test_file_analysis_source )
{
for ( act in test_file_actions )
FileAnalysis::add_action(f, act);
if ( test_file_analysis_source == "" ||
f$source == test_file_analysis_source )
{
for ( act in test_file_actions )
FileAnalysis::add_action(f, act);
local filename: string = test_get_file_name(f);
if ( filename != "" )
FileAnalysis::add_action(f,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_action(f,
[$act=FileAnalysis::ACTION_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( f?$bof_buffer )
print f$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( f?$file_type )
print "file type is set";
if ( f?$mime_type )
print "mime type is set";
break;
local filename: string = test_get_file_name(f);
if ( filename != "" )
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
FileAnalysis::add_action(f, [$act=FileAnalysis::ACTION_DATA_EVENT,
$chunk_event=file_chunk,
$stream_event=file_stream]);
}
}
event file_over_new_connection(f: fa_file)
{
print "FILE_OVER_NEW_CONNECTION";
}
event file_timeout(f: fa_file)
{
print "FILE_TIMEOUT";
}
event file_bof(f: fa_file)
{
print "FILE_BOF";
}
event file_bof_buffer(f: fa_file)
{
print "FILE_BOF_BUFFER";
if ( f?$bof_buffer )
print f$bof_buffer[0:10];
}
event file_type(f: fa_file) &priority=-5
{
}
event file_type(f: fa_file)
{
print "FILE_TYPE";
# not actually printing the values due to libmagic variances
if ( f?$file_type )
{
print "file type is set";
f$file_type = "set";
}
if ( f?$mime_type )
{
print "mime type is set";
f$mime_type = "set";
}
}
event file_gap(f: fa_file)
{
print "FILE_GAP";
}
event file_state_remove(f: fa_file)
@ -85,18 +112,6 @@ event file_state_remove(f: fa_file)
print fmt("SHA256: %s", f$info$sha256);
}
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, f: fa_file)
&priority=-5
{
if ( trig != FileAnalysis::TRIGGER_TYPE ) return;
# avoids libmagic variances across systems
if ( f?$mime_type )
f$mime_type = "set";
if ( f?$file_type )
f$file_type = "set";
}
event bro_init()
{
add test_file_actions[[$act=FileAnalysis::ACTION_MD5]];