Mirror/zeek
1
0
Fork 0
mirror of https://github.com/zeek/zeek.git synced 2025-10-10 02:28:21 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 3

FileAnalysis: add unit tests covering current protocol integration.

Browse source 
And had to make various fixes/refinements after scrutinizing results.
This commit is contained in:
Jon Siwek 2013-03-19 15:50:05 -05:00
parent b30211c178
commit 59ed5c75f1
72 changed files with 2605 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

68
testing/btest/scripts/base/frameworks/file-analysis/ftp.bro Normal file
View file

@ -0,0 +1,68 @@
# @TEST-EXEC: bro -r $TRACES/ftp/retr.trace %INPUT >out
# @TEST-EXEC: btest-diff out
# @TEST-EXEC: btest-diff thefile
global actions: set[FileAnalysis::ActionArgs];
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "ftp-data" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename="thefile"]];
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

76
testing/btest/scripts/base/frameworks/file-analysis/http/get.bro Normal file
View file

@ -0,0 +1,76 @@
# @TEST-EXEC: bro -r $TRACES/http/get.trace %INPUT >get.out
# @TEST-EXEC: bro -r $TRACES/http/get-gzip.trace %INPUT >get-gzip.out
# @TEST-EXEC: btest-diff get.out
# @TEST-EXEC: btest-diff get-gzip.out
# @TEST-EXEC: btest-diff KPVibShQgUc-file
# @TEST-EXEC: btest-diff LMA6EHLacYc-file
global actions: set[FileAnalysis::ActionArgs];
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "HTTP" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
local filename: string = fmt("%s-file", info$file_id);
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
fallthrough;
case FileAnalysis::TRIGGER_DONE:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

89
testing/btest/scripts/base/frameworks/file-analysis/http/partial-content.bro Normal file
View file

@ -0,0 +1,89 @@
# @TEST-EXEC: bro -r $TRACES/http/206_example_a.pcap %INPUT >a.out
# @TEST-EXEC: btest-diff a.out
# @TEST-EXEC: wc -c uj9AtyGOiZ8-file0 >a.size
# @TEST-EXEC: btest-diff a.size
# @TEST-EXEC: bro -r $TRACES/http/206_example_b.pcap %INPUT >b.out
# @TEST-EXEC: btest-diff b.out
# @TEST-EXEC: wc -c ns7As4DOZcj-file0 >b.size
# @TEST-EXEC: btest-diff b.size
# @TEST-EXEC: bro -r $TRACES/http/206_example_c.pcap %INPUT >c.out
# @TEST-EXEC: btest-diff c.out
# @TEST-EXEC: wc -c MHMkq2nFxej-file0 >c.size
# @TEST-EXEC: btest-diff c.size
global actions: set[FileAnalysis::ActionArgs];
global cnt: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "HTTP" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
local filename: string = fmt("%s-file%d", info$file_id, cnt);
++cnt;
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
fallthrough;
case FileAnalysis::TRIGGER_DONE:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
if ( info$actions[act]?$md5 )
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
if ( info$actions[act]?$sha1 )
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
if ( info$actions[act]?$sha256 )
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

77
testing/btest/scripts/base/frameworks/file-analysis/http/pipeline.bro Normal file
View file

@ -0,0 +1,77 @@
# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT >out
# @TEST-EXEC: btest-diff out
# @TEST-EXEC: btest-diff Z3kT1FyLnfk-file
# @TEST-EXEC: btest-diff WLJWC1FMBq9-file
# @TEST-EXEC: btest-diff Ac8PLL9KL49-file
# @TEST-EXEC: btest-diff NV2MvAX0Is4-file
# @TEST-EXEC: btest-diff YLndcRpw5Ge-file
global actions: set[FileAnalysis::ActionArgs];
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "HTTP" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
local filename: string = fmt("%s-file", info$file_id);
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
fallthrough;
case FileAnalysis::TRIGGER_DONE:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

74
testing/btest/scripts/base/frameworks/file-analysis/http/post.bro Normal file
View file

@ -0,0 +1,74 @@
# @TEST-EXEC: bro -r $TRACES/http/post.trace %INPUT >out
# @TEST-EXEC: btest-diff out
# @TEST-EXEC: btest-diff WDJLxTGN0m8-file
# @TEST-EXEC: btest-diff LkolCF6OeHh-file
global actions: set[FileAnalysis::ActionArgs];
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "HTTP" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
local filename: string = fmt("%s-file", info$file_id);
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
fallthrough;
case FileAnalysis::TRIGGER_DONE:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

68
testing/btest/scripts/base/frameworks/file-analysis/irc.bro Normal file
View file

@ -0,0 +1,68 @@
# @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT >out
# @TEST-EXEC: btest-diff out
# @TEST-EXEC: btest-diff thefile
global actions: set[FileAnalysis::ActionArgs];
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "irc-dcc-data" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename="thefile"]];
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

74
testing/btest/scripts/base/frameworks/file-analysis/smtp.bro Normal file
View file

@ -0,0 +1,74 @@
# @TEST-EXEC: bro -r $TRACES/smtp.trace %INPUT >out
# @TEST-EXEC: btest-diff out
# @TEST-EXEC: btest-diff thefile0
# @TEST-EXEC: btest-diff thefile1
# @TEST-EXEC: btest-diff thefile2
global actions: set[FileAnalysis::ActionArgs];
global cnt: count = 0;
hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info)
{
print trig;
switch ( trig ) {
case FileAnalysis::TRIGGER_NEW:
print info$file_id, info$seen_bytes, info$missing_bytes;
if ( info$source == "SMTP" )
{
for ( act in actions )
FileAnalysis::add_action(info$file_id, act);
local filename: string = fmt("thefile%d", cnt);
++cnt;
FileAnalysis::add_action(info$file_id,
[$act=FileAnalysis::ACTION_EXTRACT,
$extract_filename=filename]);
}
break;
case FileAnalysis::TRIGGER_BOF_BUFFER:
if ( info?$bof_buffer )
print info$bof_buffer[0:10];
break;
case FileAnalysis::TRIGGER_TYPE:
# not actually printing the values due to libmagic variances
if ( info?$file_type )
print "file type is set";
if ( info?$mime_type )
print "mime type is set";
break;
case FileAnalysis::TRIGGER_EOF:
print info$file_id, info$seen_bytes, info$missing_bytes;
print info$conn_uids;
print info$conn_ids;
if ( info?$total_bytes )
print "total bytes: " + fmt("%s", info$total_bytes);
if ( info?$source )
print "source: " + info$source;
for ( act in info$actions )
switch ( act$act ) {
case FileAnalysis::ACTION_MD5:
print fmt("MD5: %s", info$actions[act]$md5);
break;
case FileAnalysis::ACTION_SHA1:
print fmt("SHA1: %s", info$actions[act]$sha1);
break;
case FileAnalysis::ACTION_SHA256:
print fmt("SHA256: %s", info$actions[act]$sha256);
break;
}
break;
}
}
event bro_init()
{
add actions[[$act=FileAnalysis::ACTION_MD5]];
add actions[[$act=FileAnalysis::ACTION_SHA1]];
add actions[[$act=FileAnalysis::ACTION_SHA256]];
}

2
testing/btest/scripts/base/protocols/conn/contents-default-extract.test
View file

@ -1,3 +1,3 @@
# @TEST-EXEC: bro -f "tcp port 21" -r $TRACES/ipv6-ftp.trace "Conn::default_extract=T"
# @TEST-EXEC: bro -f "tcp port 21" -r $TRACES/ftp/ipv6.trace "Conn::default_extract=T"
# @TEST-EXEC: btest-diff contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_orig.dat
# @TEST-EXEC: btest-diff contents_[2001:470:1f11:81f:c999:d94:aa7c:2e3e]:49185-[2001:470:4867:99::21]:21_resp.dat

4
testing/btest/scripts/base/protocols/conn/polling.test
View file

@ -1,6 +1,6 @@
# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT >out1
# @TEST-EXEC: bro -b -r $TRACES/http/100-continue.trace %INPUT >out1
# @TEST-EXEC: btest-diff out1
# @TEST-EXEC: bro -b -r $TRACES/http-100-continue.trace %INPUT stop_cnt=2 >out2
# @TEST-EXEC: bro -b -r $TRACES/http/100-continue.trace %INPUT stop_cnt=2 >out2
# @TEST-EXEC: btest-diff out2
@load base/protocols/conn

2
testing/btest/scripts/base/protocols/ftp/ftp-ipv4.bro
View file

@ -1,6 +1,6 @@
# This tests both active and passive FTP over IPv4.
#
# @TEST-EXEC: bro -r $TRACES/ftp-ipv4.trace
# @TEST-EXEC: bro -r $TRACES/ftp/ipv4.trace
# @TEST-EXEC: btest-diff conn.log
# @TEST-EXEC: btest-diff ftp.log

2
testing/btest/scripts/base/protocols/ftp/ftp-ipv6.bro
View file

@ -1,6 +1,6 @@
# This tests both active and passive FTP over IPv6.
#
# @TEST-EXEC: bro -r $TRACES/ipv6-ftp.trace
# @TEST-EXEC: bro -r $TRACES/ftp/ipv6.trace
# @TEST-EXEC: btest-diff conn.log
# @TEST-EXEC: btest-diff ftp.log

2
testing/btest/scripts/base/protocols/http/100-continue.bro
View file

@ -3,7 +3,7 @@
# a given request. The http scripts should also be able log such replies
# in a way that correlates the final response with the request.
#
# @TEST-EXEC: bro -r $TRACES/http-100-continue.trace %INPUT
# @TEST-EXEC: bro -r $TRACES/http/100-continue.trace %INPUT
# @TEST-EXEC: test ! -f weird.log
# @TEST-EXEC: btest-diff http.log

2
testing/btest/scripts/base/protocols/http/http-header-crlf.bro
View file

@ -2,7 +2,7 @@
# it gets confused whether it's in a header or not; it shouldn't report
# the http_no_crlf_in_header_list wierd.
#
# @TEST-EXEC: bro -r $TRACES/http-byteranges.trace %INPUT
# @TEST-EXEC: bro -r $TRACES/http/byteranges.trace %INPUT
# @TEST-EXEC: test ! -f weird.log
# The base analysis scripts are loaded by default.

2
testing/btest/scripts/base/protocols/http/http-methods.bro
View file

@ -1,6 +1,6 @@
# This tests that the HTTP analyzer handles strange HTTP methods properly.
#
# @TEST-EXEC: bro -r $TRACES/http-methods.trace %INPUT
# @TEST-EXEC: bro -r $TRACES/http/methods.trace %INPUT
# @TEST-EXEC: btest-diff weird.log
# @TEST-EXEC: btest-diff http.log

2
testing/btest/scripts/base/protocols/http/http-mime-and-md5.bro
View file

@ -2,7 +2,7 @@
# will normalize mime types other than the target type to prevent sensitivity
# to varying versions of libmagic.
# @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output
# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output
# @TEST-EXEC: btest-diff http.log
redef HTTP::generate_md5 += /image\/png/;

2
testing/btest/scripts/base/protocols/http/http-pipelining.bro
View file

@ -1,4 +1,4 @@
# @TEST-EXEC: bro -r $TRACES/http-pipelined-requests.trace %INPUT > output
# @TEST-EXEC: bro -r $TRACES/http/pipelined-requests.trace %INPUT > output
# @TEST-EXEC: btest-diff http.log
# mime type is irrelevant to this test, so filter it out