From 27e47f0a57c253f56a48a163e1a3dc875b22ca57 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 27 Mar 2013 14:02:20 -0500 Subject: [PATCH] FileAnalysis: replace script-layer IRC file analysis. --- scripts/base/protocols/ftp/file-extract.bro | 2 +- scripts/base/protocols/ftp/main.bro | 4 +- scripts/base/protocols/irc/dcc-send.bro | 172 +++++++++++++----- .../scripts.base.protocols.irc.basic/irc.log | 6 +- ...4_1.dat => irc-dcc-item-wqKMAamJVSb-0.dat} | Bin .../irc.log | 8 +- .../base/protocols/irc/dcc-extract.test | 4 +- 7 files changed, 138 insertions(+), 58 deletions(-) rename testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/{irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat => irc-dcc-item-wqKMAamJVSb-0.dat} (100%) diff --git a/scripts/base/protocols/ftp/file-extract.bro b/scripts/base/protocols/ftp/file-extract.bro index e1e0044efb..b659e779a0 100644 --- a/scripts/base/protocols/ftp/file-extract.bro +++ b/scripts/base/protocols/ftp/file-extract.bro @@ -77,7 +77,7 @@ hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) } hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) - &priority=5 + &priority=-5 { if ( trig != FileAnalysis::TRIGGER_EOF && trig != FileAnalysis::TRIGGER_DONE ) return; diff --git a/scripts/base/protocols/ftp/main.bro b/scripts/base/protocols/ftp/main.bro index 94a3b1f222..817e00b188 100644 --- a/scripts/base/protocols/ftp/main.bro +++ b/scripts/base/protocols/ftp/main.bro @@ -359,9 +359,7 @@ event file_transferred(c: connection, prefix: string, descr: string, event connection_state_remove(c: connection) &priority=-5 { - local id = c$id; - if ( [id$resp_h, id$resp_p] in ftp_data_expected ) - delete ftp_data_expected[id$resp_h, id$resp_p]; + delete ftp_data_expected[c$id$resp_h, c$id$resp_p]; } # Use state remove event to cover connections terminated by RST. diff --git a/scripts/base/protocols/irc/dcc-send.bro b/scripts/base/protocols/irc/dcc-send.bro index 1f3e20cdaf..ead48debbb 100644 --- a/scripts/base/protocols/irc/dcc-send.bro +++ b/scripts/base/protocols/irc/dcc-send.bro @@ -28,69 +28,146 @@ export { dcc_file_size: count &log &optional; ## Sniffed mime type of the file. dcc_mime_type: string &log &optional; - + ## The file handle for the file to be extracted - extraction_file: file &log &optional; - + extraction_file: string &log &optional; + ## A boolean to indicate if the current file transfer should be extracted. extract_file: bool &default=F; - - ## The count of the number of file that have been extracted during the session. - num_extracted_files: count &default=0; }; } -global dcc_expected_transfers: table[addr, port] of Info = table(); +global dcc_expected_transfers: table[addr, port] of Info &read_expire=5mins; -event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) &priority=3 +global extract_count: count = 0; + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 { - local id = c$id; - if ( [id$resp_h, id$resp_p] !in dcc_expected_transfers ) - return; - - local irc = dcc_expected_transfers[id$resp_h, id$resp_p]; - - irc$dcc_mime_type = split1(mime_type, /;/)[1]; + if ( trig != FileAnalysis::TRIGGER_NEW ) return; + if ( ! info?$source ) return; + if ( info$source != "IRC_DATA" ) return; + if ( ! info?$conns ) return; - if ( extract_file_types == irc$dcc_mime_type ) + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + local extracting: bool = F; + + for ( cid in info$conns ) { - irc$extract_file = T; - } - - if ( irc$extract_file ) - { - local suffix = fmt("%d.dat", ++irc$num_extracted_files); - local fname = generate_extraction_filename(extraction_prefix, c, suffix); - irc$extraction_file = open(fname); + local c: connection = info$conns[cid]; + + if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; + + local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; + + if ( ! s$extract_file ) next; + + if ( ! extracting ) + { + FileAnalysis::add_action(info$file_id, + [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + extracting = T; + ++extract_count; + } + + s$extraction_file = fname; } } -event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) &priority=-4 +function set_dcc_mime(info: FileAnalysis::Info) { - local id = c$id; - if ( [id$resp_h, id$resp_p] !in dcc_expected_transfers ) + if ( ! info?$conns ) return; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; + + local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; + + s$dcc_mime_type = info$mime_type; + } + } + +function set_dcc_extraction_file(info: FileAnalysis::Info, filename: string) + { + if ( ! info?$conns ) return; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; + + local s = dcc_expected_transfers[cid$resp_h, cid$resp_p]; + + s$extraction_file = filename; + } + } + +function log_dcc(info: FileAnalysis::Info) + { + if ( ! info?$conns ) return; + + for ( cid in info$conns ) + { + local c: connection = info$conns[cid]; + + if ( [cid$resp_h, cid$resp_p] !in dcc_expected_transfers ) next; + + local irc = dcc_expected_transfers[cid$resp_h, cid$resp_p]; + + local tmp = irc$command; + irc$command = "DCC"; + Log::write(IRC::LOG, irc); + irc$command = tmp; + + # Delete these values in case another DCC transfer + # happens during the IRC session. + delete irc$extract_file; + delete irc$extraction_file; + delete irc$dcc_file_name; + delete irc$dcc_file_size; + delete irc$dcc_mime_type; + return; + } + } - local irc = dcc_expected_transfers[id$resp_h, id$resp_p]; +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=5 + { + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$mime_type ) return; + if ( ! info?$source ) return; + if ( info$source != "IRC_DATA" ) return; - local tmp = irc$command; - irc$command = "DCC"; - Log::write(IRC::LOG, irc); - irc$command = tmp; + set_dcc_mime(info); - if ( irc?$extraction_file ) - set_contents_file(id, CONTENTS_RESP, irc$extraction_file); + if ( extract_file_types !in info$mime_type ) return; - # Delete these values in case another DCC transfer - # happens during the IRC session. - delete irc$extract_file; - delete irc$extraction_file; - delete irc$dcc_file_name; - delete irc$dcc_file_size; - delete irc$dcc_mime_type; - delete dcc_expected_transfers[id$resp_h, id$resp_p]; + for ( act in info$actions ) + if ( act$act == FileAnalysis::ACTION_EXTRACT ) return; + + local fname: string = fmt("%s-%s-%d.dat", extraction_prefix, info$file_id, + extract_count); + ++extract_count; + FileAnalysis::add_action(info$file_id, [$act=FileAnalysis::ACTION_EXTRACT, + $extract_filename=fname]); + set_dcc_extraction_file(info, fname); + } + +hook FileAnalysis::policy(trig: FileAnalysis::Trigger, info: FileAnalysis::Info) + &priority=-5 + { + if ( trig != FileAnalysis::TRIGGER_TYPE ) return; + if ( ! info?$source ) return; + if ( info$source != "IRC_DATA" ) return; + + log_dcc(info); } event irc_dcc_message(c: connection, is_orig: bool, @@ -100,7 +177,7 @@ event irc_dcc_message(c: connection, is_orig: bool, { set_session(c); if ( dcc_type != "SEND" ) - return; + return; c$irc$dcc_file_name = argument; c$irc$dcc_file_size = size; local p = count_to_port(dest_port, tcp); @@ -114,3 +191,8 @@ event expected_connection_seen(c: connection, a: count) &priority=10 if ( [id$resp_h, id$resp_p] in dcc_expected_transfers ) add c$service["irc-dcc-data"]; } + +event connection_state_remove(c: connection) &priority=-5 + { + delete dcc_expected_transfers[c$id$resp_h, c$id$resp_p]; + } diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log index 46adaa4c3e..64bdb41861 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.basic/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2011-07-20-19-12-44 +#open 2013-03-27-18-51-40 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size extraction_file -#types time string addr port addr port string string string string string string count file +#types time string addr port addr port string string string string string string count string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - 1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 - -#close 2011-07-20-19-15-42 +#close 2013-03-27-18-51-40 diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb-0.dat similarity index 100% rename from testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat rename to testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc-dcc-item-wqKMAamJVSb-0.dat diff --git a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log index e204a627b1..4e70587ff0 100644 --- a/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log +++ b/testing/btest/Baseline/scripts.base.protocols.irc.dcc-extract/irc.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path irc -#open 2011-07-20-19-12-44 +#open 2013-03-27-18-49-16 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p nick user command value addl dcc_file_name dcc_file_size dcc_mime_type extraction_file -#types time string addr port addr port string string string string string string count string file +#types time string addr port addr port string string string string string string count string string 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 - - NICK bloed - - - - - 1311189164.119437 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed - USER sdkfje sdkfje Montreal.QC.CA.Undernet.org dkdkrwq - - - - 1311189174.474127 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje JOIN #easymovies (empty) - - - - -1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat -#close 2011-07-20-19-15-42 +1311189316.326025 UWkUyAuUGXf 192.168.1.77 57640 66.198.80.67 6667 bloed sdkfje DCC #easymovies (empty) ladyvampress-default(2011-07-07)-OS.zip 42208 FAKE_MIME irc-dcc-item-wqKMAamJVSb-0.dat +#close 2013-03-27-18-49-16 diff --git a/testing/btest/scripts/base/protocols/irc/dcc-extract.test b/testing/btest/scripts/base/protocols/irc/dcc-extract.test index b6bf43ac50..8a6680f99b 100644 --- a/testing/btest/scripts/base/protocols/irc/dcc-extract.test +++ b/testing/btest/scripts/base/protocols/irc/dcc-extract.test @@ -4,9 +4,9 @@ # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT # @TEST-EXEC: btest-diff irc.log -# @TEST-EXEC: btest-diff irc-dcc-item_192.168.1.77:57655-209.197.168.151:1024_1.dat +# @TEST-EXEC: btest-diff irc-dcc-item-wqKMAamJVSb-0.dat # @TEST-EXEC: bro -r $TRACES/irc-dcc-send.trace %INPUT IRC::extraction_prefix="test" -# @TEST-EXEC: test -e test_192.168.1.77:57655-209.197.168.151:1024_1.dat +# @TEST-EXEC: test -e test-wqKMAamJVSb-0.dat redef IRC::extract_file_types=/.*/;