diff --git a/CHANGES b/CHANGES index 449a924a0c..8f7b817e2a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,9 +1,18 @@ -2.1-1063 | 2013-08-13 18:36:53 -0700 +2.1-1071 | 2013-08-14 10:25:39 -0700 * Adding the unified2 analyzer that reads unified2 files from disk, turning them into events. (Seth Hall) + * Fixing intel framework tests. (Seth Hall) + +2.1-1059 | 2013-08-13 23:52:41 -0400 + + * Add file name support to intel framework. (Seth Hall) + + * Add file support to intel framework and slightly restructure + intel http handling. (Seth Hall) + 2.1-1052 | 2013-08-12 14:38:14 -0700 * Fixing bug in DNP3 analyzer flagged by compiler warning. (Robin diff --git a/VERSION b/VERSION index a873c18a34..bb8e8ac2fd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-1063 +2.1-1071 diff --git a/aux/btest b/aux/btest index 1b114401e5..be7c653dcd 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 1b114401e5eeea0ab2f0ba266f5c79f1e8060f34 +Subproject commit be7c653dcdc30384d4d17359d19d94540fdedaa5 diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 3155f5c726..7abc3e6bb8 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -202,9 +202,10 @@ rest_target(${psd} policy/frameworks/files/hash-all-files.bro) rest_target(${psd} policy/frameworks/intel/do_notice.bro) rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro) rest_target(${psd} policy/frameworks/intel/seen/dns.bro) -rest_target(${psd} policy/frameworks/intel/seen/http-host-header.bro) +rest_target(${psd} policy/frameworks/intel/seen/file-hashes.bro) +rest_target(${psd} policy/frameworks/intel/seen/file-names.bro) +rest_target(${psd} policy/frameworks/intel/seen/http-headers.bro) rest_target(${psd} policy/frameworks/intel/seen/http-url.bro) -rest_target(${psd} policy/frameworks/intel/seen/http-user-agents.bro) rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro) rest_target(${psd} policy/frameworks/intel/seen/smtp.bro) rest_target(${psd} policy/frameworks/intel/seen/ssl.bro) diff --git a/scripts/base/files/unified2/main.bro b/scripts/base/files/unified2/main.bro index 6491d8306e..a74c10e908 100644 --- a/scripts/base/files/unified2/main.bro +++ b/scripts/base/files/unified2/main.bro @@ -136,7 +136,7 @@ event Unified2::read_classification_line(desc: Input::EventDescription, tpe: Inp } } -event bro_init() +event bro_init() &priority=5 { Log::create_stream(Unified2::LOG, [$columns=Info, $ev=log_unified2]); @@ -200,8 +200,8 @@ event file_new(f: fa_file) if ( |parts| == 3 ) file_dir = parts[1]; - if ( f$source in watch_file || - compress_path(watch_dir) == file_dir ) + if ( (watch_file != "" && f$source == watch_file) || + (watch_dir != "" && compress_path(watch_dir) == file_dir) ) { Files::add_analyzer(f, Files::ANALYZER_UNIFIED2); f$u2_events = table(); diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index a201a7a041..b3dcfda00d 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -27,6 +27,9 @@ export { ## File hash which is non-hash type specific. It's up to the user to query ## for any relevant hash types. FILE_HASH, + ## File names. Typically with protocols with definite indications + ## of a file name. + FILE_NAME, ## Certificate SHA-1 hash. CERT_HASH, }; @@ -80,6 +83,10 @@ export { ## If the data was discovered within a connection, the ## connection record should go into get to give context to the data. conn: connection &optional; + + ## If the data was discovered within a file, the file record + ## should go here to provide context to the data. + f: fa_file &optional; }; ## Record used for the logging framework representing a positive @@ -95,6 +102,16 @@ export { ## this is the conn_id for the connection. id: conn_id &log &optional; + ## If a file was associated with this intelligence hit, + ## this is the uid for the file. + fuid: string &log &optional; + ## A mime type if the intelligence hit is related to a file. + ## If the $f field is provided this will be automatically filled out. + file_mime_type: string &log &optional; + ## Frequently files can be "described" to give a bit more context. + ## If the $f field is provided this field will be automatically filled out. + file_desc: string &log &optional; + ## Where the data was seen. seen: Seen &log; ## Sources which supplied data that resulted in this match. @@ -248,7 +265,25 @@ function has_meta(check: MetaData, metas: set[MetaData]): bool event Intel::match(s: Seen, items: set[Item]) &priority=5 { - local info: Info = [$ts=network_time(), $seen=s]; + local info = Info($ts=network_time(), $seen=s); + + if ( s?$f ) + { + if ( s$f?$conns && |s$f$conns| == 1 ) + { + for ( cid in s$f$conns ) + s$conn = s$f$conns[cid]; + } + + if ( ! info?$fuid ) + info$fuid = s$f$id; + + if ( ! info?$file_mime_type && s$f?$mime_type ) + info$file_mime_type = s$f$mime_type; + + if ( ! info?$file_desc ) + info$file_desc = Files::describe(s$f); + } if ( s?$conn ) { diff --git a/scripts/policy/frameworks/intel/seen/__load__.bro b/scripts/policy/frameworks/intel/seen/__load__.bro index 3ffbc35378..01034d95e2 100644 --- a/scripts/policy/frameworks/intel/seen/__load__.bro +++ b/scripts/policy/frameworks/intel/seen/__load__.bro @@ -1,8 +1,9 @@ @load ./conn-established @load ./dns -@load ./http-host-header +@load ./file-hashes +@load ./file-names +@load ./http-headers @load ./http-url -@load ./http-user-agents @load ./ssl @load ./smtp @load ./smtp-url-extraction \ No newline at end of file diff --git a/scripts/policy/frameworks/intel/seen/file-hashes.bro b/scripts/policy/frameworks/intel/seen/file-hashes.bro new file mode 100644 index 0000000000..2e56ad3c48 --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/file-hashes.bro @@ -0,0 +1,12 @@ +@load base/frameworks/intel +@load ./where-locations + +event file_hash(f: fa_file, kind: string, hash: string) + { + local seen = Intel::Seen($indicator=hash, + $indicator_type=Intel::FILE_HASH, + $f=f, + $where=Files::IN_HASH); + + Intel::seen(seen); + } \ No newline at end of file diff --git a/scripts/policy/frameworks/intel/seen/file-names.bro b/scripts/policy/frameworks/intel/seen/file-names.bro new file mode 100644 index 0000000000..ade0d0f18a --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/file-names.bro @@ -0,0 +1,11 @@ +@load base/frameworks/intel +@load ./where-locations + +event file_new(f: fa_file) + { + if ( f?$info && f$info?$filename ) + Intel::seen([$indicator=f$info$filename, + $indicator_type=Intel::FILE_NAME, + $f=f, + $where=Files::IN_NAME]); + } \ No newline at end of file diff --git a/scripts/policy/frameworks/intel/seen/http-headers.bro b/scripts/policy/frameworks/intel/seen/http-headers.bro new file mode 100644 index 0000000000..53aeec4394 --- /dev/null +++ b/scripts/policy/frameworks/intel/seen/http-headers.bro @@ -0,0 +1,46 @@ +@load base/frameworks/intel +@load ./where-locations + +event http_header(c: connection, is_orig: bool, name: string, value: string) + { + if ( is_orig ) + { + switch ( name ) + { + case "HOST": + Intel::seen([$indicator=value, + $indicator_type=Intel::DOMAIN, + $conn=c, + $where=HTTP::IN_HOST_HEADER]); + break; + + case "REFERER": + Intel::seen([$indicator=sub(value, /^.*:\/\//, ""), + $indicator_type=Intel::URL, + $conn=c, + $where=HTTP::IN_REFERRER_HEADER]); + break; + + case "X-FORWARDED-FOR": + if ( is_valid_ip(value) ) + { + local addrs = find_ip_addresses(value); + for ( i in addrs ) + { + Intel::seen([$host=to_addr(addrs[i]), + $indicator_type=Intel::ADDR, + $conn=c, + $where=HTTP::IN_X_FORWARDED_FOR_HEADER]); + } + } + break; + + case "USER-AGENT": + Intel::seen([$indicator=value, + $indicator_type=Intel::SOFTWARE, + $conn=c, + $where=HTTP::IN_USER_AGENT_HEADER]); + break; + } + } + } diff --git a/scripts/policy/frameworks/intel/seen/http-host-header.bro b/scripts/policy/frameworks/intel/seen/http-host-header.bro deleted file mode 100644 index 3fd28b8ef9..0000000000 --- a/scripts/policy/frameworks/intel/seen/http-host-header.bro +++ /dev/null @@ -1,11 +0,0 @@ -@load base/frameworks/intel -@load ./where-locations - -event http_header(c: connection, is_orig: bool, name: string, value: string) - { - if ( is_orig && name == "HOST" ) - Intel::seen([$indicator=value, - $indicator_type=Intel::DOMAIN, - $conn=c, - $where=HTTP::IN_HOST_HEADER]); - } diff --git a/scripts/policy/frameworks/intel/seen/http-user-agents.bro b/scripts/policy/frameworks/intel/seen/http-user-agents.bro deleted file mode 100644 index 7c4558d2a5..0000000000 --- a/scripts/policy/frameworks/intel/seen/http-user-agents.bro +++ /dev/null @@ -1,12 +0,0 @@ -@load base/frameworks/intel -@load ./where-locations - -event http_header(c: connection, is_orig: bool, name: string, value: string) - { - if ( is_orig && name == "USER-AGENT" ) - Intel::seen([$indicator=value, - $indicator_type=Intel::SOFTWARE, - $conn=c, - $where=HTTP::IN_USER_AGENT_HEADER]); - } - diff --git a/scripts/policy/frameworks/intel/seen/where-locations.bro b/scripts/policy/frameworks/intel/seen/where-locations.bro index 4773de9c73..0387814ea7 100644 --- a/scripts/policy/frameworks/intel/seen/where-locations.bro +++ b/scripts/policy/frameworks/intel/seen/where-locations.bro @@ -4,10 +4,14 @@ export { redef enum Intel::Where += { Conn::IN_ORIG, Conn::IN_RESP, + Files::IN_HASH, + Files::IN_NAME, DNS::IN_REQUEST, DNS::IN_RESPONSE, HTTP::IN_HOST_HEADER, + HTTP::IN_REFERRER_HEADER, HTTP::IN_USER_AGENT_HEADER, + HTTP::IN_X_FORWARDED_FOR_HEADER, HTTP::IN_URL, SMTP::IN_MAIL_FROM, SMTP::IN_RCPT_TO, diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 7d582bf82f..63b9b5998c 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -18,9 +18,10 @@ @load frameworks/intel/seen/__load__.bro @load frameworks/intel/seen/conn-established.bro @load frameworks/intel/seen/dns.bro -@load frameworks/intel/seen/http-host-header.bro +@load frameworks/intel/seen/file-hashes.bro +@load frameworks/intel/seen/file-names.bro +@load frameworks/intel/seen/http-headers.bro @load frameworks/intel/seen/http-url.bro -@load frameworks/intel/seen/http-user-agents.bro @load frameworks/intel/seen/smtp-url-extraction.bro @load frameworks/intel/seen/smtp.bro @load frameworks/intel/seen/ssl.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log index 1e46209d46..344ffa7802 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.logging/files.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path files -#open 2013-08-12-19-07-37 +#open 2013-08-14-04-50-17 #fields ts fuid tx_hosts rx_hosts conn_uids source depth analyzers mime_type filename duration local_orig is_orig seen_bytes total_bytes missing_bytes overflow_bytes timedout parent_fuid md5 sha1 sha256 extracted #types time string table[addr] table[addr] table[string] string count table[string] string string interval bool bool count count count count bool string string string string string -1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 UNIFIED2,SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file -#close 2013-08-12-19-07-37 +1362692527.009721 G75mcAsU764 192.150.187.43 141.142.228.5 UWkUyAuUGXf HTTP 0 SHA256,DATA_EVENT,MD5,EXTRACT,SHA1 text/plain - 0.000054 - F 4705 4705 0 0 F - 397168fd09991a0e712254df7bc639ac 1dd7ac0398df6cbc0696445a91ec681facf4dc47 4e7c7ef0984119447e743e3ec77e1de52713e345cde03fe7df753a35849bed18 G75mcAsU764-file +#close 2013-08-14-04-50-17 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log index 00871e7d93..27a1f2d2f8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.cluster-transparency/manager-1.intel.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path intel -#open 2013-07-19-17-05-48 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where sources -#types time string addr port addr port string enum enum table[string] -1374253548.038580 - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-1 -#close 2013-07-19-17-05-57 +#open 2013-08-14-03-46-32 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where sources +#types time string addr port addr port string string string string enum enum table[string] +1376451992.872806 - - - - - - - - 123.123.123.123 Intel::ADDR Intel::IN_ANYWHERE worker-1 +#close 2013-08-14-03-46-42 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log index 8c01ae5c27..ea57d77b18 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.input-and-match/broproc.intel.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path intel -#open 2013-07-19-17-04-26 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where sources -#types time string addr port addr port string enum enum table[string] -1374253466.857185 - - - - - e@mail.com Intel::EMAIL SOMEWHERE source1 -1374253466.857185 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE source1 -#close 2013-07-19-17-04-26 +#open 2013-08-14-03-47-03 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where sources +#types time string addr port addr port string string string string enum enum table[string] +1376452023.137179 - - - - - - - - e@mail.com Intel::EMAIL SOMEWHERE source1 +1376452023.137179 - - - - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE source1 +#close 2013-08-14-03-47-03 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log index 70d92a3604..bf9aa50fef 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.read-file-dist-cluster/manager-1.intel.log @@ -3,11 +3,11 @@ #empty_field (empty) #unset_field - #path intel -#open 2013-07-19-17-06-57 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where sources -#types time string addr port addr port string enum enum table[string] -1374253617.312158 - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1 -1374253617.312158 - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1 -1374253618.332565 - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1 -1374253618.332565 - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1 -#close 2013-07-19-17-07-06 +#open 2013-08-14-03-47-23 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p fuid file_mime_type file_desc seen.indicator seen.indicator_type seen.where sources +#types time string addr port addr port string string string string enum enum table[string] +1376452043.835810 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1 +1376452043.835810 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1 +1376452044.855238 - - - - - - - - 1.2.3.4 Intel::ADDR Intel::IN_A_TEST source1 +1376452044.855238 - - - - - - - - e@mail.com Intel::EMAIL Intel::IN_A_TEST source1 +#close 2013-08-14-03-47-32