diff --git a/CHANGES b/CHANGES index 0c7235bd47..1f64cc908a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,44 @@ +2.1-930 | 2013-07-29 15:06:07 -0700 + + * Major file analysis overhaul in naming and appearance, along with + fixes and test updates. (Seth Hall and Jon Siwek) + + Includes: + + * Added protocol description functions that provide a super + compressed log representation. (Seth Hall) + + * Added mime types to http.log (Seth Hall) + + * Add jar files to the default MHR lookups. (Seth Hall) + + * Adding CAB files for MHR checking. (Seth Hall) + + * Improve malware hash registry script. + + - Include a link to a virustotal search in the notice sub message field. + - Give all information returned from Team Cymru in the notice message. + - Add more file types to match on to the default set. + + * Make the custom libmagic database a git submodule. + + * Add an is_orig parameter to file_over_new_connection event. + + * Recorrected the module name to Files. + + * Added Files::analyzer_name to get a more readable name for a + file analyzer. + + * Improved and just overall better handled multipart mime + transfers in HTTP and SMTP. HTTP now has orig_fuids and + resp_fuids log fields since multiple "files" can be transferred + with multipart mime in a single request/response pair. SMTP has + an fuids field which has file unique IDs for all parts + transferred. FTP and IRC have a log field named fuid added + because only a single file can be transferred per irc and ftp + log line. + 2.1-895 | 2013-07-29 14:07:35 -0700 * Adding a test for a DNSKEY RR. (Robin Sommer) diff --git a/NEWS b/NEWS index c3eabf5554..de2ee1b684 100644 --- a/NEWS +++ b/NEWS @@ -80,7 +80,7 @@ New Functionality with the following user-visibible functionality (some of that was already available before, but done differently): - [TODO: This will probably change with further script updates.] + [TODO: Update with changes from 984e9793db56.] - A binary input reader interfaces the input framework with file analysis, allowing to inject files on disk into Bro's diff --git a/VERSION b/VERSION index 9e4a84ae0a..cacffbfffc 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-895 +2.1-930 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 0caafdf107..e28efc9563 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-07-25-17-54-33 +#open 2013-07-29-21-31-47 #fields name #types string scripts/base/init-bare.bro @@ -90,6 +90,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/file_analysis.bif.bro scripts/base/utils/site.bro scripts/base/utils/patterns.bro + build/scripts/base/bif/__load__.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2013-07-25-19-59-47 +#close 2013-07-29-21-31-47 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index deffbe364b..faf372222b 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2013-07-23-05-48-10 +#open 2013-07-29-21-31-48 #fields name #types string scripts/base/init-bare.bro @@ -90,6 +90,7 @@ scripts/base/init-bare.bro build/scripts/base/bif/file_analysis.bif.bro scripts/base/utils/site.bro scripts/base/utils/patterns.bro + build/scripts/base/bif/__load__.bro scripts/base/init-default.bro scripts/base/utils/addrs.bro scripts/base/utils/conn-ids.bro @@ -158,7 +159,7 @@ scripts/base/init-default.bro scripts/base/protocols/ftp/__load__.bro scripts/base/protocols/ftp/utils-commands.bro scripts/base/protocols/ftp/main.bro - scripts/base/protocols/ftp/utils.bro + scripts/base/protocols/ftp/utils.bro scripts/base/protocols/ftp/files.bro scripts/base/protocols/ftp/gridftp.bro scripts/base/protocols/ssl/__load__.bro @@ -197,4 +198,4 @@ scripts/base/init-default.bro scripts/base/files/extract/main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2013-07-23-05-48-10 +#close 2013-07-29-21-31-48 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt index e919233b79..fd998057f3 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt @@ -32,10 +32,10 @@ - - - - + + + + @@ -60,13 +60,13 @@ - - - - + + + + # Extent, type='http' -ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied mime_type md5 extracted_request_files extracted_response_files +ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types 1300475168.784020 j4u32Pc5bif 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 1300475168.916018 VW0XPVINV8a 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 1300475168.916183 3PKsZ2Uye21 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified 0 diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index 34ba063081..1910ef8e17 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -10,5 +10,8 @@ # # @TEST-EXEC: test -d $DIST/scripts # @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo "=== $script" >>allerrors; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0 -# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | grep -v '===' | sort | uniq > unique_errors +# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | fgrep -v -f %INPUT | grep -v '===' | sort | uniq > unique_errors # @TEST-EXEC: btest-diff unique_errors + +# White-list of tests to exclude because of cyclic load dependencies. +scripts/base/protocols/ftp/utils.bro