From b2a3723a1d870d7da93a7ed83d692bfbd188fd18 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:39:46 -0700 Subject: [PATCH 1/9] Fixing tests that need a diff canonifier. --- .../policy.misc.loaded-scripts/output | 42 +++++++++---------- testing/btest/core/load-pkg.bro | 2 +- testing/btest/core/load-unload.bro | 2 +- testing/btest/policy/misc/loaded-scripts.bro | 2 +- 4 files changed, 24 insertions(+), 24 deletions(-) diff --git a/testing/btest/Baseline/policy.misc.loaded-scripts/output b/testing/btest/Baseline/policy.misc.loaded-scripts/output index fe360fd38f..ca3eb3f396 100644 --- a/testing/btest/Baseline/policy.misc.loaded-scripts/output +++ b/testing/btest/Baseline/policy.misc.loaded-scripts/output @@ -1,21 +1,21 @@ -0, /Users/seth/bro.git9/policy/bro.init -1, /Users/seth/bro.git9/build/src/const.bif.bro -1, /Users/seth/bro.git9/build/src/types.bif.bro -1, /Users/seth/bro.git9/build/src/strings.bif.bro -1, /Users/seth/bro.git9/build/src/bro.bif.bro -1, /Users/seth/bro.git9/build/src/reporter.bif.bro -1, /Users/seth/bro.git9/build/src/event.bif.bro -1, /Users/seth/bro.git9/policy/utils/site.bro -2, /Users/seth/bro.git9/policy/utils/pattern.bro -1, /Users/seth/bro.git9/policy/frameworks/logging/__load__.bro -2, /Users/seth/bro.git9/policy/frameworks/logging/base.bro -3, /Users/seth/bro.git9/build/src/logging.bif.bro -2, /Users/seth/bro.git9/policy/frameworks/logging/plugins/ascii.bro -1, /Users/seth/bro.git9/policy/frameworks/packet-filter/__load__.bro -2, /Users/seth/bro.git9/policy/frameworks/packet-filter/base.bro -3, /Users/seth/bro.git9/policy/frameworks/notice/__load__.bro -4, /Users/seth/bro.git9/policy/frameworks/notice/base.bro -4, /Users/seth/bro.git9/policy/frameworks/notice/weird.bro -5, /Users/seth/bro.git9/policy/utils/conn-ids.bro -2, /Users/seth/bro.git9/policy/frameworks/packet-filter/netstats.bro -0, /Users/seth/bro.git9/testing/btest/.tmp/policy.misc.loaded-scripts/loaded-scripts.bro +0, /Users/jsiwek/tmp/bro/policy/bro.init +1, /Users/jsiwek/tmp/bro/build/src/const.bif.bro +1, /Users/jsiwek/tmp/bro/build/src/types.bif.bro +1, /Users/jsiwek/tmp/bro/build/src/strings.bif.bro +1, /Users/jsiwek/tmp/bro/build/src/bro.bif.bro +1, /Users/jsiwek/tmp/bro/build/src/reporter.bif.bro +1, /Users/jsiwek/tmp/bro/build/src/event.bif.bro +1, /Users/jsiwek/tmp/bro/policy/site.bro +2, /Users/jsiwek/tmp/bro/policy/utils/pattern.bro +1, /Users/jsiwek/tmp/bro/policy/frameworks/logging/__load__.bro +2, /Users/jsiwek/tmp/bro/policy/frameworks/logging/base.bro +3, /Users/jsiwek/tmp/bro/build/src/logging.bif.bro +2, /Users/jsiwek/tmp/bro/policy/frameworks/logging/plugins/ascii.bro +1, /Users/jsiwek/tmp/bro/policy/frameworks/packet-filter/__load__.bro +2, /Users/jsiwek/tmp/bro/policy/frameworks/packet-filter/base.bro +3, /Users/jsiwek/tmp/bro/policy/frameworks/notice/__load__.bro +4, /Users/jsiwek/tmp/bro/policy/frameworks/notice/base.bro +4, /Users/jsiwek/tmp/bro/policy/frameworks/notice/weird.bro +5, /Users/jsiwek/tmp/bro/policy/utils/conn_ids.bro +2, /Users/jsiwek/tmp/bro/policy/frameworks/packet-filter/netstats.bro +0, /Users/jsiwek/tmp/bro/testing/btest/.tmp/policy.misc.loaded-scripts/loaded-scripts.bro diff --git a/testing/btest/core/load-pkg.bro b/testing/btest/core/load-pkg.bro index 9cb67b1ed3..2bd566a27f 100644 --- a/testing/btest/core/load-pkg.bro +++ b/testing/btest/core/load-pkg.bro @@ -3,6 +3,6 @@ # @TEST-EXEC: cp %INPUT foo/test.bro # @TEST-EXEC: bro misc/loaded-scripts foo >output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff loaded_scripts.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff loaded_scripts.log print "Foo loaded"; diff --git a/testing/btest/core/load-unload.bro b/testing/btest/core/load-unload.bro index cab7fd1414..553260f395 100644 --- a/testing/btest/core/load-unload.bro +++ b/testing/btest/core/load-unload.bro @@ -3,6 +3,6 @@ # @TEST-EXEC: echo 'print "oops12345";' >dontloadmebro.bro # @TEST-EXEC: bro %INPUT misc/loaded-scripts dontloadmebro > output # @TEST-EXEC: btest-diff output -# @TEST-EXEC: btest-diff loaded_scripts.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff loaded_scripts.log @unload dontloadmebro diff --git a/testing/btest/policy/misc/loaded-scripts.bro b/testing/btest/policy/misc/loaded-scripts.bro index 775a939196..589dc207b2 100644 --- a/testing/btest/policy/misc/loaded-scripts.bro +++ b/testing/btest/policy/misc/loaded-scripts.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro %INPUT >output -# @TEST-EXEC: btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output event bro_script_loaded(path: string, level: count) { From 38bc5603585fc0a2de83dc1d8bcb4070c45197ad Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 16:29:15 -0700 Subject: [PATCH 2/9] Fixing 0-chunk bug in remote logging. --- src/RemoteSerializer.cc | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 56f0772a26..ca534e00fc 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2558,7 +2558,10 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st // Do we have enough space in the buffer? If not, flush first. if ( len > (LOG_BUFFER_SIZE - peer->log_buffer_used) ) - FlushLogBuffer(peer); + { + if ( ! FlushLogBuffer(peer) ) + return false; + } // If the data is actually larger than our complete buffer, just send it out. if ( len > LOG_BUFFER_SIZE ) @@ -2569,8 +2572,8 @@ bool RemoteSerializer::SendLogWrite(Peer* peer, EnumVal* id, EnumVal* writer, st peer->log_buffer_used += len; assert(peer->log_buffer_used <= LOG_BUFFER_SIZE); - FlushLogBuffer(peer); - return false; + FlushLogBuffer(peer); // FIXME: This should go away, but then the unit test fails. See #498. + return true; error: FatalError(io->Error()); @@ -2582,7 +2585,7 @@ bool RemoteSerializer::FlushLogBuffer(Peer* p) if ( p->state == Peer::CLOSING ) return false; - if ( ! p->log_buffer ) + if ( ! (p->log_buffer && p->log_buffer_used) ) return true; SendToChild(MSG_LOG_WRITE, p, p->log_buffer, p->log_buffer_used); From 2afb3ea1a1f067e4e74f1f1cef368de2e5353d2a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:35:41 -0700 Subject: [PATCH 3/9] Updating submodule(s). --- aux/broccoli | 2 +- aux/btest | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broccoli b/aux/broccoli index 9866a00e78..f145396e43 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 9866a00e78b088315e632960aaa230635bca326a +Subproject commit f145396e43342b8756d68628ac1483ed4509ec39 diff --git a/aux/btest b/aux/btest index a9aeb2e1a8..32da06af20 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit a9aeb2e1a8434c583c75f5941b58dc69a7517444 +Subproject commit 32da06af20e7cbea7d91cfecf28d0f3ee7ba68be From ad7654745f3674050b9722523ecb11bd369c0176 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 15 Jul 2011 12:33:21 -0500 Subject: [PATCH 4/9] Fix istate.events tests. The receiver now registers for signature_match event so that the http.log comes out the same as the sender's side. --- testing/btest/Baseline/istate.events-ssl/receiver.http.log | 2 +- testing/btest/Baseline/istate.events-ssl/sender.http.log | 2 +- testing/btest/Baseline/istate.events/receiver.http.log | 2 +- testing/btest/Baseline/istate.events/sender.http.log | 2 +- testing/btest/istate/events-ssl.bro | 2 +- testing/btest/istate/events.bro | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/testing/btest/Baseline/istate.events-ssl/receiver.http.log b/testing/btest/Baseline/istate.events-ssl/receiver.http.log index 29a1956eed..2e56522dae 100644 --- a/testing/btest/Baseline/istate.events-ssl/receiver.http.log +++ b/testing/btest/Baseline/istate.events-ssl/receiver.http.log @@ -1,2 +1,2 @@ # ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_content_length response_content_length status_code status_msg filename tags username password proxied mime_type md5 extraction_file -1310590727.30137 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - - - - +1310750785.32134 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events-ssl/sender.http.log b/testing/btest/Baseline/istate.events-ssl/sender.http.log index c4ba804670..2e56522dae 100644 --- a/testing/btest/Baseline/istate.events-ssl/sender.http.log +++ b/testing/btest/Baseline/istate.events-ssl/sender.http.log @@ -1,2 +1,2 @@ # ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_content_length response_content_length status_code status_msg filename tags username password proxied mime_type md5 extraction_file -1310590727.30137 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - text/html - - +1310750785.32134 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/receiver.http.log b/testing/btest/Baseline/istate.events/receiver.http.log index 93e0f6264a..38ba563dc7 100644 --- a/testing/btest/Baseline/istate.events/receiver.http.log +++ b/testing/btest/Baseline/istate.events/receiver.http.log @@ -1,2 +1,2 @@ # ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_content_length response_content_length status_code status_msg filename tags username password proxied mime_type md5 extraction_file -1310590707.11117 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - - - - +1310750770.8185 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - text/html - - diff --git a/testing/btest/Baseline/istate.events/sender.http.log b/testing/btest/Baseline/istate.events/sender.http.log index 4a70b73996..38ba563dc7 100644 --- a/testing/btest/Baseline/istate.events/sender.http.log +++ b/testing/btest/Baseline/istate.events/sender.http.log @@ -1,2 +1,2 @@ # ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_content_length response_content_length status_code status_msg filename tags username password proxied mime_type md5 extraction_file -1310590707.11117 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - text/html - - +1310750770.8185 56gKBmhBBB6 141.42.64.125 56730 125.190.109.199 80 GET www.icir.org / - Wget/1.10 - 9130 200 OK - - - - - text/html - - diff --git a/testing/btest/istate/events-ssl.bro b/testing/btest/istate/events-ssl.bro index 7cf4387780..b3e02b5e73 100644 --- a/testing/btest/istate/events-ssl.bro +++ b/testing/btest/istate/events-ssl.bro @@ -52,7 +52,7 @@ event bro_init() redef peer_description = "events-rcv"; redef Communication::nodes += { - ["foo"] = [$host = 127.0.0.1, $events = /http_.*/, $connect=T, $ssl=T] + ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T, $ssl=T] }; redef ssl_ca_certificate = "../ca_cert.pem"; diff --git a/testing/btest/istate/events.bro b/testing/btest/istate/events.bro index 8f5f538b70..3aaca2d3b9 100644 --- a/testing/btest/istate/events.bro +++ b/testing/btest/istate/events.bro @@ -48,7 +48,7 @@ event bro_init() redef peer_description = "events-rcv"; redef Communication::nodes += { - ["foo"] = [$host = 127.0.0.1, $events = /http_.*/, $connect=T] + ["foo"] = [$host = 127.0.0.1, $events = /http_.*|signature_match/, $connect=T] }; event remote_connection_closed(p: event_peer) From 1b7bf298969aa906774b4f0f51eaeeea4f3c1db2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:37:08 -0700 Subject: [PATCH 5/9] Updating submodule(s). --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index f145396e43..8fdd0f2f82 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit f145396e43342b8756d68628ac1483ed4509ec39 +Subproject commit 8fdd0f2f82bb29352b301131ec99a3040f4e1dae From de58e54397f1d5e892db38158660d88dca8ff1ad Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:38:23 -0700 Subject: [PATCH 6/9] Updating submodule(s). --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 32da06af20..3b23a30015 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 32da06af20e7cbea7d91cfecf28d0f3ee7ba68be +Subproject commit 3b23a300152405135088aeb9fc0f411c295193b1 From 5d7af681f449383dc07ee3965ed5f7959b4d1a56 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:46:44 -0700 Subject: [PATCH 7/9] Fixing comment. --- src/PktSrc.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/PktSrc.cc b/src/PktSrc.cc index 7e178d69e2..b2287bf8e8 100644 --- a/src/PktSrc.cc +++ b/src/PktSrc.cc @@ -512,8 +512,7 @@ PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter, return; // We don't put file sources into non-blocking mode as - // otherwise we would not be able to identify the EOF - // via next_packet(). + // otherwise we would not be able to identify the EOF. selectable_fd = fileno(pcap_file(pd)); From 6253c7c02cbe5987eadb622012c59d0a2f8c9df2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:49:52 -0700 Subject: [PATCH 8/9] Removing undocumented -H command line flag. Closes #496. --- src/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cc b/src/main.cc index 641535f870..ec32de1060 100644 --- a/src/main.cc +++ b/src/main.cc @@ -438,7 +438,7 @@ int main(int argc, char** argv) opterr = 0; char opts[256]; - safe_strncpy(opts, "B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGHLOPSWdghvZ", + safe_strncpy(opts, "B:D:e:f:I:i:K:n:p:R:r:s:T:t:U:w:x:X:y:Y:z:CFGLOPSWdghvZ", sizeof(opts)); #ifdef USE_PERFTOOLS From 2e9b64a7ddf1a6c4389c7a0dc847f03f83db8aa0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 19 Jul 2011 17:54:57 -0700 Subject: [PATCH 9/9] Updating submodule(s). --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 8fdd0f2f82..e90325bc2b 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 8fdd0f2f82bb29352b301131ec99a3040f4e1dae +Subproject commit e90325bc2bd3023f4a5f48446a91725192eafef4