diff --git a/CHANGES b/CHANGES index 94a548f5cb..31803f5b47 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,18 @@ +2.5-beta-28 | 2016-09-27 11:44:33 -0700 + + * Check if the number of fields in a log write are equal to the + number of fields required. Addresses BIT-1683. (Johanna Amann) + + * Fix a small memory leak for disabled log writers. (Johanna Amann) + + * Fix loaded-scripts.bro to work with arbitrary indentation levels. + Addresses BIT-1691 (Johanna Amann) + + * In interesting-hostnames.bro, move lookup_addr() outside of when + statement to avoid expensive cloning of full connection record. + Addresses BIT-1670. (Justin Azoff) + 2.5-beta-21 | 2016-09-26 10:15:23 -0700 * Fix a debugger bug where it would not support statements like print(3). diff --git a/VERSION b/VERSION index 4fc51f4a1e..90bb498cc5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-beta-21 +2.5-beta-28 diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index a075701151..14e131c755 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -175,10 +175,20 @@ void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields) } -void WriterFrontend::Write(int num_fields, Value** vals) +void WriterFrontend::Write(int arg_num_fields, Value** vals) { if ( disabled ) + { + DeleteVals(arg_num_fields, vals); return; + } + + if ( arg_num_fields != num_fields ) + { + reporter->Warning("WriterFrontend %s expected %d fields in write, got %d. Skipping line.", name, num_fields, arg_num_fields); + DeleteVals(arg_num_fields, vals); + return; + } if ( remote ) remote_serializer->SendLogWrite(stream, @@ -189,7 +199,7 @@ void WriterFrontend::Write(int num_fields, Value** vals) if ( ! backend ) { - DeleteVals(vals); + DeleteVals(arg_num_fields, vals); return; } @@ -262,7 +272,7 @@ void WriterFrontend::Rotate(const char* rotated_path, double open, double close, log_mgr->FinishedRotation(this, 0, 0, 0, 0, false, terminating); } -void WriterFrontend::DeleteVals(Value** vals) +void WriterFrontend::DeleteVals(int num_fields, Value** vals) { // Note this code is duplicated in Manager::DeleteVals(). for ( int i = 0; i < num_fields; i++ ) diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index 994c82c513..e343f326bf 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -203,7 +203,7 @@ public: protected: friend class Manager; - void DeleteVals(threading::Value** vals); + void DeleteVals(int num_fields, threading::Value** vals); EnumVal* stream; EnumVal* writer; diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-1.reporter.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-1.reporter.log new file mode 100644 index 0000000000..b7d8c111cf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster-error/manager-1.reporter.log @@ -0,0 +1,13 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path reporter +#open 2016-09-22-23-31-34 +#fields _write_ts _stream _system_name ts level message location +#types time string string time enum string string +1474587094.261799 reporter manager-1 0.000000 Reporter::WARNING WriterFrontend communication/Log::WRITER_ASCII expected 11 fields in write, got 8. Skipping line. (empty) +1474587094.261799 reporter manager-1 0.000000 Reporter::WARNING WriterFrontend communication/Log::WRITER_ASCII expected 11 fields in write, got 8. Skipping line. (empty) +1474587094.261799 reporter manager-1 0.000000 Reporter::WARNING WriterFrontend communication/Log::WRITER_ASCII expected 11 fields in write, got 8. Skipping line. (empty) +1474587099.984660 reporter manager-1 0.000000 Reporter::INFO received termination signal (empty) +#close 2016-09-22-23-31-40 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster/manager-1.http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster/manager-1.http.log new file mode 100644 index 0000000000..1d1e9c5637 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-cluster/manager-1.http.log @@ -0,0 +1,23 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open 2016-09-22-22-00-07 +#fields _write_ts _stream _system_name ts uid id_orig_h id_orig_p id_resp_h id_resp_p trans_depth method host uri referrer version user_agent request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types +#types time string string time string addr port addr port count string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] +1474581606.495669 http worker-1 1474581606.435795 CUM0KZ3MLUfNB0cl11 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ 1.1 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.627575 http worker-1 1474581606.567793 CwjjYJ2WqgTbAqiHl6 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.628102 http worker-1 1474581606.567958 C3eiCBGOLw3VtHfOj 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.630935 http worker-1 1474581606.570133 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.664441 http worker-1 1474581606.604082 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.664505 http worker-1 1474581606.604071 CykQaM33ztNt0csB9a 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.666635 http worker-1 1474581606.606595 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.674440 http worker-1 1474581606.614462 CiyBAq1bBLNaTiTAc 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.688069 http worker-1 1474581606.627709 CwjjYJ2WqgTbAqiHl6 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.688573 http worker-1 1474581606.628211 C3eiCBGOLw3VtHfOj 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.691698 http worker-1 1474581606.631039 Ck51lg1bScffFj34Ri 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.726568 http worker-1 1474581606.666394 CtxTCR2Yer0FR1tIBg 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.726713 http worker-1 1474581606.666368 CykQaM33ztNt0csB9a 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +1474581606.726840 http worker-1 1474581606.666702 CLNN1k2QMum1aexUK7 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ 1.0 Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - (empty) - - - - - - - - - +#close 2016-09-22-22-00-12 diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro new file mode 100644 index 0000000000..6ac7a5efce --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster-error.bro @@ -0,0 +1,49 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT" +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff manager-1/reporter.log + + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +@load base/protocols/conn + +redef Log::default_scope_sep="_"; + +type Extension: record { + write_ts: time &log; + stream: string &log; + system_name: string &log; +}; + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +function add_extension(path: string): Extension + { + return Extension($write_ts = network_time(), + $stream = path, + $system_name = peer_description); + } + +redef Log::default_ext_func = add_extension; + +@endif + +event terminate_me() { + terminate(); +} + +event remote_connection_closed(p: event_peer) { + schedule 1sec { terminate_me() }; +} + diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro new file mode 100644 index 0000000000..fb51251f8c --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-cluster.bro @@ -0,0 +1,45 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/wikipedia.trace %INPUT" +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff manager-1/http.log + + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +@load base/protocols/conn + +redef Log::default_scope_sep="_"; + +type Extension: record { + write_ts: time &log; + stream: string &log; + system_name: string &log; +}; + +function add_extension(path: string): Extension + { + return Extension($write_ts = network_time(), + $stream = path, + $system_name = peer_description); + } + +redef Log::default_ext_func = add_extension; + +event terminate_me() { + terminate(); +} + +event remote_connection_closed(p: event_peer) { + schedule 1sec { terminate_me() }; +} +