diff --git a/CHANGES b/CHANGES index e3d20b84b6..03b067f290 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,38 @@ +2.1-498 | 2013-05-03 17:44:08 -0700 + + * Table lookups return copy of non-const &default vals. This + prevents unintentional modifications to the &default value itself. + Addresses #981. (Jon Siwek) + +2.1-496 | 2013-05-03 15:54:47 -0700 + + * Fix memory leak and unnecessary allocations in OpaqueVal. + Addresses #986. (Matthias Vallentin) + +2.1-492 | 2013-05-02 12:46:26 -0700 + + * Work-around for sumstats framework not propagating updates after + intermediate check in cluster environments. (Bernhard Amann) + + * Always apply tcp_connection_attempt. Before this change it was + only applied when a connection_attempt() event handler was + defined. (Robin Sommer) + + * Fixing coverage.bare-mode-errors test. (Robin Sommer) + +2.1-487 | 2013-05-01 18:03:22 -0700 + + * Always apply tcp_connection_attempt timer, even if no + connection_attempt() event handler is defined. (Robin Sommer) + +2.1-486 | 2013-05-01 15:28:45 -0700 + + * New framework for computing summary statistics in + base/framework/sumstats. This replaces the metrics frameworks, and + comes with a number of applications build on top, see NEWS. More + documentation to follow. (Seth Hall) + 2.1-397 | 2013-04-29 21:19:00 -0700 * Fixing memory leaks in CompHash implementation. Addresses #987. diff --git a/VERSION b/VERSION index 4809e9f2e9..7de1d6c4ee 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-397 +2.1-498 diff --git a/scripts/base/frameworks/sumstats/cluster.bro b/scripts/base/frameworks/sumstats/cluster.bro index 9ee63a674e..be0a5b5ded 100644 --- a/scripts/base/frameworks/sumstats/cluster.bro +++ b/scripts/base/frameworks/sumstats/cluster.bro @@ -124,7 +124,9 @@ event SumStats::send_data(uid: string, ssid: string, data: ResultTable) if ( |data| == 0 ) done = T; - event SumStats::cluster_ss_response(uid, ssid, local_data, done); + # Note: copy is needed to compensate serialization caching issue. This should be + # changed to something else later. + event SumStats::cluster_ss_response(uid, ssid, copy(local_data), done); if ( ! done ) schedule 0.01 sec { SumStats::send_data(uid, ssid, data) }; } @@ -150,7 +152,10 @@ event SumStats::cluster_key_request(uid: string, ssid: string, key: Key) if ( ssid in result_store && key in result_store[ssid] ) { #print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data); - event SumStats::cluster_key_response(uid, ssid, key, result_store[ssid][key]); + + # Note: copy is needed to compensate serialization caching issue. This should be + # changed to something else later. + event SumStats::cluster_key_response(uid, ssid, key, copy(result_store[ssid][key])); } else { diff --git a/scripts/base/frameworks/sumstats/plugins/average.bro b/scripts/base/frameworks/sumstats/plugins/average.bro index a409bb9408..ad82a91d20 100644 --- a/scripts/base/frameworks/sumstats/plugins/average.bro +++ b/scripts/base/frameworks/sumstats/plugins/average.bro @@ -1,4 +1,4 @@ -@load base/frameworks/sumstats +@load base/frameworks/sumstats/main module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/max.bro b/scripts/base/frameworks/sumstats/plugins/max.bro index 6167d31f10..f9ff9258ee 100644 --- a/scripts/base/frameworks/sumstats/plugins/max.bro +++ b/scripts/base/frameworks/sumstats/plugins/max.bro @@ -1,4 +1,4 @@ -@load base/frameworks/sumstats +@load base/frameworks/sumstats/main module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/min.bro b/scripts/base/frameworks/sumstats/plugins/min.bro index a15ed0e733..95d492f428 100644 --- a/scripts/base/frameworks/sumstats/plugins/min.bro +++ b/scripts/base/frameworks/sumstats/plugins/min.bro @@ -1,4 +1,4 @@ -@load base/frameworks/sumstats +@load base/frameworks/sumstats/main module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index d0587bde08..dc2f438c79 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -1,4 +1,4 @@ -@load base/frameworks/sumstats +@load base/frameworks/sumstats/main @load base/utils/queue module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/std-dev.bro b/scripts/base/frameworks/sumstats/plugins/std-dev.bro index 6411fe4bce..0f32e25a68 100644 --- a/scripts/base/frameworks/sumstats/plugins/std-dev.bro +++ b/scripts/base/frameworks/sumstats/plugins/std-dev.bro @@ -1,5 +1,5 @@ +@load base/frameworks/sumstats/main @load ./variance -@load base/frameworks/sumstats module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/sum.bro b/scripts/base/frameworks/sumstats/plugins/sum.bro index 3e5b28e2be..db2246742b 100644 --- a/scripts/base/frameworks/sumstats/plugins/sum.bro +++ b/scripts/base/frameworks/sumstats/plugins/sum.bro @@ -1,4 +1,4 @@ -@load base/frameworks/sumstats +@load base/frameworks/sumstats/main module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/unique.bro b/scripts/base/frameworks/sumstats/plugins/unique.bro index a407a487a2..ef62caaffa 100644 --- a/scripts/base/frameworks/sumstats/plugins/unique.bro +++ b/scripts/base/frameworks/sumstats/plugins/unique.bro @@ -1,4 +1,4 @@ -@load base/frameworks/sumstats +@load base/frameworks/sumstats/main module SumStats; diff --git a/scripts/base/frameworks/sumstats/plugins/variance.bro b/scripts/base/frameworks/sumstats/plugins/variance.bro index 1e7a00ea97..773c7d697c 100644 --- a/scripts/base/frameworks/sumstats/plugins/variance.bro +++ b/scripts/base/frameworks/sumstats/plugins/variance.bro @@ -1,5 +1,5 @@ +@load base/frameworks/sumstats/main @load ./average -@load base/frameworks/sumstats module SumStats; diff --git a/src/NetVar.cc b/src/NetVar.cc index 248ae15e1a..012e4a85bc 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -239,6 +239,11 @@ TableType* record_field_table; StringVal* cmd_line_bpf_filter; +OpaqueType* md5_type; +OpaqueType* sha1_type; +OpaqueType* sha256_type; +OpaqueType* entropy_type; + #include "const.bif.netvar_def" #include "types.bif.netvar_def" #include "event.bif.netvar_def" @@ -298,6 +303,11 @@ void init_general_global_var() cmd_line_bpf_filter = internal_val("cmd_line_bpf_filter")->AsStringVal(); + + md5_type = new OpaqueType("md5"); + sha1_type = new OpaqueType("sha1"); + sha256_type = new OpaqueType("sha256"); + entropy_type = new OpaqueType("entropy"); } void init_net_var() @@ -346,7 +356,7 @@ void init_net_var() opt_internal_int("tcp_excessive_data_without_further_acks"); x509_type = internal_type("X509")->AsRecordType(); - + socks_address = internal_type("SOCKS::Address")->AsRecordType(); non_analyzed_lifetime = opt_internal_double("non_analyzed_lifetime"); diff --git a/src/NetVar.h b/src/NetVar.h index 2561fa0ad9..d7590b20e7 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -243,6 +243,12 @@ extern TableType* record_field_table; extern StringVal* cmd_line_bpf_filter; +class OpaqueType; +extern OpaqueType* md5_type; +extern OpaqueType* sha1_type; +extern OpaqueType* sha256_type; +extern OpaqueType* entropy_type; + // Initializes globals that don't pertain to network/event analysis. extern void init_general_global_var(); diff --git a/src/OpaqueVal.cc b/src/OpaqueVal.cc index 51f975edf8..19346e52f2 100644 --- a/src/OpaqueVal.cc +++ b/src/OpaqueVal.cc @@ -1,4 +1,5 @@ #include "OpaqueVal.h" +#include "NetVar.h" #include "Reporter.h" #include "Serializer.h" @@ -72,6 +73,10 @@ bool HashVal::DoUnserialize(UnserialInfo* info) return UNSERIALIZE(&valid); } +MD5Val::MD5Val() : HashVal(md5_type) + { + } + void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH]) { MD5_CTX h; @@ -189,6 +194,10 @@ bool MD5Val::DoUnserialize(UnserialInfo* info) return true; } +SHA1Val::SHA1Val() : HashVal(sha1_type) + { + } + void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]) { SHA_CTX h; @@ -297,6 +306,10 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info) return true; } +SHA256Val::SHA256Val() : HashVal(sha256_type) + { + } + void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]) { SHA256_CTX h; @@ -410,6 +423,9 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info) return true; } +EntropyVal::EntropyVal() : OpaqueVal(entropy_type) + { + } bool EntropyVal::Feed(const void* data, size_t size) { diff --git a/src/OpaqueVal.h b/src/OpaqueVal.h index 0428e50bdb..78fa5da5e9 100644 --- a/src/OpaqueVal.h +++ b/src/OpaqueVal.h @@ -36,7 +36,7 @@ public: u_char key[MD5_DIGEST_LENGTH], u_char result[MD5_DIGEST_LENGTH]); - MD5Val() : HashVal(new OpaqueType("md5")) { } + MD5Val(); protected: friend class Val; @@ -55,7 +55,7 @@ class SHA1Val : public HashVal { public: static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]); - SHA1Val() : HashVal(new OpaqueType("sha1")) { } + SHA1Val(); protected: friend class Val; @@ -74,7 +74,7 @@ class SHA256Val : public HashVal { public: static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]); - SHA256Val() : HashVal(new OpaqueType("sha256")) { } + SHA256Val(); protected: friend class Val; @@ -91,7 +91,7 @@ private: class EntropyVal : public OpaqueVal { public: - EntropyVal() : OpaqueVal(new OpaqueType("entropy")) { } + EntropyVal(); bool Feed(const void* data, size_t size); bool Get(double *r_ent, double *r_chisq, double *r_mean, diff --git a/src/TCP.cc b/src/TCP.cc index da977d8157..c291f8e76c 100644 --- a/src/TCP.cc +++ b/src/TCP.cc @@ -566,7 +566,7 @@ void TCP_Analyzer::UpdateInactiveState(double t, else endpoint->SetState(TCP_ENDPOINT_SYN_SENT); - if ( connection_attempt ) + if ( tcp_attempt_delay ) ADD_ANALYZER_TIMER(&TCP_Analyzer::AttemptTimer, t + tcp_attempt_delay, 1, TIMER_TCP_ATTEMPT); @@ -1497,24 +1497,7 @@ void TCP_Analyzer::ExpireTimer(double t) if ( resp->state == TCP_ENDPOINT_INACTIVE ) { - if ( (orig->state == TCP_ENDPOINT_SYN_SENT || - orig->state == TCP_ENDPOINT_SYN_ACK_SENT) ) - { - if ( ! connection_attempt ) - { - // Time out the connection attempt, - // since the AttemptTimer isn't going - // to do it for us, and we don't want - // to clog the data structures with - // old, failed attempts. - Event(connection_timeout); - is_active = 0; - sessions->Remove(Conn()); - return; - } - } - - else if ( orig->state == TCP_ENDPOINT_INACTIVE ) + if ( orig->state == TCP_ENDPOINT_INACTIVE ) { // Nothing ever happened on this connection. // This can occur when we see a trashed diff --git a/src/Val.cc b/src/Val.cc index dd86e71a9e..33b2d0eacd 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1749,7 +1749,7 @@ Val* TableVal::Default(Val* index) if ( def_val->Type()->Tag() != TYPE_FUNC || same_type(def_val->Type(), Type()->YieldType()) ) - return def_val->Ref(); + return def_attr->AttrExpr()->IsConst() ? def_val->Ref() : def_val->Clone(); const Func* f = def_val->AsFunc(); val_list* vl = new val_list(); diff --git a/testing/btest/Baseline/language.table-default-record/out b/testing/btest/Baseline/language.table-default-record/out new file mode 100644 index 0000000000..aeb44cf221 --- /dev/null +++ b/testing/btest/Baseline/language.table-default-record/out @@ -0,0 +1,7 @@ +0 +0 +0 +0 +{ + +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout index 2a53389dc3..a5428dd3b7 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout +++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.cluster-intermediate-update/manager-1..stdout @@ -1 +1,3 @@ -A test metric threshold was crossed with a value of: 100.0 +A test metric threshold was crossed with a value of: 101.0 +End of epoch handler was called +101.0 diff --git a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log index b2a8ef7d4c..8529e18186 100644 --- a/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log +++ b/testing/btest/Baseline/scripts.base.protocols.socks.trace1/socks.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path socks -#open 2012-06-20-17-23-38 +#open 2013-05-02-01-02-50 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version user status request.host request.name request_p bound.host bound.name bound_p #types time string addr port addr port count string string addr string port addr string port -1340213015.276495 UWkUyAuUGXf 10.0.0.55 53994 60.190.189.214 8124 5 - succeeded - www.osnews.com 80 192.168.0.31 - 2688 -#close 2012-06-20-17-28-10 +1340213015.276495 arKYeMETxOg 10.0.0.55 53994 60.190.189.214 8124 5 - succeeded - www.osnews.com 80 192.168.0.31 - 2688 +#close 2013-05-02-01-02-50 diff --git a/testing/btest/coverage/bare-mode-errors.test b/testing/btest/coverage/bare-mode-errors.test index 894c9e67f4..da968d5601 100644 --- a/testing/btest/coverage/bare-mode-errors.test +++ b/testing/btest/coverage/bare-mode-errors.test @@ -3,12 +3,13 @@ # scripts that block after loading, e.g. start listening on a socket. # # Commonly, this test may fail if one forgets to @load some base/ scripts -# when writing a new bro scripts. +# when writing a new bro scripts. Look into "allerrors" to find out +# which script had trouble. # # @TEST-SERIALIZE: comm # # @TEST-EXEC: test -d $DIST/scripts -# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo $script; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0 -# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | sort | uniq > unique_errors +# @TEST-EXEC: for script in `find $DIST/scripts/ -name \*\.bro -not -path '*/site/*'`; do echo "=== $script" >>allerrors; if echo "$script" | egrep -q 'communication/listen|controllee'; then rm -rf load_attempt .bgprocs; btest-bg-run load_attempt bro -b $script; btest-bg-wait -k 2; cat load_attempt/.stderr >>allerrors; else bro -b $script 2>>allerrors; fi done || exit 0 +# @TEST-EXEC: cat allerrors | grep -v "received termination signal" | grep -v '===' | sort | uniq > unique_errors # @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then cp unique_errors unique_errors_no_elasticsearch; fi # @TEST-EXEC: if [ $(grep -c LibCURL_INCLUDE_DIR-NOTFOUND $BUILD/CMakeCache.txt) -ne 0 ]; then btest-diff unique_errors_no_elasticsearch; else btest-diff unique_errors; fi diff --git a/testing/btest/language/table-default-record.bro b/testing/btest/language/table-default-record.bro new file mode 100644 index 0000000000..3894f3ac09 --- /dev/null +++ b/testing/btest/language/table-default-record.bro @@ -0,0 +1,24 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +type Foo: record { + x: count &default=0; +}; + +global foo: table[count] of Foo = {} &default=[]; + +# returns the &default value as usual +print(foo[0]$x); +print(foo[1]$x); + +# these are essentially no-ops since a copy of the &default value is returned +# by the lookup +foo[0]$x = 0; +foo[1]$x = 1; + +# the &default value isn't modified +print(foo[0]$x); +print(foo[1]$x); + +# table membership isn't modified +print(foo); diff --git a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro index 303a0dc852..bed1793721 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/cluster-intermediate-update.bro @@ -4,7 +4,7 @@ # @TEST-EXEC: sleep 3 # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT -# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: btest-diff manager-1/.stdout @TEST-START-FILE cluster-layout.bro @@ -20,8 +20,15 @@ redef Log::default_rotation_interval = 0secs; event bro_init() &priority=5 { local r1: SumStats::Reducer = [$stream="test.metric", $apply=set(SumStats::SUM)]; - SumStats::create([$epoch=1hr, + SumStats::create([$epoch=10secs, $reducers=set(r1), + $epoch_finished(data: SumStats::ResultTable) = + { + print "End of epoch handler was called"; + for ( res in data ) + print data[res]["test.metric"]$sum; + terminate(); + }, $threshold_val(key: SumStats::Key, result: SumStats::Result) = { return double_to_count(result["test.metric"]$sum); @@ -30,7 +37,6 @@ event bro_init() &priority=5 $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = { print fmt("A test metric threshold was crossed with a value of: %.1f", result["test.metric"]$sum); - terminate(); }]); } @@ -52,8 +58,13 @@ event remote_connection_handshake_done(p: event_peer) if ( p$descr == "manager-1" ) { if ( Cluster::node == "worker-1" ) + { schedule 0.1sec { do_stats(1) }; + schedule 5secs { do_stats(60) }; + } if ( Cluster::node == "worker-2" ) - schedule 0.5sec { do_stats(99) }; + schedule 0.5sec { do_stats(40) }; } } + +