From 663082e2d51ada8011cd64a27c54ee533ce18c5b Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 5 May 2013 11:18:19 -0700 Subject: [PATCH 1/5] reservoir sampler. untested. --- .../frameworks/sumstats/plugins/__load__.bro | 1 + .../base/frameworks/sumstats/plugins/last.bro | 49 ++++++++ .../frameworks/sumstats/plugins/sample.bro | 116 ++++++++++++++---- .../.stdout | 9 ++ .../scripts/base/frameworks/sumstats/last.bro | 47 +++++++ 5 files changed, 198 insertions(+), 24 deletions(-) create mode 100644 scripts/base/frameworks/sumstats/plugins/last.bro create mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.last/.stdout create mode 100644 testing/btest/scripts/base/frameworks/sumstats/last.bro diff --git a/scripts/base/frameworks/sumstats/plugins/__load__.bro b/scripts/base/frameworks/sumstats/plugins/__load__.bro index 3b2bb553e6..c0ee3a6767 100644 --- a/scripts/base/frameworks/sumstats/plugins/__load__.bro +++ b/scripts/base/frameworks/sumstats/plugins/__load__.bro @@ -1,4 +1,5 @@ @load ./average +@load ./last @load ./max @load ./min @load ./sample diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro new file mode 100644 index 0000000000..d0587bde08 --- /dev/null +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -0,0 +1,49 @@ +@load base/frameworks/sumstats +@load base/utils/queue + +module SumStats; + +export { + redef record Reducer += { + ## A number of sample Observations to collect. + samples: count &default=0; + }; + + redef record ResultVal += { + ## This is the queue where samples are maintained. Use the + ## :bro:see:`SumStats::get_samples` function to get a vector of the samples. + samples: Queue::Queue &optional; + }; + + ## Get a vector of sample Observation values from a ResultVal. + global get_samples: function(rv: ResultVal): vector of Observation; +} + +function get_samples(rv: ResultVal): vector of Observation + { + local s: vector of Observation = vector(); + if ( rv?$samples ) + Queue::get_vector(rv$samples, s); + return s; + } + +hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) + { + if ( r$samples > 0 ) + { + if ( ! rv?$samples ) + rv$samples = Queue::init([$max_len=r$samples]); + Queue::put(rv$samples, obs); + } + } + +hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) + { + # Merge $samples + if ( rv1?$samples && rv2?$samples ) + result$samples = Queue::merge(rv1$samples, rv2$samples); + else if ( rv1?$samples ) + result$samples = rv1$samples; + else if ( rv2?$samples ) + result$samples = rv2$samples; + } diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index dc2f438c79..b04d2cf57c 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -1,49 +1,117 @@ @load base/frameworks/sumstats/main -@load base/utils/queue module SumStats; export { + redef enum Calculation += { + ## Get uniquely distributed random samples from the observation stream + SAMPLE + }; + redef record Reducer += { ## A number of sample Observations to collect. - samples: count &default=0; + num_samples: count &default=0; }; redef record ResultVal += { - ## This is the queue where samples are maintained. Use the - ## :bro:see:`SumStats::get_samples` function to get a vector of the samples. - samples: Queue::Queue &optional; - }; + ## This is the vector in which the samples are maintained. + sample_vector: vector of Observation &default=vector(); - ## Get a vector of sample Observation values from a ResultVal. - global get_samples: function(rv: ResultVal): vector of Observation; + ## Number of total observed elements. + sample_elements: count &default=0; + }; } -function get_samples(rv: ResultVal): vector of Observation +redef record ResultVal += { + # Internal use only. This is not meant to be publically available + # and just a copy of num_samples from the Reducer. Needed for availability + # in the compose hook. + num_samples: count &default=0; +}; + +hook init_resultval_hook(r: Reducer, rv: ResultVal) { - local s: vector of Observation = vector(); - if ( rv?$samples ) - Queue::get_vector(rv$samples, s); - return s; + if ( SAMPLE in r$apply ) + rv$num_samples = r$num_samples; } +function sample_add_sample(obs:Observation, rv: ResultVal) + { + ++rv$sample_elements; + + if ( |rv$sample_vector| < rv$num_samples ) + rv$sample_vector[|rv$sample_vector|] = obs; + else + { + local ra = rand(rv$sample_elements); + if ( ra < rv$num_samples ) + rv$sample_vector[ra] = obs; + } + + } + hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { - if ( r$samples > 0 ) + if ( SAMPLE in r$apply ) { - if ( ! rv?$samples ) - rv$samples = Queue::init([$max_len=r$samples]); - Queue::put(rv$samples, obs); + sample_add_sample(obs, rv); } } hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) { - # Merge $samples - if ( rv1?$samples && rv2?$samples ) - result$samples = Queue::merge(rv1$samples, rv2$samples); - else if ( rv1?$samples ) - result$samples = rv1$samples; - else if ( rv2?$samples ) - result$samples = rv2$samples; + if ( rv1$num_samples != rv2$num_samples ) + { + Reporter::error("Merging sample sets with differing sizes is not supported"); + return; + } + + local num_samples = rv1$num_samples; + + if ( |rv1$sample_vector| > num_samples || |rv2$sample_vector| > num_samples ) + { + Reporter::error("Sample vector with too many elements. Aborting."); + return; + } + + + if ( |rv1$sample_vector| != num_samples && |rv2$sample_vector| < num_samples ) + { + if ( |rv1$sample_vector| != rv1$sample_elements || |rv2$sample_vector| < rv2$sample_elements ) + { + Reporter::error("Mismatch in sample element size and tracking. Aborting merge"); + return; + } + + for ( i in rv1$sample_vector ) + sample_add_sample(rv1$sample_vector[i], result); + + for ( i in rv2$sample_vector) + sample_add_sample(rv2$sample_vector[i], result); + } + else + { + local other_vector: vector of Observation; + local othercount: count; + if ( rv1$sample_elements > rv2$sample_elements ) + { + result$sample_vector = copy(rv1$sample_vector); + other_vector = rv2$sample_vector; + othercount = rv2$sample_elements; + } + else + { + result$sample_vector = copy(rv2$sample_vector); + other_vector = rv1$sample_vector; + othercount = rv1$sample_elements; + } + + local totalcount = rv1$sample_elements + rv2$sample_elements; + + for ( i in other_vector ) + { + if ( rand(totalcount) <= othercount ) + result$sample_vector[i] = other_vector[i]; + } + } } diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.last/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.last/.stdout new file mode 100644 index 0000000000..35219765af --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.last/.stdout @@ -0,0 +1,9 @@ +6.5.4.3 +[[num=2, dbl=, str=]] +1 +1.2.3.4 +[[num=5, dbl=, str=], [num=51, dbl=, str=]] +20 +7.2.1.5 +[[num=1, dbl=, str=]] +1 diff --git a/testing/btest/scripts/base/frameworks/sumstats/last.bro b/testing/btest/scripts/base/frameworks/sumstats/last.bro new file mode 100644 index 0000000000..e0cef0ec10 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/sumstats/last.bro @@ -0,0 +1,47 @@ +# @TEST-EXEC: bro %INPUT +# @TEST-EXEC: btest-diff .stdout + +event bro_init() &priority=5 + { + local r1: SumStats::Reducer = [$stream="test.metric", + $apply=set(SumStats::SAMPLE), $num_samples=2]; + SumStats::create([$epoch=3secs, + $reducers=set(r1), + $epoch_finished(data: SumStats::ResultTable) = + { + for ( key in data ) + { + print key$host; + local r = data[key]["test.metric"]; + print r$sample_vector; + print r$sample_elements; + } + } + ]); + + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=5]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=22]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=94]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=50]); + # I checked the random numbers. seems legit. + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test.metric", [$host=1.2.3.4], [$num=51]); + + SumStats::observe("test.metric", [$host=6.5.4.3], [$num=2]); + SumStats::observe("test.metric", [$host=7.2.1.5], [$num=1]); + } + From d939c2bdfc045132ce71a59572399bedbf0f84a2 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Mon, 13 May 2013 22:11:17 -0700 Subject: [PATCH 2/5] add tests for sampler --- .../frameworks/sumstats/plugins/sample.bro | 1 + .../manager-1..stdout | 12 ++ .../.stdout | 0 .../frameworks/sumstats/sample-cluster.bro | 111 ++++++++++++++++++ .../sumstats/{last.bro => sample.bro} | 0 5 files changed, 124 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout rename testing/btest/Baseline/{scripts.base.frameworks.sumstats.last => scripts.base.frameworks.sumstats.sample}/.stdout (100%) create mode 100644 testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro rename testing/btest/scripts/base/frameworks/sumstats/{last.bro => sample.bro} (100%) diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index b04d2cf57c..12394fa0e9 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -67,6 +67,7 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) } local num_samples = rv1$num_samples; + result$num_samples = num_samples; if ( |rv1$sample_vector| > num_samples || |rv2$sample_vector| > num_samples ) { diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout new file mode 100644 index 0000000000..579bd109d9 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout @@ -0,0 +1,12 @@ +6.5.4.3 +[[num=2, dbl=, str=], [num=5, dbl=, str=]] +2 +10.10.10.10 +[[num=5, dbl=, str=]] +1 +1.2.3.4 +[[num=5, dbl=, str=], [num=22, dbl=, str=], [num=94, dbl=, str=], [num=91, dbl=, str=], [num=52, dbl=, str=]] +0 +7.2.1.5 +[[num=1, dbl=, str=], [num=91, dbl=, str=]] +2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.last/.stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample/.stdout similarity index 100% rename from testing/btest/Baseline/scripts.base.frameworks.sumstats.last/.stdout rename to testing/btest/Baseline/scripts.base.frameworks.sumstats.sample/.stdout diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro new file mode 100644 index 0000000000..c83cf7028e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro @@ -0,0 +1,111 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 BROPATH=$BROPATH:.. CLUSTER_NODE=manager-1 bro %INPUT +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT +# @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT +# @TEST-EXEC: btest-bg-wait 15 + +# @TEST-EXEC: btest-diff manager-1/.stdout + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth1"], +}; +@TEST-END-FILE + +redef Log::default_rotation_interval = 0secs; + +global n = 0; + +event bro_init() &priority=5 + { + local r1: SumStats::Reducer = [$stream="test", $apply=set(SumStats::SAMPLE), $num_samples=5]; + SumStats::create([$epoch=5secs, + $reducers=set(r1), + $epoch_finished(rt: SumStats::ResultTable) = + { + for ( key in rt ) + { + print key$host; + local r = rt[key]["test"]; + print r$sample_vector; + print r$sample_elements; + } + + terminate(); + }]); + } + +event remote_connection_closed(p: event_peer) + { + terminate(); + } + +global ready_for_data: event(); +redef Cluster::manager2worker_events += /^ready_for_data$/; + +event ready_for_data() + { + if ( Cluster::node == "worker-1" ) + { + SumStats::observe("test", [$host=1.2.3.4], [$num=5]); + SumStats::observe("test", [$host=1.2.3.4], [$num=22]); + SumStats::observe("test", [$host=1.2.3.4], [$num=94]); + SumStats::observe("test", [$host=1.2.3.4], [$num=50]); + # I checked the random numbers. seems legit. + SumStats::observe("test", [$host=1.2.3.4], [$num=51]); + SumStats::observe("test", [$host=1.2.3.4], [$num=61]); + SumStats::observe("test", [$host=1.2.3.4], [$num=61]); + SumStats::observe("test", [$host=1.2.3.4], [$num=71]); + SumStats::observe("test", [$host=1.2.3.4], [$num=81]); + SumStats::observe("test", [$host=1.2.3.4], [$num=91]); + SumStats::observe("test", [$host=1.2.3.4], [$num=101]); + SumStats::observe("test", [$host=1.2.3.4], [$num=111]); + SumStats::observe("test", [$host=1.2.3.4], [$num=121]); + SumStats::observe("test", [$host=1.2.3.4], [$num=131]); + SumStats::observe("test", [$host=1.2.3.4], [$num=141]); + SumStats::observe("test", [$host=1.2.3.4], [$num=151]); + SumStats::observe("test", [$host=1.2.3.4], [$num=161]); + SumStats::observe("test", [$host=1.2.3.4], [$num=171]); + SumStats::observe("test", [$host=1.2.3.4], [$num=181]); + SumStats::observe("test", [$host=1.2.3.4], [$num=191]); + + SumStats::observe("test", [$host=6.5.4.3], [$num=2]); + SumStats::observe("test", [$host=7.2.1.5], [$num=1]); + } + if ( Cluster::node == "worker-2" ) + { + SumStats::observe("test", [$host=1.2.3.4], [$num=75]); + SumStats::observe("test", [$host=1.2.3.4], [$num=30]); + SumStats::observe("test", [$host=1.2.3.4], [$num=3]); + SumStats::observe("test", [$host=1.2.3.4], [$num=57]); + SumStats::observe("test", [$host=1.2.3.4], [$num=52]); + SumStats::observe("test", [$host=1.2.3.4], [$num=61]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=1.2.3.4], [$num=95]); + SumStats::observe("test", [$host=6.5.4.3], [$num=5]); + SumStats::observe("test", [$host=7.2.1.5], [$num=91]); + SumStats::observe("test", [$host=10.10.10.10], [$num=5]); + } + } + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + +global peer_count = 0; +event remote_connection_handshake_done(p: event_peer) &priority=-5 + { + ++peer_count; + if ( peer_count == 2 ) + event ready_for_data(); + } + +@endif diff --git a/testing/btest/scripts/base/frameworks/sumstats/last.bro b/testing/btest/scripts/base/frameworks/sumstats/sample.bro similarity index 100% rename from testing/btest/scripts/base/frameworks/sumstats/last.bro rename to testing/btest/scripts/base/frameworks/sumstats/sample.bro From b0c4dcdfedeeaba5b4b4e7e8522ff5058e3d5c25 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 15 May 2013 01:09:52 -0700 Subject: [PATCH 3/5] make last plugin nicer and samplify sqli detector --- .../base/frameworks/sumstats/plugins/last.bro | 45 ++++++++++--------- scripts/policy/protocols/http/detect-sqli.bro | 8 ++-- 2 files changed, 29 insertions(+), 24 deletions(-) diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index d0587bde08..58baa85b98 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -4,46 +4,51 @@ module SumStats; export { + redef enum Calculation += { + ## Keep last X observations in Queue + LAST + }; + redef record Reducer += { - ## A number of sample Observations to collect. - samples: count &default=0; + ## number of elements to keep. + num_last_elements: count &default=0; }; redef record ResultVal += { - ## This is the queue where samples are maintained. Use the - ## :bro:see:`SumStats::get_samples` function to get a vector of the samples. - samples: Queue::Queue &optional; + ## This is the queue where elements are maintained. Use the + ## :bro:see:`SumStats::get_elements` function to get a vector of the samples. + last_elements: Queue::Queue &optional; }; - ## Get a vector of sample Observation values from a ResultVal. - global get_samples: function(rv: ResultVal): vector of Observation; + ## Get a vector of element values from a ResultVal. + global get_elements: function(rv: ResultVal): vector of Observation; } -function get_samples(rv: ResultVal): vector of Observation +function get_elements(rv: ResultVal): vector of Observation { local s: vector of Observation = vector(); - if ( rv?$samples ) - Queue::get_vector(rv$samples, s); + if ( rv?$last_elements ) + Queue::get_vector(rv$last_elements, s); return s; } hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) { - if ( r$samples > 0 ) + if ( LAST in r$apply && r$num_last_elements > 0 ) { - if ( ! rv?$samples ) - rv$samples = Queue::init([$max_len=r$samples]); - Queue::put(rv$samples, obs); + if ( ! rv?$last_elements ) + rv$last_elements = Queue::init([$max_len=r$num_last_elements]); + Queue::put(rv$last_elements, obs); } } hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) { # Merge $samples - if ( rv1?$samples && rv2?$samples ) - result$samples = Queue::merge(rv1$samples, rv2$samples); - else if ( rv1?$samples ) - result$samples = rv1$samples; - else if ( rv2?$samples ) - result$samples = rv2$samples; + if ( rv1?$last_elements && rv2?$last_elements ) + result$last_elements = Queue::merge(rv1$last_elements, rv2$last_elements); + else if ( rv1?$last_elements ) + result$last_elements = rv1$last_elements; + else if ( rv2?$last_elements ) + result$last_elements = rv2$last_elements; } diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index 11dba0dc46..40d3805b92 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -63,7 +63,7 @@ event bro_init() &priority=3 # Add filters to the metrics so that the metrics framework knows how to # determine when it looks like an actual attack and how to respond when # thresholds are crossed. - local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples]; + local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples]; SumStats::create([$epoch=sqli_requests_interval, $reducers=set(r1), $threshold_val(key: SumStats::Key, result: SumStats::Result) = @@ -76,12 +76,12 @@ event bro_init() &priority=3 local r = result["http.sqli.attacker"]; NOTICE([$note=SQL_Injection_Attacker, $msg="An SQL injection attacker was discovered!", - $email_body_sections=vector(format_sqli_samples(SumStats::get_samples(r))), + $email_body_sections=vector(format_sqli_samples(r$sample_vector)), $src=key$host, $identifier=cat(key$host)]); }]); - local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM), $samples=collect_SQLi_samples]; + local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples]; SumStats::create([$epoch=sqli_requests_interval, $reducers=set(r2), $threshold_val(key: SumStats::Key, result: SumStats::Result) = @@ -94,7 +94,7 @@ event bro_init() &priority=3 local r = result["http.sqli.victim"]; NOTICE([$note=SQL_Injection_Victim, $msg="An SQL injection victim was discovered!", - $email_body_sections=vector(format_sqli_samples(SumStats::get_samples(r))), + $email_body_sections=vector(format_sqli_samples(r$sample_vector)), $src=key$host, $identifier=cat(key$host)]); }]); From 80962ad74be68eab2e4623737ba367e17ecd4a71 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 15 May 2013 09:44:43 -0700 Subject: [PATCH 4/5] change names of data structures after talking with seth --- .../base/frameworks/sumstats/plugins/last.bro | 8 ++--- .../frameworks/sumstats/plugins/sample.bro | 32 +++++++++---------- scripts/policy/protocols/http/detect-sqli.bro | 4 +-- .../frameworks/sumstats/sample-cluster.bro | 2 +- .../base/frameworks/sumstats/sample.bro | 2 +- 5 files changed, 24 insertions(+), 24 deletions(-) diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index 58baa85b98..1c70db372c 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -5,7 +5,7 @@ module SumStats; export { redef enum Calculation += { - ## Keep last X observations in Queue + ## Keep last X observations in a queue LAST }; @@ -16,15 +16,15 @@ export { redef record ResultVal += { ## This is the queue where elements are maintained. Use the - ## :bro:see:`SumStats::get_elements` function to get a vector of the samples. + ## :bro:see:`SumStats::get_elements` function to get a vector of the current element values. last_elements: Queue::Queue &optional; }; ## Get a vector of element values from a ResultVal. - global get_elements: function(rv: ResultVal): vector of Observation; + global get_last_elements: function(rv: ResultVal): vector of Observation; } -function get_elements(rv: ResultVal): vector of Observation +function get_last_elements(rv: ResultVal): vector of Observation { local s: vector of Observation = vector(); if ( rv?$last_elements ) diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index 12394fa0e9..328067c939 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -15,7 +15,7 @@ export { redef record ResultVal += { ## This is the vector in which the samples are maintained. - sample_vector: vector of Observation &default=vector(); + samples: vector of Observation &default=vector(); ## Number of total observed elements. sample_elements: count &default=0; @@ -39,13 +39,13 @@ function sample_add_sample(obs:Observation, rv: ResultVal) { ++rv$sample_elements; - if ( |rv$sample_vector| < rv$num_samples ) - rv$sample_vector[|rv$sample_vector|] = obs; + if ( |rv$samples| < rv$num_samples ) + rv$samples[|rv$samples|] = obs; else { local ra = rand(rv$sample_elements); if ( ra < rv$num_samples ) - rv$sample_vector[ra] = obs; + rv$samples[ra] = obs; } } @@ -69,26 +69,26 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) local num_samples = rv1$num_samples; result$num_samples = num_samples; - if ( |rv1$sample_vector| > num_samples || |rv2$sample_vector| > num_samples ) + if ( |rv1$samples| > num_samples || |rv2$samples| > num_samples ) { Reporter::error("Sample vector with too many elements. Aborting."); return; } - if ( |rv1$sample_vector| != num_samples && |rv2$sample_vector| < num_samples ) + if ( |rv1$samples| != num_samples && |rv2$samples| < num_samples ) { - if ( |rv1$sample_vector| != rv1$sample_elements || |rv2$sample_vector| < rv2$sample_elements ) + if ( |rv1$samples| != rv1$sample_elements || |rv2$samples| < rv2$sample_elements ) { Reporter::error("Mismatch in sample element size and tracking. Aborting merge"); return; } - for ( i in rv1$sample_vector ) - sample_add_sample(rv1$sample_vector[i], result); + for ( i in rv1$samples ) + sample_add_sample(rv1$samples[i], result); - for ( i in rv2$sample_vector) - sample_add_sample(rv2$sample_vector[i], result); + for ( i in rv2$samples) + sample_add_sample(rv2$samples[i], result); } else { @@ -96,14 +96,14 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) local othercount: count; if ( rv1$sample_elements > rv2$sample_elements ) { - result$sample_vector = copy(rv1$sample_vector); - other_vector = rv2$sample_vector; + result$samples = copy(rv1$samples); + other_vector = rv2$samples; othercount = rv2$sample_elements; } else { - result$sample_vector = copy(rv2$sample_vector); - other_vector = rv1$sample_vector; + result$samples = copy(rv2$samples); + other_vector = rv1$samples; othercount = rv1$sample_elements; } @@ -112,7 +112,7 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) for ( i in other_vector ) { if ( rand(totalcount) <= othercount ) - result$sample_vector[i] = other_vector[i]; + result$samples[i] = other_vector[i]; } } } diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index 40d3805b92..8671bbd165 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -76,7 +76,7 @@ event bro_init() &priority=3 local r = result["http.sqli.attacker"]; NOTICE([$note=SQL_Injection_Attacker, $msg="An SQL injection attacker was discovered!", - $email_body_sections=vector(format_sqli_samples(r$sample_vector)), + $email_body_sections=vector(format_sqli_samples(r$samples)), $src=key$host, $identifier=cat(key$host)]); }]); @@ -94,7 +94,7 @@ event bro_init() &priority=3 local r = result["http.sqli.victim"]; NOTICE([$note=SQL_Injection_Victim, $msg="An SQL injection victim was discovered!", - $email_body_sections=vector(format_sqli_samples(r$sample_vector)), + $email_body_sections=vector(format_sqli_samples(r$samples)), $src=key$host, $identifier=cat(key$host)]); }]); diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro index c83cf7028e..458bef01de 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro @@ -31,7 +31,7 @@ event bro_init() &priority=5 { print key$host; local r = rt[key]["test"]; - print r$sample_vector; + print r$samples; print r$sample_elements; } diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample.bro b/testing/btest/scripts/base/frameworks/sumstats/sample.bro index e0cef0ec10..04d7b4f256 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/sample.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/sample.bro @@ -13,7 +13,7 @@ event bro_init() &priority=5 { print key$host; local r = data[key]["test.metric"]; - print r$sample_vector; + print r$samples; print r$sample_elements; } } From ab6d5b08a81c866a836d92d7d3f41c0e68f0ec72 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 15 May 2013 11:33:25 -0700 Subject: [PATCH 5/5] finishing touches, make test more robust, rename function in last again --- .../base/frameworks/sumstats/plugins/last.bro | 4 ++-- .../frameworks/sumstats/plugins/sample.bro | 1 + .../manager-1..stdout | 12 ------------ .../out | 18 ++++++++++++++++++ .../frameworks/sumstats/sample-cluster.bro | 8 +++++--- 5 files changed, 26 insertions(+), 17 deletions(-) delete mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout create mode 100644 testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/out diff --git a/scripts/base/frameworks/sumstats/plugins/last.bro b/scripts/base/frameworks/sumstats/plugins/last.bro index 1c70db372c..e2cf31c902 100644 --- a/scripts/base/frameworks/sumstats/plugins/last.bro +++ b/scripts/base/frameworks/sumstats/plugins/last.bro @@ -21,10 +21,10 @@ export { }; ## Get a vector of element values from a ResultVal. - global get_last_elements: function(rv: ResultVal): vector of Observation; + global get_last: function(rv: ResultVal): vector of Observation; } -function get_last_elements(rv: ResultVal): vector of Observation +function get_last(rv: ResultVal): vector of Observation { local s: vector of Observation = vector(); if ( rv?$last_elements ) diff --git a/scripts/base/frameworks/sumstats/plugins/sample.bro b/scripts/base/frameworks/sumstats/plugins/sample.bro index 328067c939..b722d6d7a3 100644 --- a/scripts/base/frameworks/sumstats/plugins/sample.bro +++ b/scripts/base/frameworks/sumstats/plugins/sample.bro @@ -108,6 +108,7 @@ hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) } local totalcount = rv1$sample_elements + rv2$sample_elements; + result$sample_elements = totalcount; for ( i in other_vector ) { diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout b/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout deleted file mode 100644 index 579bd109d9..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/manager-1..stdout +++ /dev/null @@ -1,12 +0,0 @@ -6.5.4.3 -[[num=2, dbl=, str=], [num=5, dbl=, str=]] -2 -10.10.10.10 -[[num=5, dbl=, str=]] -1 -1.2.3.4 -[[num=5, dbl=, str=], [num=22, dbl=, str=], [num=94, dbl=, str=], [num=91, dbl=, str=], [num=52, dbl=, str=]] -0 -7.2.1.5 -[[num=1, dbl=, str=], [num=91, dbl=, str=]] -2 diff --git a/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/out b/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/out new file mode 100644 index 0000000000..2451b82f45 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.sumstats.sample-cluster/out @@ -0,0 +1,18 @@ +1 +1.2.3.4 +10.10.10.10 +2 +2 +34 +6.5.4.3 +7.2.1.5 +[num=1, dbl=, str=] +[num=2, dbl=, str=] +[num=22, dbl=, str=] +[num=5, dbl=, str=] +[num=5, dbl=, str=] +[num=5, dbl=, str=] +[num=52, dbl=, str=] +[num=91, dbl=, str=] +[num=91, dbl=, str=] +[num=94, dbl=, str=] diff --git a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro index 458bef01de..1b0f0eec94 100644 --- a/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro +++ b/testing/btest/scripts/base/frameworks/sumstats/sample-cluster.bro @@ -5,8 +5,8 @@ # @TEST-EXEC: btest-bg-run worker-1 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-1 bro %INPUT # @TEST-EXEC: btest-bg-run worker-2 BROPATH=$BROPATH:.. CLUSTER_NODE=worker-2 bro %INPUT # @TEST-EXEC: btest-bg-wait 15 - -# @TEST-EXEC: btest-diff manager-1/.stdout +# @TEST-EXEC: cat manager-1/.stdout | sort > out +# @TEST-EXEC: btest-diff out @TEST-START-FILE cluster-layout.bro redef Cluster::nodes = { @@ -31,7 +31,9 @@ event bro_init() &priority=5 { print key$host; local r = rt[key]["test"]; - print r$samples; + for ( sample in r$samples ) { + print r$samples[sample]; + } print r$sample_elements; }