From 2cf90d986e15e643fb7e4e86ca39833a0f245b54 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 5 Nov 2013 11:34:32 -0500 Subject: [PATCH 01/23] Fix resp_size in ssh.log, require a minimum resp_size for the heuristic. Some work on geodata, but still a WIP. --- scripts/base/protocols/ssh/main.bro | 16 +++++++++------- scripts/policy/protocols/ssh/geo-data.bro | 22 +++++++++++++++------- 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index 5b452d432c..fd0abad67d 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -107,10 +107,10 @@ function check_ssh_connection(c: connection, done: bool) # this matches the conditions for a failed login. Failed # logins are only detected at connection state removal. - if ( # Require originators to have sent at least 50 bytes. - c$orig$size > 50 && + if ( # Require originators and responders to have sent at least 50 bytes. + c$orig$size > 50 && c$resp$size > 50 && # Responders must be below 4000 bytes. - c$resp$size < 4000 && + c$resp$size < authentication_data_size && # Responder must have sent fewer than 40 packets. c$resp$num_pkts < 40 && # If there was a content gap we can't reliably do this heuristic. @@ -122,7 +122,7 @@ function check_ssh_connection(c: connection, done: bool) event SSH::heuristic_failed_login(c); } - if ( c$resp$size > authentication_data_size ) + if ( c$resp$size >= authentication_data_size ) { c$ssh$status = "success"; event SSH::heuristic_successful_login(c); @@ -132,7 +132,7 @@ function check_ssh_connection(c: connection, done: bool) { # If this connection is still being tracked, then it's possible # to watch for it to be a successful connection. - if ( c$resp$size > authentication_data_size ) + if ( c$resp$size >= authentication_data_size ) { c$ssh$status = "success"; event SSH::heuristic_successful_login(c); @@ -150,8 +150,6 @@ function check_ssh_connection(c: connection, done: bool) # after detection is done. c$ssh$done=T; - Log::write(SSH::LOG, c$ssh); - if ( skip_processing_after_detection ) { # Stop watching this connection, we don't care about it anymore. @@ -164,7 +162,11 @@ function check_ssh_connection(c: connection, done: bool) event connection_state_remove(c: connection) &priority=-5 { if ( c?$ssh ) + { check_ssh_connection(c, T); + c$ssh$resp_size = c$resp$size; + Log::write(SSH::LOG, c$ssh); + } } event ssh_watcher(c: connection) diff --git a/scripts/policy/protocols/ssh/geo-data.bro b/scripts/policy/protocols/ssh/geo-data.bro index 3abc19d337..a5fed986ef 100644 --- a/scripts/policy/protocols/ssh/geo-data.bro +++ b/scripts/policy/protocols/ssh/geo-data.bro @@ -24,21 +24,29 @@ export { const watched_countries: set[string] = {"RO"} &redef; } +function get_location(c: connection): geo_location + { + local lookup_ip = (c$ssh$direction == OUTBOUND) ? c$id$resp_h : c$id$orig_h; + return lookup_location(lookup_ip); + } + event SSH::heuristic_successful_login(c: connection) &priority=5 { - local location: geo_location; - location = (c$ssh$direction == OUTBOUND) ? - lookup_location(c$id$resp_h) : lookup_location(c$id$orig_h); - # Add the location data to the SSH record. - c$ssh$remote_location = location; + c$ssh$remote_location = get_location(c); - if ( location?$country_code && location$country_code in watched_countries ) + if ( c$ssh$remote_location?$country_code && c$ssh$remote_location$country_code in watched_countries ) { NOTICE([$note=Watched_Country_Login, $conn=c, $msg=fmt("SSH login %s watched country: %s", (c$ssh$direction == OUTBOUND) ? "to" : "from", - location$country_code)]); + c$ssh$remote_location$country_code)]); } } + +event SSH::heuristic_failed_login(c: connection) &priority=5 + { + # Add the location data to the SSH record. + c$ssh$remote_location = get_location(c); + } From d108481e730cd14140895f708722d6d0b55eb988 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 5 Nov 2013 11:58:00 -0500 Subject: [PATCH 02/23] Remove resp_size from the log. Refactor when we write out to the log a bit. Geodata now works reliably. --- scripts/base/protocols/ssh/main.bro | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/scripts/base/protocols/ssh/main.bro b/scripts/base/protocols/ssh/main.bro index fd0abad67d..33b0c84147 100644 --- a/scripts/base/protocols/ssh/main.bro +++ b/scripts/base/protocols/ssh/main.bro @@ -37,12 +37,6 @@ export { client: string &log &optional; ## Software string from the server. server: string &log &optional; - ## Amount of data returned from the server. This is currently - ## the only measure of the success heuristic and it is logged to - ## assist analysts looking at the logs to make their own - ## determination about the success on a case-by-case basis. - resp_size: count &log &default=0; - ## Indicate if the SSH session is done being watched. done: bool &default=F; }; @@ -159,13 +153,23 @@ function check_ssh_connection(c: connection, done: bool) } +event heuristic_successful_login(c: connection) &priority=-5 + { + Log::write(SSH::LOG, c$ssh); + } + +event heuristic_failed_login(c: connection) &priority=-5 + { + Log::write(SSH::LOG, c$ssh); + } + event connection_state_remove(c: connection) &priority=-5 { if ( c?$ssh ) { check_ssh_connection(c, T); - c$ssh$resp_size = c$resp$size; - Log::write(SSH::LOG, c$ssh); + if ( c$ssh$status == "undetermined" ) + Log::write(SSH::LOG, c$ssh); } } From 09779836cbbea6744114fba67bf0aa277cce4131 Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 5 Nov 2013 12:06:33 -0500 Subject: [PATCH 03/23] Update VirusTotal URL to work with changes to their website. --- scripts/policy/frameworks/files/detect-MHR.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro index 5ed8715c9b..753372e82e 100644 --- a/scripts/policy/frameworks/files/detect-MHR.bro +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -48,7 +48,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt("https://www.virustotal.com/en/search/?query=%s", hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } From 8ad2ab44e2f077d20bbb5142a97b80fdf0d7a40c Mon Sep 17 00:00:00 2001 From: Vlad Grigorescu Date: Tue, 5 Nov 2013 17:10:19 -0500 Subject: [PATCH 04/23] Change MHR notice sub message URL to a redef. --- scripts/policy/frameworks/files/detect-MHR.bro | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/policy/frameworks/files/detect-MHR.bro b/scripts/policy/frameworks/files/detect-MHR.bro index 753372e82e..1a26b9be32 100644 --- a/scripts/policy/frameworks/files/detect-MHR.bro +++ b/scripts/policy/frameworks/files/detect-MHR.bro @@ -23,6 +23,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold @@ -48,7 +53,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/search/?query=%s", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } From dc6ef20511e08114445c23de9eb084425c65d241 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 6 Nov 2013 00:23:51 -0800 Subject: [PATCH 05/23] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- aux/broctl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/aux/broccoli b/aux/broccoli index 02e5c1e1f9..96127785c7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 02e5c1e1f993ef0fea3e2a59c34df9f40839e398 +Subproject commit 96127785c7301c65b4f93d7e3091e8255ba00ca5 diff --git a/aux/broctl b/aux/broctl index aeb8501a50..1e7df03f6c 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit aeb8501a50dcf33c53e7fe776b6e333327c11861 +Subproject commit 1e7df03f6cb79872392ae74cf23da0d8a03c31df From bd4cc712c5124a26b8925a04fcc391d1881b3998 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 6 Nov 2013 07:22:27 -0800 Subject: [PATCH 06/23] Updating submodule(s). [nomail] --- aux/broccoli | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broccoli b/aux/broccoli index 96127785c7..8600d3afc7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 96127785c7301c65b4f93d7e3091e8255ba00ca5 +Subproject commit 8600d3afc748aadbdd729c7bac9e3f69e829c23f From fab47cc7498464a8826a7a91f0b97d0963b32064 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 6 Nov 2013 13:52:29 -0500 Subject: [PATCH 07/23] Added a document for the SumStats framework. --- doc/frameworks/index.rst | 1 + doc/frameworks/sumstats-countconns.bro | 36 +++++++++ doc/frameworks/sumstats-toy-scan.bro | 45 +++++++++++ doc/frameworks/sumstats.rst | 102 +++++++++++++++++++++++++ testing/btest/Traces/nmap-vsn.trace | Bin 0 -> 33196 bytes 5 files changed, 184 insertions(+) create mode 100644 doc/frameworks/sumstats-countconns.bro create mode 100644 doc/frameworks/sumstats-toy-scan.bro create mode 100644 doc/frameworks/sumstats.rst create mode 100644 testing/btest/Traces/nmap-vsn.trace diff --git a/doc/frameworks/index.rst b/doc/frameworks/index.rst index d5b771b15e..f8c681d795 100644 --- a/doc/frameworks/index.rst +++ b/doc/frameworks/index.rst @@ -13,4 +13,5 @@ Frameworks logging notice signatures + sumstats diff --git a/doc/frameworks/sumstats-countconns.bro b/doc/frameworks/sumstats-countconns.bro new file mode 100644 index 0000000000..a10be54376 --- /dev/null +++ b/doc/frameworks/sumstats-countconns.bro @@ -0,0 +1,36 @@ +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } \ No newline at end of file diff --git a/doc/frameworks/sumstats-toy-scan.bro b/doc/frameworks/sumstats-toy-scan.bro new file mode 100644 index 0000000000..c435fb8997 --- /dev/null +++ b/doc/frameworks/sumstats-toy-scan.bro @@ -0,0 +1,45 @@ +@load base/frameworks/sumstats + +# We use the connection_attempted event limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } \ No newline at end of file diff --git a/doc/frameworks/sumstats.rst b/doc/frameworks/sumstats.rst new file mode 100644 index 0000000000..e06ceaf2c8 --- /dev/null +++ b/doc/frameworks/sumstats.rst @@ -0,0 +1,102 @@ +================== +Summary Statistics +================== + +.. rst-class:: opening + + Measuring aspects of network traffic is an extremely common task in Bro. + Bro provides data structures which make this very easy as wellin + simplistic cases such as size limited trace file processing. In real- + world deployments though, there are difficulties that arise from + clusterization (many processes sniffing traffic) and unbounded data sets + (traffic never stops). The Summary Statistics (otherwise referred to as + SumStats) framework aims to define a mechanism for consuming unbounded + data sets and making them measurable in practice on large clustered and + non-clustered Bro deployments. + +.. contents:: + +Overview +======== + +The Sumstat processing flow is broken into three pieces. Observations, where +some aspect of an event is observed and fed into the Sumstats framework. +Reducers, where observations are collected and measured, typically by taking +some sort of summary statistic measurement like average or variance (among +others). Sumstats, where reducers have an epoch (time interval) that their +measurements are performed over along with callbacks for monitoring thresholds +or viewing the collected and measured data. + +Terminology +=========== + + Observation + + A single point of data. Observations have a few components of their + own. They are part of an arbitrarily named observation stream, they + have a key that is something the observation is about, and the actual + observation itself. + + Reducer + + Calculations are applied to an observation stream here to reduce the + full unbounded set of observations down to a smaller representation. + Results are collected within each reducer per-key so care must be + taken to keep the total number of keys tracked down to a reasonable + level. + + Sumstat + + The final definition of a Sumstat where one or more reducers is + collected over an interval, also known as an epoch. Thresholding can + be applied here along with a callback in the event that a threshold is + crossed. Additionally, a callback can be provided to access each + result (per-key) at the end of each epoch. + +Examples +======== + +These examples may seem very simple to an experienced Bro script developer and +they're intended to look that way. Keep in mind that these scripts will work +on small single process Bro instances as well as large many-worker clusters. +The complications from dealing with flow based load balancing can be ignored +by developers writing scripts that use Sumstats due to it's built in cluster +transparency. + +Printing the number of connections +---------------------------------- + +Sumstats provides a simple way of approaching the problem of trying to count +the number of connections over a given time interval. Here is a script with +inline documentation that does this with the Sumstats framework: + +.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-countconns.bro + +When run on a sample PCAP file from the Bro test suite, the following output +is created: + +.. btest:: sumstats-countconns + + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro + + +Toy Scan detection +------------------ + +Taking the previous example even further, we can implement a simple detection +to demonstrate the thresholding functionality. This example is a toy to +demonstate how thresholding works in Sumstats and is not meant to be a real- +world functional example, that is left to the scan.bro script that is included +with Bro. + +.. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro + +Let's see if there any hosts that crossed the threshold in a PCAP file +containing a host running nmap: + +.. btest:: sumstats-toy-scan + + @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro + +It seems the host running nmap was detected! + diff --git a/testing/btest/Traces/nmap-vsn.trace b/testing/btest/Traces/nmap-vsn.trace new file mode 100644 index 0000000000000000000000000000000000000000..b276ed3d2f19d5a8e941a38eda8f09d4081d58b0 GIT binary patch literal 33196 zcmbW=3w+J>AII_EnQfLf47poa*CX!1RRxwJsi&%tcZd2q^ikRD6 zc9Ba_$t6@SyHxHIl`#I_*Eu`ene{)~`}gQP9?oMuKcDaS_x<|$e$V-SzdM-nQY?ldh3R_*wpLrdr|6GT!;&+4;3VU&-pY@Psg=eLPTep0#g(WVx zJ#1dK)a91_@yL1^=6|+Ud)>PAU|-Af^7dJ0D^$p4v-uVA>0(XfXWm}U=jVsnMk`XFm7(ii!tA`2r2DAEr( zS&?@kmn*UeGAOpN)xxwdFK3%Odz+)5)fiz~KiqdzCVkA&7wo2|nalLGU+twVE7^y4 zT*NBBKxn4`WLHHNMeYo`tKdDk8nWfRf~Ly^Un2|1&XEfo1Ga-bq#L(Wv> z>&SM?g?dIKUsdD?-IibE#+Eqy3hJsv=T&&1V$lmJ$3f}v^M%v;9xes|* zk>4YI*9$rKBbO`kAo8pt4*Sk^d<27_!iP|R}pzok(H2z#tAutkn0p#1(~MEs>p!xLTQ7Mn-p0c`MV(mv64+q~XcI_JFJ%<9d&euFP%%AVI>A+IUhW#;v}I`NK+Smh@Pcc%%mqaq(b zzNW~g$i<3mhWt*E&5<`18HNmbN4U2wkda2_^t{o-{W_d-?|Gq_^t_ZmvU^_3)GW(B z@~xXIXj6Kg{b$cTZ?sabR+MXnd$}szJh>*ty(W|ymV>MhG2akN<(SGigh z_gt<6O1Z)**A2IFna{Z5?48}qH7Jd8O}(C7t~O~l%if|=&pEDgHSXZKT-7HF?bz1Q zj*5H``KFOMJ)qLfy@%4>Ym{<|8iiiVu2H)*Z_7R^bNrvK8tuBnonGP|J@9DJYHQr1 z`y~_oV%hBI52g87_9L~DhP$HIkMMlI_A9k+Pp!L85$^OO$T5oSh+L(}PRN6bd=%;P zu29d;$Ph(#K|ZF)uE=Ob+L4PC`51DCA|sGz6&Z;vF;!@lZpbEz?2dd=kv)*375O-F znId~4D@?Ol9nE8am-C5XIi4Plk%#%3fsx1h6Xv<`5YGE`Hxsdm3C1g5#2TOPA>t!7Z+pf$BHBxhEt{U@93Pd85&A@L z^0!pvlgQ^4`80B}BA-F7R^+qD9~IdLc|(!UA*;S8v`Sy(BZ};Yd|8ptBWEkJKXQ{I zUqGHzB7Aoh^(#1LC8o&4n{^R@Zax5}dk>ikwiX4wjSL6g_ z*n->{Q1)2EF%x#KlJRP0MjO6YR+@jmTRO)YsaeBTKi|{LgIBIDQ6+Q2l$<$=85wwc zF*+y}u(3H~4x;xm(R0Qdx6iN}MQ&ttu%t=$=)g6@UTHG)V!YA6PVRVvs6kjaYt6q%vO&ydB7 z3-$aQ*;tWZAp0nCHF92{P};S~WJRt+W+*Zqd3On+v>T9h6}b`FO_7_CBNdr|T&Tz` z$Q_E@iY!w~xVMSOmWoV5_Elsua-t$rkZTmV19?J`Um*+KBi!4a$eN1Wh3u@z-N;uJ zxd$1m$ZwEa6}cCgrpRxRg-Z*q@*T3SB7ZsIMV>@%QDiFeq$1OhzU72_n}MvU$X}6mMgE3-U6H>dkCYecc@}9^ z5ac;zup-YRJ1X)5a)=@?B4;Wx6S+;1myqd-`~&G%QD~LR$hwNWf{ak)U&v@h{*7Fw z$bXREDDoQe4@KTUmaZhUN*3}RMV3H*uE>(e{faDw%u?h%$fRn*9l8%0^1gS$KR0QF z?5oJe$oQE;&R)n2MNUMPm?h+#glwV6DabLig`CrnYZV!TJg&(1kVQTaN;@4HF-MTI zkrNd80dl<}=OK?Naz3)kheAESL~e-{fLt6W$kWKL6nO@@Wr>ip z3^GlTWs!xK3OUOo>nO4U@-ao;i;PxeWn}CpLOtsu_pSH7ZLOtqeVKEuWj9||JUn;F ze5&8Z`7MN5TV^h^x@<0ME!Sr9j*D0U<$VkOFU|YNl@$cJ1{qRCknbXUC~^ieUD>zk zg&eNzXDozVph$1zw~F*Z{;fz~WJP5sWMO0{MfxF!D)KJmhl(tMJf-a03_!l3?At7g zY^ChJtcFZhWDVpwMb<=?R`yWVLWU`_HnN{0Ly%3BosjjAeH2+AIZ2TXkR6rXmro#v zD)K30HD&kZ%g7Ci9EzN;>`IJ6?o{Nf$V^4PhAge@gnS*@UXjtrmlQbyIaiS*k(V0? z`!**d0~-l)D)Jd+-{u13aYe=<%PKn|KSs7vVwaDDqF_5Jg@^PFLi0Pl_y!{9BRbkUf;$t*4N+l~tn^ zkxwYH60)SSf-?virpPMDfr_k(oTm7(=E!b}3`4%D$QHFE_{x*c|)TN1n)8v+p|D(y>RI6BCFotw{|>`j{1$u5#tyAAaQ${cTy-~RBEhp15*&Vb--igxKIEW3Bt ztXZxa?F{i;qxY0@wWnNb6!{49uadcKWNLI<-?^Sjjjm_q^qn)dghQM z+o`B|`c5OWF7*HRfTS3rH?imRfOEElM+ZN<+ZDas0MF55+@j}ijizUbzKxSMovm@v zmTCwIlbmh_ddo6SA%%A4NtevNJMDkzJ6|n%1t!1SO{(d0df? zA-$B9sS(J-%8HXns4C1=&cELy(Utau{;3B8MXYV z==od&o>hPDhqj)=17ohAVOwa*!ftBWEb`1LPJ(&Ox44 zxN>ZT3DJ^BtYJ68y_s?6VoVjdxtc z`gVz@Sc~n%x@VaymMtOu9MXK>vxiu7EAWnsSj(4sik04&SU)7?#>!}lH1qlpVs#q9 zJ1%1R$L0RsKkN1r!onXP&kR29>FgPN%Zxde{qUX{o6(%={q&K1Gx%73=KA#EK@+{e ztJum=uBgR=EQ^d)WO-zYA}b&-DDqxp*pl4)|1T%XHMdM#UZIa=pD33xXV^yHXwwD> zoG5qh*PQ$BcB0(YFU^T^j6QRy)$Ukm{ zAm2rPs>m70(kq3WyOGTmxd+)>k>4OkD{?P#i6Xy6ex=Cokf#;-12W)Kp;Znb>nidv zGG>*K^GD=bMgD|5q{yF<9lsDt`wMcgB99|u6nO#}ugH_gql!#LW+^fa8MIosw;9Ov zHG=#V>G!1|e?!((cMUiKaGZlFjxkZuZkiRJMJko2e&?*;@!HT?y?4Za@ zWbN9*Z%Vy{j8fzu$c-UF&dbQ%bp&|@d0LTwAp`0OIsZmBROCO%#}#=E8Lh}0$oY!Q zLe|?8V0ANd5?R9cwbcW0H9Do2kMB+w{wul%+mfN}V<#>2RS4WO*d2)ol zcqlM2H^+bIT#HY7Ok#e!lBq~S-Xwz$1SYv9xpU{5_p8Sw=5LQlvN3OxzCQ#eyC=D0 v=bC-aW0IbZB=LEZ4E{ba#VyI5JJ*5>9+UKPBw3d?$)N87cep3<*tz}(qO1kB literal 0 HcmV?d00001 From 9ed5f8bae8d7d86e105d8273366ab5fc99711117 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 6 Nov 2013 14:56:32 -0600 Subject: [PATCH 08/23] Update tests and baselines for sumstats docs --- .../output | 40 +++++++++++++++ .../output | 49 +++++++++++++++++++ .../btest-doc.sphinx.sumstats-countconns#1 | 9 ++++ .../btest-doc.sphinx.sumstats-toy-scan#1 | 9 ++++ ...c_frameworks_sumstats-countconns_bro.btest | 40 +++++++++++++++ ...doc_frameworks_sumstats-toy-scan_bro.btest | 49 +++++++++++++++++++ .../doc/sphinx/sumstats-countconns.btest | 1 + .../btest/doc/sphinx/sumstats-toy-scan.btest | 1 + 8 files changed, 198 insertions(+) create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output create mode 100644 testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output create mode 100644 testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 create mode 100644 testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest create mode 100644 testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest create mode 100644 testing/btest/doc/sphinx/sumstats-countconns.btest create mode 100644 testing/btest/doc/sphinx/sumstats-toy-scan.btest diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output new file mode 100644 index 0000000000..0ec0c9ce70 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-countconns_bro/output @@ -0,0 +1,40 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-countconns.bro + +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } \ No newline at end of file diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output new file mode 100644 index 0000000000..31b84f2cb8 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output @@ -0,0 +1,49 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-toy-scan.bro + +@load base/frameworks/sumstats + +# We use the connection_attempted event limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } \ No newline at end of file diff --git a/testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 b/testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 new file mode 100644 index 0000000000..b7126957ce --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.sumstats-countconns/btest-doc.sphinx.sumstats-countconns#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r workshop_2011_browse.trace sumstats-countconns.bro + Number of connections established: 6 + diff --git a/testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 b/testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 new file mode 100644 index 0000000000..f5db95c5f3 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.sumstats-toy-scan/btest-doc.sphinx.sumstats-toy-scan#1 @@ -0,0 +1,9 @@ +.. rst-class:: btest-cmd + + .. code-block:: none + :linenos: + :emphasize-lines: 1,1 + + # bro -r nmap-vsn.trace sumstats-toy-scan.bro + 192.168.1.71 attempted 5 or more connections + diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest new file mode 100644 index 0000000000..0ec0c9ce70 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-countconns_bro.btest @@ -0,0 +1,40 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-countconns.bro + +@load base/frameworks/sumstats + +event connection_established(c: connection) + { + # Make an observation! + # This observation is global so the key is empty. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn established", + SumStats::Key(), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn established" observation stream + # and uses the summing calculation on the observations. + local r1 = SumStats::Reducer($stream="conn established", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # We give it an arbitrary name and make it collect data every minute. + # The reducer is then attached and a $epoch_result callback is given + # to finally do something with the data collected. + SumStats::create([$name = "counting connections", + $epoch = 1min, + $reducers = set(r1), + $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) = + { + # This is the body of the callback that is called when a single + # result has been collected. We are just printing the total number + # of connections that were seen. The $sum field is provided as a + # double type value so we need to use %f as the format specifier. + print fmt("Number of connections established: %.0f", result["conn established"]$sum); + }]); + } \ No newline at end of file diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest new file mode 100644 index 0000000000..31b84f2cb8 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest @@ -0,0 +1,49 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +sumstats-toy-scan.bro + +@load base/frameworks/sumstats + +# We use the connection_attempted event limit our observations to those +# which were attempted and not successful. +event connection_attempt(c: connection) + { + # Make an observation! + # This observation is about the host attempting the connection. + # Each established connection counts as one so the observation is always 1. + SumStats::observe("conn attempted", + SumStats::Key($host=c$id$orig_h), + SumStats::Observation($num=1)); + } + +event bro_init() + { + # Create the reducer. + # The reducer attaches to the "conn attempted" observation stream + # and uses the summing calculation on the observations. Keep + # in mind that there will be one result per key (connection originator). + local r1 = SumStats::Reducer($stream="conn attempted", + $apply=set(SumStats::SUM)); + + # Create the final sumstat. + # This is slightly different from the last example since we're providing + # a callback to calculate a value to check against the threshold with + # $threshold_val. The actual threshold itself is provided with $threshold. + # Another callback is + SumStats::create([$name = "finding scanners", + $epoch = 5min, + $reducers = set(r1), + # Provide a threshold. + $threshold = 5.0, + # Provide a callback to calculate a value from the result + # to check against the threshold field. + $threshold_val(key: SumStats::Key, result: SumStats::Result) = + { + return result["conn attempted"]$sum; + }, + # Provide a callback for when a key crosses the threshold. + $threshold_crossed(key: SumStats::Key, result: SumStats::Result) = + { + print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); + }]); + } \ No newline at end of file diff --git a/testing/btest/doc/sphinx/sumstats-countconns.btest b/testing/btest/doc/sphinx/sumstats-countconns.btest new file mode 100644 index 0000000000..fd375af5fb --- /dev/null +++ b/testing/btest/doc/sphinx/sumstats-countconns.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro diff --git a/testing/btest/doc/sphinx/sumstats-toy-scan.btest b/testing/btest/doc/sphinx/sumstats-toy-scan.btest new file mode 100644 index 0000000000..8756f1cfc0 --- /dev/null +++ b/testing/btest/doc/sphinx/sumstats-toy-scan.btest @@ -0,0 +1 @@ +@TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/nmap-vsn.trace ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro From 85d8653bce626f566791b051d73ac8daa5e7bee0 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 6 Nov 2013 15:58:24 -0600 Subject: [PATCH 09/23] Update docs and tests for a recent change to detect-MHR.bro --- doc/scripting/index.rst | 4 ++-- .../output | 7 ++++++- .../output | 5 +++++ .../output | 2 +- ...de-scripts_policy_frameworks_files_detect-MHR_bro.btest | 7 ++++++- ...-scripts_policy_frameworks_files_detect-MHR_bro@3.btest | 5 +++++ ...-scripts_policy_frameworks_files_detect-MHR_bro@4.btest | 2 +- 7 files changed, 26 insertions(+), 6 deletions(-) diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index c6b479a7af..16ca6b6d58 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -66,7 +66,7 @@ are ensuring the Files framework, the Notice framework and the script to hash al been loaded by Bro. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 10-31 + :lines: 10-36 The export section redefines an enumerable constant that describes the type of notice we will generate with the Notice framework. Bro @@ -87,7 +87,7 @@ Up until this point, the script has merely done some basic setup. With the next the script starts to define instructions to take in a given event. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 33-57 + :lines: 38-62 The workhorse of the script is contained in the event handler for ``file_hash``. The :bro:see:`file_hash` event allows scripts to access diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output index 9f05c43669..aa4509513f 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro/output @@ -27,6 +27,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold @@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output index 92f077de7d..30c6b1040e 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output @@ -18,6 +18,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output index 57d2d7ee50..64ef286c39 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output @@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest index 9f05c43669..aa4509513f 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro.btest @@ -27,6 +27,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold @@ -52,7 +57,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest index 92f077de7d..30c6b1040e 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest @@ -18,6 +18,11 @@ export { /application\/jar/ | /video\/mp4/ &redef; + ## The Match notice has a sub message with a URL where you can get more + ## information about the file. The %s will be replaced with the SHA-1 + ## hash of the file. + const match_sub_url = "https://www.virustotal.com/en/search/?query=%s" &redef; + ## The malware hash registry runs each malware sample through several ## A/V engines. Team Cymru returns a percentage to indicate how ## many A/V engines flagged the sample as malicious. This threshold diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest index 57d2d7ee50..64ef286c39 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest @@ -20,7 +20,7 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( mhr_detect_rate >= notice_threshold ) { local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); - local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + local virustotal_url = fmt(match_sub_url, hash); NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); } } From 410e4babd045035503696e29639a1b8875e4e04e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 6 Nov 2013 18:56:39 -0600 Subject: [PATCH 10/23] Fix typos in sumstats doc and update doc test --- doc/frameworks/sumstats-toy-scan.bro | 6 +++--- doc/frameworks/sumstats.rst | 18 +++++++++--------- .../output | 6 +++--- ...-doc_frameworks_sumstats-toy-scan_bro.btest | 6 +++--- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/frameworks/sumstats-toy-scan.bro b/doc/frameworks/sumstats-toy-scan.bro index c435fb8997..8e035142d7 100644 --- a/doc/frameworks/sumstats-toy-scan.bro +++ b/doc/frameworks/sumstats-toy-scan.bro @@ -1,6 +1,6 @@ @load base/frameworks/sumstats -# We use the connection_attempted event limit our observations to those +# We use the connection_attempt event to limit our observations to those # which were attempted and not successful. event connection_attempt(c: connection) { @@ -25,7 +25,7 @@ event bro_init() # This is slightly different from the last example since we're providing # a callback to calculate a value to check against the threshold with # $threshold_val. The actual threshold itself is provided with $threshold. - # Another callback is + # Another callback is provided for when a key crosses the threshold. SumStats::create([$name = "finding scanners", $epoch = 5min, $reducers = set(r1), @@ -42,4 +42,4 @@ event bro_init() { print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); }]); - } \ No newline at end of file + } diff --git a/doc/frameworks/sumstats.rst b/doc/frameworks/sumstats.rst index e06ceaf2c8..057cc87fc1 100644 --- a/doc/frameworks/sumstats.rst +++ b/doc/frameworks/sumstats.rst @@ -5,9 +5,9 @@ Summary Statistics .. rst-class:: opening Measuring aspects of network traffic is an extremely common task in Bro. - Bro provides data structures which make this very easy as wellin - simplistic cases such as size limited trace file processing. In real- - world deployments though, there are difficulties that arise from + Bro provides data structures which make this very easy as well in + simplistic cases such as size limited trace file processing. In + real-world deployments though, there are difficulties that arise from clusterization (many processes sniffing traffic) and unbounded data sets (traffic never stops). The Summary Statistics (otherwise referred to as SumStats) framework aims to define a mechanism for consuming unbounded @@ -60,7 +60,7 @@ These examples may seem very simple to an experienced Bro script developer and they're intended to look that way. Keep in mind that these scripts will work on small single process Bro instances as well as large many-worker clusters. The complications from dealing with flow based load balancing can be ignored -by developers writing scripts that use Sumstats due to it's built in cluster +by developers writing scripts that use Sumstats due to its built-in cluster transparency. Printing the number of connections @@ -80,18 +80,18 @@ is created: @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/workshop_2011_browse.trace ${DOC_ROOT}/frameworks/sumstats-countconns.bro -Toy Scan detection +Toy scan detection ------------------ Taking the previous example even further, we can implement a simple detection to demonstrate the thresholding functionality. This example is a toy to -demonstate how thresholding works in Sumstats and is not meant to be a real- -world functional example, that is left to the scan.bro script that is included -with Bro. +demonstrate how thresholding works in Sumstats and is not meant to be a +real-world functional example, that is left to the +:doc:`/scripts/policy/misc/scan` script that is included with Bro. .. btest-include:: ${DOC_ROOT}/frameworks/sumstats-toy-scan.bro -Let's see if there any hosts that crossed the threshold in a PCAP file +Let's see if there are any hosts that crossed the threshold in a PCAP file containing a host running nmap: .. btest:: sumstats-toy-scan diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output index 31b84f2cb8..b1b46b3b39 100644 --- a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_sumstats-toy-scan_bro/output @@ -4,7 +4,7 @@ sumstats-toy-scan.bro @load base/frameworks/sumstats -# We use the connection_attempted event limit our observations to those +# We use the connection_attempt event to limit our observations to those # which were attempted and not successful. event connection_attempt(c: connection) { @@ -29,7 +29,7 @@ event bro_init() # This is slightly different from the last example since we're providing # a callback to calculate a value to check against the threshold with # $threshold_val. The actual threshold itself is provided with $threshold. - # Another callback is + # Another callback is provided for when a key crosses the threshold. SumStats::create([$name = "finding scanners", $epoch = 5min, $reducers = set(r1), @@ -46,4 +46,4 @@ event bro_init() { print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); }]); - } \ No newline at end of file + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest index 31b84f2cb8..b1b46b3b39 100644 --- a/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest +++ b/testing/btest/doc/sphinx/include-doc_frameworks_sumstats-toy-scan_bro.btest @@ -4,7 +4,7 @@ sumstats-toy-scan.bro @load base/frameworks/sumstats -# We use the connection_attempted event limit our observations to those +# We use the connection_attempt event to limit our observations to those # which were attempted and not successful. event connection_attempt(c: connection) { @@ -29,7 +29,7 @@ event bro_init() # This is slightly different from the last example since we're providing # a callback to calculate a value to check against the threshold with # $threshold_val. The actual threshold itself is provided with $threshold. - # Another callback is + # Another callback is provided for when a key crosses the threshold. SumStats::create([$name = "finding scanners", $epoch = 5min, $reducers = set(r1), @@ -46,4 +46,4 @@ event bro_init() { print fmt("%s attempted %.0f or more connections", key$host, result["conn attempted"]$sum); }]); - } \ No newline at end of file + } From da9c31db55c414853771a68e194f312eb1f9de52 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 00:36:46 -0800 Subject: [PATCH 11/23] Fixing warnings during doc build. Includes adding a stub file that has been missing for a while already. --- CHANGES | 237 +++++++++--------- VERSION | 2 +- doc/scripts/notices.rst | 8 + .../base/frameworks/packet-filter/cluster.bro | 3 + 4 files changed, 133 insertions(+), 117 deletions(-) create mode 100644 doc/scripts/notices.rst diff --git a/CHANGES b/CHANGES index b2028b68d6..60efc5fe36 100644 --- a/CHANGES +++ b/CHANGES @@ -1,20 +1,25 @@ +2.2-beta-199 | 2013-11-07 00:36:46 -0800 + + * Fixing warnings during doc build. (Robin Sommer) + 2.2-beta-198 | 2013-11-06 22:54:30 -0800 * Update docs and tests for a recent change to detect-MHR.bro (Daniel Thayer) * Update tests and baselines for sumstats docs. (Daniel Thayer) - + 2.2-beta-194 | 2013-11-06 14:39:50 -0500 - * Remove resp_size from the ssh log. Refactor when we write out to the log a bit. Geodata now works reliably. (Vlad Grigorescu) + * Remove resp_size from the ssh log. Refactor when we write out to + the log a bit. Geodata now works reliably. (Vlad Grigorescu) - * Update VirusTotal URL to work with changes to their website and changed it to a redef. (Vlad Grigorescu) + * Update VirusTotal URL to work with changes to their website and + changed it to a redef. (Vlad Grigorescu) * Added a document for the SumStats framework. (Seth Hall) - 2.2-beta-184 | 2013-11-03 22:53:42 -0800 * Remove swig-ruby from required packages section of install doc. @@ -47,7 +52,7 @@ (Vlad Grigorescu) * New version of the threading queue deadlock fix. (Robin Sommer) - + * Updating README with download/git information. (Robin Sommer) 2.2-beta-161 | 2013-10-25 15:48:15 -0700 @@ -76,7 +81,7 @@ 2.2-beta-152 | 2013-10-24 18:16:49 -0700 * Fix for input readers occasionally dead-locking. (Robin Sommer) - + 2.2-beta-151 | 2013-10-24 16:52:26 -0700 * Updating submodule(s). @@ -96,7 +101,7 @@ (Daniel Thayer) * Intel framework notes added to NEWS. (Seth Hall) - + * Temporary OSX Mavericks libc++ issue workaround for getline() problem in ASCII reader. (Bernhard Amann) @@ -112,16 +117,16 @@ 2.2-beta-133 | 2013-10-23 09:50:16 -0700 * Fix record coercion tolerance of optional fields. (Jon Siwek) - + * Add NEWS about incompatible local.bro changes, addresses BIT-1047. (Jon Siwek) * Fix minor formatting problem in NEWS. (Jon Siwek) - + 2.2-beta-129 | 2013-10-23 09:47:29 -0700 * Another batch of documentation fixes and updates. (Daniel Thayer) - + 2.2-beta-114 | 2013-10-18 14:17:57 -0700 * Moving the SQLite examples into separate Bro files to turn them @@ -130,7 +135,7 @@ 2.2-beta-112 | 2013-10-18 13:47:13 -0700 * A larger chunk of documentation fixes and cleanup. (Daniel Thayer) - + Apart from many smaller improves this includes in particular: * Add README files for most Bro frameworks and base/protocols. @@ -164,7 +169,7 @@ 2.2-beta-68 | 2013-10-14 09:26:09 -0700 * Add check for curl command to active-http.test. (Daniel Thayer) - + 2.2-beta-64 | 2013-10-14 09:20:04 -0700 * Review usage of Reporter::InternalError, addresses BIT-1045. @@ -172,7 +177,7 @@ Replaced some with InternalWarning or AnalyzerError, the later being a new method which signals the analyzer to not process further input. (Jon Siwek) - + * Add new event for TCP content file write failures: "contents_file_write_failure". (Jon Siwek) @@ -183,9 +188,9 @@ 2.2-beta-55 | 2013-10-10 13:36:38 -0700 * A couple of new TLS extension numbers. (Bernhard Amann) - + * Suport for three more new TLS ciphers. (Bernhard Amann) - + * Removing ICSI notary from default site config. (Robin Sommer) 2.2-beta-51 | 2013-10-07 17:33:56 -0700 @@ -194,9 +199,9 @@ (Robin Sommer) * Fixing the historical CHANGES record. (Robin Sommer) - + * Updating copyright notice. (Robin Sommer) - + 2.2-beta-38 | 2013-10-02 11:03:29 -0700 * Fix uninitialized (or unused) fields. (Jon Siwek) @@ -206,31 +211,31 @@ * Remove dead/unfinished code in unary not expression. (Jon Siwek) * Fix logic for failed DNS TXT lookups. (Jon Siwek) - + * A couple null ptr checks. (Jon Siwek) - + * Improve return value checking and error handling. (Jon Siwek) - + * Remove unused variable assignments. (Jon Siwek) - + * Prevent division/modulo by zero in scripts. (Jon Siwek) - + * Fix unintentional always-false condition. (Jon Siwek) - + * Fix invalidated iterator usage. (Jon Siwek) * Fix DNS_Mgr iterator mismatch. (Jon Siwek) - + * Set safe umask when creating script profiler tmp files. (Jon Siwek) * Fix nesting/indent level whitespace mismatch. (Jon Siwek) - + * Add checks to avoid improper negative values use. (Jon Siwek) - + 2.2-beta-18 | 2013-10-02 10:28:17 -0700 * Add support for further TLS cipher suites. (Bernhard Amann) - + 2.2-beta-13 | 2013-10-01 11:31:55 -0700 * Updating bifcl usage message. (Robin Sommer) @@ -244,7 +249,7 @@ 2.2-beta-4 | 2013-09-24 13:23:30 -0700 * Fix for setting REPO in Makefile. (Robin Sommer) - + * Whitespace fix. (Robin Sommer) * Removing :doc: roles so that we can render this with docutils @@ -261,9 +266,9 @@ * Updating NEWS. (Robin Sommer) * Fixing an always false condition. (Robin Sommer) - + * Fix required for compiling with clang 3.3. (Robin Sommer) - + 2.1-1377 | 2013-09-20 14:38:15 -0700 * Updates to the scripting introduction. (Scott Runnels) @@ -278,7 +283,7 @@ 2.1-1364 | 2013-09-19 15:12:08 -0700 * Add links to Intelligence Framework documentation. (Daniel Thayer) - + * Update Mozilla root CA list. (Bernhard Amann, Jon Siwek) * Update documentation of required packages. (Daniel Thayer) @@ -298,27 +303,27 @@ 2.1-1352 | 2013-09-18 14:42:28 -0700 * Fix a number of compiler warnings. (Daniel Thayer) - + * Fix cmake warning about ENABLE_PERFTOOLS not being used. (Daniel Thayer) 2.1-1344 | 2013-09-16 16:20:55 -0500 * Refactor Analyzer::AddChildAnalyzer and usages. (Jon Siwek) - + * Minor refactor to SSL BinPAC grammer. (Jon Siwek) * Minor refactor to Broxygen enum comments. (Jon Siwek) - + * Fix possible (unlikely) use of uninitialized value. (Jon Siwek) - + * Fix/improve dereference-before-null-checks. (Jon Siwek) - + * Fix out-of-bounds memory accesses, and remove a variable-length-array usage. (Jon Siwek) * Fix potential mem leak. (Jon Siwek) - + * Fix double-free and deallocator mismatch. (Jon Siwek) * Fix another function val reference counting bug. (Jon Siwek) @@ -349,7 +354,7 @@ * Reorganized and signifcantly extended documentation. This includes two new chapters contributed by Scott Runnels. - + 2.1-1216 | 2013-08-31 10:39:40 -0700 @@ -367,25 +372,25 @@ 2.1-1154 | 2013-08-30 08:27:45 -0700 * Fix global opaque val segfault. Addresses BIT-1071. (Jon Siwek) - + * Fix malloc/delete mismatch. (Jon Siwek) - + * Fix invalid pointer dereference in AsciiFormatter. (Jon Siwek) 2.1-1150 | 2013-08-29 13:43:01 -0700 * Fix input framework memory leaks. (Jon Siwek) - + * Fix memory leak in SOCKS analyzer for bad addr types. (Jon Siwek) - + * Fix Bloom filter memory leaks. (Jon Siwek) 2.1-1144 | 2013-08-28 18:51:06 -0700 * Add bits_per_uid unit test. Addresses BIT-1016. (Jon Siwek) - + * UID optimizations. Addresses BIT-1016. (Jon Siwek) - + * Added a $unique_max field to Reducers for the SumStats::UNIQUE calculation, and using the new option in scan.bro and the FTP bruteforce detection. (Seth Hall) @@ -398,11 +403,11 @@ 2.1-1135 | 2013-08-27 12:16:26 -0700 * More SumStats fixes. (Seth Hall) - + * Increase UIDs to 96 bits. (Jon Siwek) - + - The bit-length is adjustable via redef'ing bits_per_uid. - + - Prefix 'C' is added to connection UIDS (including IP tunnels) and 'F' to files. @@ -411,9 +416,9 @@ 2.1-1128 | 2013-08-24 10:27:29 -0700 * Remove code relict in input framework. (Jon Siwek) - + * Fix documentation for mkdir BIF. (Jon Siwek) - + * File extraction tweaks. (Jon Siwek) - Default extraction limit of 100MB now provided via a tuning @@ -427,11 +432,11 @@ 2.1-1124 | 2013-08-23 16:33:52 -0700 * Fixed a number of object bugs DNP3 analyzer. (Hui Lin) - + 2.1-1122 | 2013-08-22 16:52:27 -0700 * Use macros to create file analyzer plugin classes. (Jon Siwek) - + * Add options to limit extracted file sizes w/ 100MB default. (Jon Siwek) @@ -441,13 +446,13 @@ improvements. (Jon Siwek) * Make memory leak tests able to time out. (Jon Siwek) - + * Fix a compiler warning regarding strncat misuse. (Jon Siwek) - + 2.1-1103 | 2013-08-21 19:11:34 -0400 * A number of sumstats fixes. (Seth Hall, Vlad Grigorescu) - + * Fix memory leak w/ when statements. Addresses BIT-1058. (Jon Siwek) @@ -478,12 +483,12 @@ turning them into events. (Seth Hall) * Fixing intel framework tests. (Seth Hall) - + 2.1-1059 | 2013-08-13 23:52:41 -0400 * Add file name support to intel framework. (Seth Hall) - * Add file support to intel framework and slightly restructure + * Add file support to intel framework and slightly restructure intel http handling. (Seth Hall) 2.1-1052 | 2013-08-12 14:38:14 -0700 @@ -505,9 +510,9 @@ 2.1-1039 | 2013-08-09 15:30:15 -0700 * Fix mem leak in DHCP analyzer. (Jon Siwek) - + * Fix a unit test outdated by recent sumstats changes. (Jon Siwek) - + 2.1-1036 | 2013-08-05 17:29:11 -0400 * Fix the SSL infinite loop I just created. (Seth Hall) @@ -562,7 +567,7 @@ 2.1-1009 | 2013-08-02 17:19:08 -0700 * A number of exec module and raw input reader fixes. (Jon Siwek) - + 2.1-1007 | 2013-08-01 15:41:54 -0700 * More function documentation. (Bernhard Amann) @@ -634,11 +639,11 @@ compressed log representation. (Seth Hall) * Added mime types to http.log (Seth Hall) - + * Add jar files to the default MHR lookups. (Seth Hall) - + * Adding CAB files for MHR checking. (Seth Hall) - + * Improve malware hash registry script. - Include a link to a virustotal search in the notice sub message field. @@ -671,15 +676,15 @@ * Updates for the Intel Framework. (Seth Hall) - - policy/frameworks/intel/seen is the new location for the + - policy/frameworks/intel/seen is the new location for the scripts that push data into the intel framework for checking. - - The new policy/frameworks/intel/do_notice script adds an + - The new policy/frameworks/intel/do_notice script adds an example mechanism for data driven notices. - - Remove the Intel insertion after heuristically detecting SSH + - Remove the Intel insertion after heuristically detecting SSH bruteforcing. - + - Intel importing format has changed (refer to docs). - All string matching is now case insensitive. @@ -740,7 +745,7 @@ make it deterministic. (Robin Sommer) * Small raw reader tweaks that got left our earlier. (Robin Sommer) - + 2.1-814 | 2013-07-15 18:18:20 -0700 * Fixing raw reader crash when accessing nonexistant file, and @@ -866,12 +871,12 @@ input data on to the file analysis framework. (Jon Siwek) * File analysis framework interface simplifications. (Jon Siwek) - + - Remove script-layer data input interface (will be managed directly by input framework later). - Only track files internally by file id hash. Chance of collision - too small to justify also tracking unique file string. + too small to justify also tracking unique file string. 2.1-741 | 2013-06-07 17:28:50 -0700 @@ -922,14 +927,14 @@ 2.1-659 | 2013-05-24 17:24:18 -0700 * Fix broken/missing documentation. (Jon Siwek) - + * Fixing test that would fail without ES/curl support. (Robin Sommer) 2.1-656 | 2013-05-17 15:58:07 -0700 * Fix mutex lock problem for writers. (Bernhard Amann) - + 2.1-654 | 2013-05-17 13:49:52 -0700 * Tweaks to sqlite3 configuration to address threading issues. @@ -947,9 +952,9 @@ 2.1-647 | 2013-05-17 07:47:14 -0700 * Fixing Broxygen generation to have BROMAGIC set. (Robin Sommer) - + * Fix for 'fchmod undeclared here' on FreeBSD. (Robin Sommer) - + * CMake policy fix to avoid errors with older versions. (Robin Sommer) @@ -1062,7 +1067,7 @@ 2.1-386 | 2013-03-22 12:41:50 -0700 * Added reverse() function to strings.bif. (Yun Zheng Hu) - + 2.1-384 | 2013-03-22 12:10:14 -0700 * Fix record constructors in table initializer indices. Addresses @@ -1071,16 +1076,16 @@ 2.1-382 | 2013-03-22 12:01:34 -0700 * Add support for 802.1ah (Q-in-Q). Addresses #641. (Seth Hall) - + 2.1-380 | 2013-03-18 12:18:10 -0700 * Fix gcc compile warnings in base64 encoder and benchmark reader. (Bernhard Amann) - + 2.1-377 | 2013-03-17 17:36:09 -0700 * Fixing potential leak in DNS error case. (Vlad Grigorescu) - + 2.1-375 | 2013-03-17 13:14:26 -0700 * Add base64 encoding functionality, including new BiFs @@ -1092,14 +1097,14 @@ * Adding a test for extract-certs-pem.pem. (Robin Sommer) * Renaming Base64Decoder to Base64Converter. (Robin Sommer) - + 2.1-366 | 2013-03-17 12:35:59 -0700 * Correctly handle DNS lookups for software version ranges. (Seth Hall) * Improvements to vulnerable software detection. (Seth Hall) - + - Add a DNS based updating method. This needs to be tested still. @@ -1133,9 +1138,9 @@ 2.1-351 | 2013-03-07 13:27:29 -0800 * Fix new/delete mismatch. Addresses #958. (Jacob Baines) - + * Fix compiler warnings. (Jon Siwek) - + 2.1-347 | 2013-03-06 16:48:44 -0800 * Remove unused parameter from vector assignment method. (Bernhard Amann) @@ -1184,9 +1189,9 @@ 2.1-328 | 2013-02-05 01:34:29 -0500 - * New script to query the ICSI Certificate Notary + * New script to query the ICSI Certificate Notary (http://notary.icsi.berkeley.edu/) over DNS and add information - to the SSL log at runtime. (Matthias Vallentin) + to the SSL log at runtime. (Matthias Vallentin) * Add delayed logging to SSL base scripts. (Matthias Vallentin) @@ -1237,7 +1242,7 @@ * Changing test=suite's btest call to use "-j" instead of "-j 5". (Robin Sommer) - + * Require "case" blocks to end with either "break", "return", or a new "fallthrough" statement that passes control on to the subsequent case. This gives us the best mix of safety, @@ -1253,7 +1258,7 @@ ElasticSearch writer. (Gilbert Clark) * Removing unused class member. (Robin Sommer) - + * Add opaque type-ignoring for the accept_unsupported_types input framework option. (Bernhard Amann) @@ -1298,7 +1303,7 @@ sha256_*, and entropy_*, respectively. Note that these functions have changed their signatures to work with opaques types rather than global state as it was before. - + 2.1-240 | 2012-12-20 15:21:07 -0800 * Improve error for invalid use of types as values. Addresses #923. @@ -1423,7 +1428,7 @@ 2.1-195 | 2012-12-03 14:50:33 -0800 * Catching out-of-memory in patricia tree code. (Bill Parker) - + 2.1-194 | 2012-12-03 14:36:26 -0800 * Renaming ASCII writer filter option 'only_single_header_row' to @@ -1484,7 +1489,7 @@ Hall) * Adding NEWS placeholder for hooks and CSV mode. (Robin Sommer) - + 2.1-178 | 2012-11-23 19:35:32 -0800 * The ASCII writer now supports a new filter config option @@ -1539,7 +1544,7 @@ 2.1-112 | 2012-11-05 13:58:20 -0800 - * New base script for detecting cases of checksum offloading. + * New base script for detecting cases of checksum offloading. Reporter messages will now tell if one has bad checksums. (Seth Hall) @@ -1549,9 +1554,9 @@ 2.1-109 | 2012-11-05 13:39:34 -0800 * Add detection rate threshold for MHR. (Vlad Grigorescu) - + * lookup_hostname_txt fixes. (Vlad Grigorescu) - + 2.1-104 | 2012-11-01 10:37:50 -0700 * A new built-in function lookup_hostname_txt() provides support for @@ -1676,7 +1681,7 @@ Addresses #877. (Jon Siwek) * Add --with-curl option to ./configure. Addresses #877. (Jon Siwek) - + 2.1-61 | 2012-10-12 09:32:48 -0700 * Fix bug in the input framework: the config table did not work. @@ -1719,7 +1724,7 @@ * Remove deprecated script functionality (see NEWS for details). (Daniel Thayer) - + 2.1-39 | 2012-09-29 14:09:16 -0700 * Reliability adjustments to istate tests with network @@ -1731,7 +1736,7 @@ an error. (Daniel Thayer) * Fix parsing of large integers on 32-bit systems. (Daniel Thayer) - + * Serialize language.when unit test with the "comm" group. (Jon Siwek) @@ -1742,7 +1747,7 @@ 2.1-26 | 2012-09-23 08:46:03 -0700 * Add an item to FAQ page about broctl options. (Daniel Thayer) - + * Add more language tests. We now have tests of all built-in Bro data types (including different representations of constant values, and max./min. values), keywords, and operators (including @@ -1765,7 +1770,7 @@ * Adjusting some unit tests that do cluster communication. (Jon Siwek) * Small change to non-blocking DNS initialization. (Jon Siwek) - + * Reorder a few statements in scan.l to make 1.5msecs etc work. Adresses #872. (Bernhard Amann) @@ -1797,9 +1802,9 @@ Siwek) * Parse 64-bit consts in Bro scripts correctly. (Bernhard Amann) - + * Output 64-bit counts correctly on 32-bit machines (Bernhard Amann) - + * Input framework fixes, including: (Bernhard Amann) - One of the change events got the wrong parameters. @@ -1813,7 +1818,7 @@ - Hashing of lines just containing zero-length-strings was broken. - Make set_separators different from , work for input framework. - + - Input framework was not handling counts and ints out of 32-bit-range correctly. @@ -1821,20 +1826,20 @@ the line, log it, and continue. * Update documentation for builtin types. (Daniel Thayer) - + - Add missing description of interval "msec" unit. - + - Improved description of pattern by clarifying the issue of operand order and difference between exact and embedded matching. * Documentation fixes for signature 'eval' conditions. (Jon Siwek) - + * Remove orphaned 1.5 unit tests. (Jon Siwek) * Add type checking for signature 'eval' condition functions. (Jon Siwek) - + * Adding an identifier to the SMTP blocklist notices for duplicate suppression. (Seth Hall) @@ -1865,7 +1870,7 @@ 2.1-beta-31 | 2012-08-21 15:46:05 -0700 * Tweak to rotate-custom.bro unit test. (Jon Siwek) - + * Ignore small mem leak every rotation interval for dataseries logs. (Jon Siwek) @@ -1920,13 +1925,13 @@ 2.1-beta-6 | 2012-08-10 12:22:52 -0700 * Fix bug in input framework with an edge case. (Bernhard Amann) - + * Fix small bug in input framework test script. (Bernhard Amann) - + 2.1-beta-3 | 2012-08-03 10:46:49 -0700 * Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer) - + * Fix configure script to exit with non-zero status on error (Jon Siwek) @@ -1977,7 +1982,7 @@ * Input framework: Make want_record=T the default for events (Bernhard Amann) - + * Changing the start/end markers in logs to open/close now reflecting wall clock. (Robin Sommer) @@ -1988,16 +1993,16 @@ * New test for input framework that fails to find a file. (Robin Sommer) - + * Improving error handling for threads. (Robin Sommer) - + * Tweaking the custom-rotate test to produce stable output. (Robin Sommer) 2.0-884 | 2012-07-26 14:33:21 -0700 * Add comprehensive error handling for close() calls. (Jon Siwek) - + * Add more test cases for input framework. (Bernhard Amann) * Input framework: make error output for non-matching event types @@ -2006,14 +2011,14 @@ 2.0-877 | 2012-07-25 17:20:34 -0700 * Fix double close() in FilerSerializer class. (Jon Siwek) - + * Fix build warnings. (Daniel Thayer) * Fixes to ElasticSearch plugin to make libcurl handle http responses correctly. (Seth Hall) * Fixing FreeBSD compiler error. (Robin Sommer) - + * Silencing compiler warnings. (Robin Sommer) 2.0-871 | 2012-07-25 13:08:00 -0700 @@ -2032,7 +2037,7 @@ 2.0-866 | 2012-07-24 16:02:07 -0700 * Correct a typo in usage message. (Daniel Thayer) - + * Fix file permissions of log files (which were created with execute permissions after a recent change). (Daniel Thayer) @@ -4704,7 +4709,7 @@ away. (Robin Sommer) - Smarter way to increase the communication module's pipe's socket - buffer size, resulting in a value closer to the allowed maximum. + buffer size, resulting in a value closer to the allowed maximum. (Craig Leres) - BroControl now also maintains links from the log archive to the @@ -4747,7 +4752,7 @@ - http-header.bro now includes a global "include_header: set[string]" If it contains any strings, then only those headers will be processed. If left empty, then you continue to get the current behavior of processing all - headers. (Robin Sommer). + headers. (Robin Sommer). - Several changes to drop.bro (Robin Sommer): diff --git a/VERSION b/VERSION index 4498114245..77286a3a7b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-beta-198 +2.2-beta-199 diff --git a/doc/scripts/notices.rst b/doc/scripts/notices.rst new file mode 100644 index 0000000000..412cd58509 --- /dev/null +++ b/doc/scripts/notices.rst @@ -0,0 +1,8 @@ + +.. Not nice but I don't find a way to link to the notice index +.. directly from the upper level TOC tree. + +Notices +======= + +See the `Bro Notice Index <../bro-noticeindex.html>`_. diff --git a/scripts/base/frameworks/packet-filter/cluster.bro b/scripts/base/frameworks/packet-filter/cluster.bro index 34f0600d18..6e41a6045f 100644 --- a/scripts/base/frameworks/packet-filter/cluster.bro +++ b/scripts/base/frameworks/packet-filter/cluster.bro @@ -1,4 +1,7 @@ +@load base/frameworks/cluster +@load ./main + module PacketFilter; event remote_connection_handshake_done(p: event_peer) &priority=3 From ba258ab8658f18b40ef9ee22f6b4ebca34e35d60 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 01:21:20 -0800 Subject: [PATCH 12/23] Updating CHANGES and VERSION. --- CHANGES | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 60efc5fe36..8244679423 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.2 | 2013-11-07 00:46:48 -0800 + + * Release 2.2. + 2.2-beta-199 | 2013-11-07 00:36:46 -0800 * Fixing warnings during doc build. (Robin Sommer) diff --git a/VERSION b/VERSION index 77286a3a7b..8bbe6cf74a 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-beta-199 +2.2 From 18ba9a1626f54ab4d42f652891d64eeaa7daf451 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 01:22:05 -0800 Subject: [PATCH 13/23] Updating submodule(s). [nomail] --- CHANGES | 2 +- aux/broctl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 8244679423..3362dfc0c2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.2 | 2013-11-07 00:46:48 -0800 +2.2 | 2013-11-07 01:22:05 -0800 * Release 2.2. diff --git a/aux/broctl b/aux/broctl index 1e7df03f6c..c663eb51bc 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 1e7df03f6cb79872392ae74cf23da0d8a03c31df +Subproject commit c663eb51bc259ecb8cf98ed6cfe9e5a8ef34a6f7 From 1164fdfb46c1a1f20c6e31a84cda231509d63d8d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 02:18:11 -0800 Subject: [PATCH 14/23] Removing location information from ssh.log in external tests. --- CHANGES | 5 +++- testing/scripts/diff-canonifier-external | 4 +++ testing/scripts/diff-remove-fields | 34 ++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 1 deletion(-) create mode 100755 testing/scripts/diff-remove-fields diff --git a/CHANGES b/CHANGES index 3362dfc0c2..80122962ff 100644 --- a/CHANGES +++ b/CHANGES @@ -1,8 +1,11 @@ -2.2 | 2013-11-07 01:22:05 -0800 +2.2 | 2013-11-07 02:18:11 -0800 * Release 2.2. + * Removing location information from ssh.log in external tests. + (Robin Sommer) + 2.2-beta-199 | 2013-11-07 00:36:46 -0800 * Fixing warnings during doc build. (Robin Sommer) diff --git a/testing/scripts/diff-canonifier-external b/testing/scripts/diff-canonifier-external index 37a51fa72f..aabe9218e6 100755 --- a/testing/scripts/diff-canonifier-external +++ b/testing/scripts/diff-canonifier-external @@ -8,6 +8,10 @@ if [ "$1" == "capture_loss.log" ]; then addl="`dirname $0`/diff-remove-fractions" fi +if [ "$1" == "ssh.log" ]; then + addl="`dirname $0`/diff-remove-fields remote_location" +fi + `dirname $0`/diff-remove-timestamps \ | `dirname $0`/diff-remove-uids \ | `dirname $0`/diff-remove-file-ids \ diff --git a/testing/scripts/diff-remove-fields b/testing/scripts/diff-remove-fields new file mode 100755 index 0000000000..7f18748a5f --- /dev/null +++ b/testing/scripts/diff-remove-fields @@ -0,0 +1,34 @@ +#! /usr/bin/env bash +# +# A diff canonifier that removes all fields starting with a given +# prefix. + +if [ $# != 1 ]; then + echo "usage: `basename $0` " + exit 1 +fi + +awk -v "PREFIX=$1" ' + +BEGIN { FS="\t"; OFS="\t"; } + +/^#fields/ { + for ( i = 2; i <= NF; ++i ) + { + if ( index($i, PREFIX) == 1 ) + rem[i-1] = 1; + } + print; + next; +} + +{ + for ( i in rem ) + # Mark that it iss set, but ignore content. + $i = "+"; + + print; +} + +' + From 91a6fb9af7bd9ad014715fd059b3da18b58f5107 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 07:09:06 -0800 Subject: [PATCH 15/23] Updating submodule(s). [nomail] --- CHANGES | 2 +- aux/broctl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 80122962ff..a618d2bf48 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.2 | 2013-11-07 02:18:11 -0800 +2.2 | 2013-11-07 07:09:06 -0800 * Release 2.2. diff --git a/aux/broctl b/aux/broctl index c663eb51bc..6a62d12b47 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit c663eb51bc259ecb8cf98ed6cfe9e5a8ef34a6f7 +Subproject commit 6a62d12b478fe6df06746e0cb739cabc1157b203 From 48117eda3789d25d65669c1b071092ab94417ba2 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 10:25:50 -0800 Subject: [PATCH 16/23] Removing one more "beta". --- NEWS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 1b6f3114d0..4ae5669466 100644 --- a/NEWS +++ b/NEWS @@ -4,8 +4,8 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as BroControl and Broccoli, come with their own ``CHANGES``.) -Bro 2.2 Beta -============ +Bro 2.2 +======= New Functionality ----------------- From 071bd2689dbcc36c747831da4fe2f5230eeee7c8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Nov 2013 10:26:35 -0800 Subject: [PATCH 17/23] Updating CHANGES and VERSION. --- CHANGES | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index a618d2bf48..3221435a9a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.2 | 2013-11-07 07:09:06 -0800 +2.2 | 2013-11-07 10:25:50 -0800 * Release 2.2. From 1e43dfc46aee65a3845cf17bc9207190a20387ac Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Fri, 8 Nov 2013 16:49:03 -0500 Subject: [PATCH 18/23] Fix the irc_reply event for certain server message types. --- src/analyzer/protocol/irc/IRC.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/analyzer/protocol/irc/IRC.cc b/src/analyzer/protocol/irc/IRC.cc index 71f591635f..d621ce2cce 100644 --- a/src/analyzer/protocol/irc/IRC.cc +++ b/src/analyzer/protocol/irc/IRC.cc @@ -164,7 +164,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) if ( code > 0 ) { switch ( code ) { - // Ignore unimportant messages. + /* case 1: // RPL_WELCOME case 2: // RPL_YOURHOST case 3: // RPL_CREATED @@ -183,6 +183,7 @@ void IRC_Analyzer::DeliverStream(int length, const u_char* line, bool orig) case 376: // RPL_ENDOFMOTD case 331: // RPL_NOTOPIC break; + */ // Count of users, services and servers in whole network. case 251: From cb6f6467c7f93da79c755479cc8eba40f6c75ae3 Mon Sep 17 00:00:00 2001 From: Jon Crussell Date: Sat, 9 Nov 2013 18:04:36 -0800 Subject: [PATCH 19/23] Fixed Segmentation fault in SQLite Writer. Segmentation fault caused by accessing fields with pos which is one-based for setting SQLite field values. Fix is to simply subtract one from pos. Discovered when trying to store HTTP traffic to a SQLite database with the following Bro script: event bro_init() { local filter: Log::Filter = [ $name = "sqlite", $path = "http", $config = table(["tablename"] = "http_logs"), $writer = Log::WRITER_SQLITE ]; Log::add_filter(HTTP::LOG, filter); } --- src/logging/writers/SQLite.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/logging/writers/SQLite.cc b/src/logging/writers/SQLite.cc index 37e3134659..46d1f17130 100644 --- a/src/logging/writers/SQLite.cc +++ b/src/logging/writers/SQLite.cc @@ -308,7 +308,7 @@ int SQLite::AddParams(Value* val, int pos) if ( j > 0 ) desc.AddRaw(set_separator); - io->Describe(&desc, val->val.set_val.vals[j], fields[pos]->name); + io->Describe(&desc, val->val.set_val.vals[j], fields[pos-1]->name); } desc.RemoveEscapeSequence(set_separator); @@ -330,7 +330,7 @@ int SQLite::AddParams(Value* val, int pos) if ( j > 0 ) desc.AddRaw(set_separator); - io->Describe(&desc, val->val.vector_val.vals[j], fields[pos]->name); + io->Describe(&desc, val->val.vector_val.vals[j], fields[pos-1]->name); } desc.RemoveEscapeSequence(set_separator); From 81d0def32706f0a6b387876f651bd244b6c90036 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Sun, 10 Nov 2013 22:04:16 -0800 Subject: [PATCH 20/23] Add minimal testcase for sqlite writer crash. The writer did not work with a non-empty set or vector as the last element in the logged column. --- .../ssh.select | 1 + .../base/frameworks/logging/sqlite/set.bro | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select create mode 100644 testing/btest/scripts/base/frameworks/logging/sqlite/set.bro diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select new file mode 100644 index 0000000000..71bf36b865 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.sqlite.set/ssh.select @@ -0,0 +1 @@ +CC,AA,BB diff --git a/testing/btest/scripts/base/frameworks/logging/sqlite/set.bro b/testing/btest/scripts/base/frameworks/logging/sqlite/set.bro new file mode 100644 index 0000000000..0da88d5e38 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/logging/sqlite/set.bro @@ -0,0 +1,50 @@ +# +# Check if set wors in last position (the describe call in sqlite.cc has a good +# chance of being off by one if someone changes it). +# +# @TEST-REQUIRES: which sqlite3 +# @TEST-REQUIRES: has-writer SQLite +# @TEST-GROUP: sqlite +# +# @TEST-EXEC: bro -b %INPUT +# @TEST-EXEC: sqlite3 ssh.sqlite 'select * from ssh' > ssh.select +# @TEST-EXEC: btest-diff ssh.select +# +# Testing all possible types. + +redef LogSQLite::unset_field = "(unset)"; + +module SSH; + +export { + redef enum Log::ID += { LOG }; + + type Log: record { + ss: set[string]; + } &log; +} + +function foo(i : count) : string + { + if ( i > 0 ) + return "Foo"; + else + return "Bar"; + } + +event bro_init() +{ + Log::create_stream(SSH::LOG, [$columns=Log]); + Log::remove_filter(SSH::LOG, "default"); + + local filter: Log::Filter = [$name="sqlite", $path="ssh", $writer=Log::WRITER_SQLITE]; + Log::add_filter(SSH::LOG, filter); + + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(SSH::LOG, [ + $ss=set("AA", "BB", "CC") + ]); +} + From f18f9511ac0279224116b85f9a7e25a84f8547bb Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 15 Nov 2013 07:05:15 -0800 Subject: [PATCH 21/23] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index 94eddbcaf5..4908cbe2be 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.2-6 | 2013-11-15 07:05:15 -0800 + + * Make "install-example-configs" target use DESTDIR. (Jon Siwek) + 2.2-5 | 2013-11-11 13:47:54 -0800 * Fix the irc_reply event for certain server message types. (Seth diff --git a/VERSION b/VERSION index 9cf2e10ba7..91e93dce0e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-5 +2.2-6 diff --git a/aux/binpac b/aux/binpac index 0f20a50afa..54b321009b 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 0f20a50afacb68154b4035b6da63164d154093e4 +Subproject commit 54b321009b750268526419bdbd841f421c839313 diff --git a/aux/bro-aux b/aux/bro-aux index ce8a9733c4..ebf9c0d88a 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit ce8a9733c4f8c4c8e8b2e9e440acfcf985f39cd8 +Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8 diff --git a/aux/broccoli b/aux/broccoli index 8600d3afc7..17ec437752 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 8600d3afc748aadbdd729c7bac9e3f69e829c23f +Subproject commit 17ec437752837fb4214abfb0a2da49df74668d5d diff --git a/aux/broctl b/aux/broctl index 6a62d12b47..6e01d6972f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 6a62d12b478fe6df06746e0cb739cabc1157b203 +Subproject commit 6e01d6972f02d68ee82d05f392d1a00725595b7f diff --git a/cmake b/cmake index d902e23fd1..e7a46cb82e 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit d902e23fd14624eb9caf0b4a0e693014bf5bd684 +Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20 From 3effe5df084255dbe7f8613862f522007b0b3133 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 15 Nov 2013 11:39:27 -0600 Subject: [PATCH 22/23] Update local.bro for Bro 2.2 Removed some commented-out code that no longer works in Bro 2.2 (because Notice::policy is now a hook). --- scripts/site/local.bro | 4 ---- 1 file changed, 4 deletions(-) diff --git a/scripts/site/local.bro b/scripts/site/local.bro index 2f4eed9865..ddaee42a93 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -30,10 +30,6 @@ # This adds signatures to detect cleartext forward and reverse windows shells. @load-sigs frameworks/signatures/detect-windows-shells -# Uncomment the following line to begin receiving (by default hourly) emails -# containing all of your notices. -# redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 0] }; - # Load all of the scripts that detect software in various protocols. @load protocols/ftp/software @load protocols/smtp/software From dc52846b6f1b6ae4db4c5815a6e376170082d24e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 18 Nov 2013 14:03:21 -0800 Subject: [PATCH 23/23] Updating submodule(s). [nomail] --- CHANGES | 5 +++++ VERSION | 2 +- aux/btest | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 4908cbe2be..31fddce833 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.2-9 | 2013-11-18 14:03:21 -0800 + + * Update local.bro for Bro >= 2.2. The commented out Notice::policy + example didn't work anymore. (Daniel Thayer) + 2.2-6 | 2013-11-15 07:05:15 -0800 * Make "install-example-configs" target use DESTDIR. (Jon Siwek) diff --git a/VERSION b/VERSION index 91e93dce0e..ac3e73a062 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.2-6 +2.2-9 diff --git a/aux/btest b/aux/btest index c16e914075..26c3136d56 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit c16e91407595f9bf748b0b18e3b7566ccfaa5327 +Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9