From 6ab5701ad023aeaa2d1d2d86902febb1f6fae7b8 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 9 Mar 2015 12:33:56 -0700 Subject: [PATCH 01/22] Update certificate validation script - new version will cache valid intermediate chains that it encounters on the wire and use those to try to validate chains that might be missing intermediate certificates. This vastly improves the number of certificates that Bro can validate. The only drawback is that now validation behavior is not entirely predictable anymore - the certificate of a server can fail to validate when Bro just started up (due to the intermediate missing), and succeed later, when the intermediate can be found in the cache. Has been tested on big-ish clusters and should not introduce any performance problems. --- .../policy/protocols/ssl/validate-certs.bro | 129 +++++++++++++++--- .../ssl.log | 15 ++ .../ssl-all.log | 23 ++++ .../ssl.log | 11 -- .../Traces/tls/missing-intermediate.pcap | Bin 0 -> 13449 bytes .../protocols/ssl/validate-certs-cluster.bro | 37 +++++ .../policy/protocols/ssl/validate-certs.bro | 7 +- 7 files changed, 191 insertions(+), 31 deletions(-) create mode 100644 testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-cluster/ssl.log create mode 100644 testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl-all.log delete mode 100644 testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log create mode 100644 testing/btest/Traces/tls/missing-intermediate.pcap create mode 100644 testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro diff --git a/scripts/policy/protocols/ssl/validate-certs.bro b/scripts/policy/protocols/ssl/validate-certs.bro index 19b0b70806..d2b3befaed 100644 --- a/scripts/policy/protocols/ssl/validate-certs.bro +++ b/scripts/policy/protocols/ssl/validate-certs.bro @@ -1,4 +1,6 @@ ##! Perform full certificate chain validation for SSL certificates. +# Also caches all intermediate certificates encountered so far and use them +# for future validations. @load base/frameworks/notice @load base/protocols/ssl @@ -19,12 +21,92 @@ export { }; ## MD5 hash values for recently validated chains along with the - ## validation status message are kept in this table to avoid constant + ## validation status are kept in this table to avoid constant ## validation every time the same certificate chain is seen. global recently_validated_certs: table[string] of string = table() - &read_expire=5mins &synchronized &redef; + &read_expire=5mins &redef; + + ## Event from a worker to the manager that it has encountered a new + ## valid intermediate + global intermediate_add: event(key: string, value: vector of opaque of x509); + + ## Event from the manager to the workers that a new intermediate chain + ## is to be added + global new_intermediate: event(key: string, value: vector of opaque of x509); } +global intermediate_cache: table[string] of vector of opaque of x509; + +@if ( Cluster::is_enabled() ) +@load base/frameworks/cluster +redef Cluster::manager2worker_events += /SSL::intermediate_add/; +redef Cluster::worker2manager_events += /SSL::new_intermediate/; +@endif + + +function add_to_cache(key: string, value: vector of opaque of x509) + { + intermediate_cache[key] = value; +@if ( Cluster::is_enabled() ) + event SSL::new_intermediate(key, value); +@endif + } + +@if ( Cluster::is_enabled() && Cluster::local_node_type() != Cluster::MANAGER ) +event SSL::intermediate_add(key: string, value: vector of opaque of x509) + { + intermediate_cache[key] = value; + } +@endif + +@if ( Cluster::is_enabled() && Cluster::local_node_type() == Cluster::MANAGER ) +event SSL::new_intermediate(key: string, value: vector of opaque of x509) + { + if ( key in intermediate_cache ) + return; + + intermediate_cache[key] = value; + event SSL::intermediate_add(key, value); + } +@endif + +function cache_validate(chain: vector of opaque of x509): string + { + local chain_hash: vector of string = vector(); + + for ( i in chain ) + chain_hash[i] = sha1_hash(x509_get_certificate_string(chain[i])); + + local chain_id = join_string_vec(chain_hash, "."); + + # If we tried this certificate recently, just return the cached result. + if ( chain_id in recently_validated_certs ) + return recently_validated_certs[chain_id]; + + local result = x509_verify(chain, root_certs); + recently_validated_certs[chain_id] = result$result_string; + + # if we have a working chain where we did not store the intermediate certs + # in our cache yet - do so + if ( result$result_string == "ok" && result?$chain_certs && |result$chain_certs| > 2 ) + { + local result_chain = result$chain_certs; + local icert = x509_parse(result_chain[1]); + if ( icert$subject !in intermediate_cache ) + { + local cachechain: vector of opaque of x509; + for ( i in result_chain ) + { + if ( i >=1 && i<=|result_chain|-2 ) + cachechain[i-1] = result_chain[i]; + } + add_to_cache(icert$subject, cachechain); + } + } + + return result$result_string; + } + event ssl_established(c: connection) &priority=3 { # If there aren't any certs we can't very well do certificate validation. @@ -32,9 +114,30 @@ event ssl_established(c: connection) &priority=3 ! c$ssl$cert_chain[0]?$x509 ) return; - local chain_id = join_string_vec(c$ssl$cert_chain_fuids, "."); - local hash = c$ssl$cert_chain[0]$sha1; + local intermediate_chain: vector of opaque of x509 = vector(); + local issuer = c$ssl$cert_chain[0]$x509$certificate$issuer; + local result: string; + # look if we already have a working chain for the issuer of this cert. + # If yes, try this chain first instead of using the chain supplied from + # the server. + if ( issuer in intermediate_cache ) + { + intermediate_chain[0] = c$ssl$cert_chain[0]$x509$handle; + for ( i in intermediate_cache[issuer] ) + intermediate_chain[i+1] = intermediate_cache[issuer][i]; + + result = cache_validate(intermediate_chain); + if ( result == "ok" ) + { + c$ssl$validation_status = result; + return; + } + } + + # validation with known chains failed or there was no fitting intermediate + # in our store. + # Fall back to validating the certificate with the server-supplied chain local chain: vector of opaque of x509 = vector(); for ( i in c$ssl$cert_chain ) { @@ -42,24 +145,14 @@ event ssl_established(c: connection) &priority=3 chain[i] = c$ssl$cert_chain[i]$x509$handle; } - if ( chain_id in recently_validated_certs ) - { - c$ssl$validation_status = recently_validated_certs[chain_id]; - } - else - { - local result = x509_verify(chain, root_certs); - c$ssl$validation_status = result$result_string; - recently_validated_certs[chain_id] = result$result_string; - } + result = cache_validate(chain); + c$ssl$validation_status = result; - if ( c$ssl$validation_status != "ok" ) + if ( result != "ok" ) { local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status); NOTICE([$note=Invalid_Server_Cert, $msg=message, $sub=c$ssl$subject, $conn=c, - $identifier=cat(c$id$resp_h,c$id$resp_p,hash,c$ssl$validation_status)]); + $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status)]); } } - - diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-cluster/ssl.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-cluster/ssl.log new file mode 100644 index 0000000000..df2cdf9732 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-cluster/ssl.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-03-09-19-32-44 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1425929564.247511 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FTzCuuqU5y7w85H89 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate +1425929565.270104 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FXzQOu1ZSKSF7H8Ez6 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate +1425929566.843026 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T F5l2dVkZHiwiOWR67,Fkw2ETDXfIXIvatba,Fbgf8A3V6m8v33wTcj (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +1425929571.372511 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FhEtvg4pQ90832J56f (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +1425929567.865619 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T Fyc6cQ2rMCAhpIGcM5,FoJ8j735m9ogDYopYj,FHaYhA3ykzVlKPnnsc (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +1425929572.395104 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FwZZ8034tgyXSponwg (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +#close 2015-03-09-19-32-53 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl-all.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl-all.log new file mode 100644 index 0000000000..77ba9233ae --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl-all.log @@ -0,0 +1,23 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-03-09-19-44-42 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1394745602.951961 CXWv6p3arKYeMETxOg 192.168.4.149 60539 87.98.220.10 443 TLSv10 TLS_DHE_RSA_WITH_AES_256_CBC_SHA - - F - - T F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl (empty) CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - certificate has expired +1394745618.791420 CjhGID4nQcgTWjvg4c 192.168.4.149 60540 122.1.240.204 443 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - F - - T F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h (empty) CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - ok +#close 2015-03-09-19-44-42 +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-03-09-19-44-42 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1417039703.224578 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FghNi02cFL9n6ttuMa (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate +1417039705.820093 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T Fz7gr4fSm2T2sEyDl,FhjNBG25vvoBO6CS79,FQFHJA20WL56NP6LXk (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +1417039710.349578 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FRcFYq3e3hgYkZ8dS1 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +#close 2015-03-09-19-44-42 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log deleted file mode 100644 index a464c64670..0000000000 --- a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log +++ /dev/null @@ -1,11 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path ssl -#open 2014-08-08-17-13-58 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status -#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string -1394745602.951961 CXWv6p3arKYeMETxOg 192.168.4.149 60539 87.98.220.10 443 TLSv10 TLS_DHE_RSA_WITH_AES_256_CBC_SHA - - F - - T F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl (empty) CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - certificate has expired -1394745618.791420 CjhGID4nQcgTWjvg4c 192.168.4.149 60540 122.1.240.204 443 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - F - - T F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h (empty) CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - ok -#close 2014-08-08-17-13-58 diff --git a/testing/btest/Traces/tls/missing-intermediate.pcap b/testing/btest/Traces/tls/missing-intermediate.pcap new file mode 100644 index 0000000000000000000000000000000000000000..9f44e3e4d25f2f584096819fd7ac4ace9b758455 GIT binary patch literal 13449 zcmeHOc|29y+h1oi3&&XG6v=EK^N=DTWK49ZB;%1OPDGibXfPy=N(hxoQW~y0CEYR< zH&-f(kOn1@S>CmeWW4V0{yy&?@8|v8UF(y*?fre$de(P;_j;c7R9jkj1`F_D;=lt8 zJh8V7vfa1|2Q1*vXaiDv+YdzLk=<99TeAW~0Q8Es>j6C?0E|5R;Yjki{jFy(x5Z+e zOwgMPu%n}$q5xp=gdz+MhsR)8+3^Zz-lD(5W5>`#yeH7daK~(np;Pq$*h%a=fFh$7 zFt=OJxRX=*p#{?dK0R>$-Y;~3qNB*@fx+OhXx}eD^p)@&Z8Q3=dc4O77&^2hAj)?I*;5Y}OFj?;Fs;P*AI1EM!12A$pTz+nKrB3eA z(PkFe601!Lq?(TIngq|PTYN}1TUWYe7XYaYgdxw6W=Jyl8SEenoCaHg7tjKlU@zFi zP-ZAHWEoP7c?<$b2f9EF>}Du2=Rxv0H1%?b` z34@z~2PeTYpajx@K44++GFTX#3@ji3JcEzH3IqX(!NtGf3y8pLoIn`f%nlzHV&2XI9~6KrBR~)mU4v*ydQBmYQHVc5wyd32toCQKIr5yVrSRM|K0}$2$ zf(;;8BGCjhBpR=h!R$YaoaFhI-Xk#x7m-aiZgpHA7Y4&Y07%qB3J1bQ#I3^(h>8QELs^7~k99_Pm7k9@&6jN8=SvUp^CsIld(&JY zq$@=b;U6>26&xI_=n_Pu1uFUlcp%Z3#giR}Apl%7W*&go*|E_W49Jt>@av*`XjLz? z5>F|9(Ds1F`(4d_MK0Y~cyFM;^p6YbBVrrQoIl&R4}D*dI9P(EBLnxw_Gv3<2s*)5^P1+;OW~ z!cKAO$!okn5>IUNa3{V`3=P^~XO`ihv-sltr+NPEt_O!*Kfc@5O&WY{xy7kPohWbp zz=N{)+%{60%|!td8S&%t%OAQPCHf?fvg+6x^3)Uz_fmfo;XYr^$+;)FV<#2|btEeq z^ALLEJ~9vbP>h7ZjN%Ch)EIz>p%<3mQSY+K5itY;3l8(^CoGe3V)M}Z@i;6Y1JOZm z7so>$AV`K{hJ4&F_5^s(!ks{Onul^AML|Q+CBPe@q7RetHi#8sLFCvRw;B^S;)`Js z!XPUU-EpH>j38waBY#rrZ-|LRMUOuZB|ZqDBeLV?2qtdfPdq=DK>t}fi5UI57aOPj zk8Lx@2**cfa#&0PFN_FA@-H!r^0boD)YVatzfMq_cwLIhGVqG==T>G*m>l<1}#YIz9P_vRO*KzU8p4u4m46p7 zXnbARXU|fQzK*Vedw4xn;9$Z29q-D`g6^uBjvi?^pHwtysN}RIT&e@galnETZHd3sWs3XF!Kth#21uDN0{92%G zy7B`}>Pl4kiw{jz{-bA@$}jw*LZWx7@^eEJ6!jrQT?v0i+w`wGP?g}B?W?{y+?{;% zY$T6(^`v#e2lShQBKk+-zz+CrMB8*kj8__V4n$!x_zH*jbLGKAMAr#K9{7dD0J6>A zm0Y_?)DlGV{^>i}_b=M$U0>w8;WW>F!?+F-QrjoRUhkJKuGbe%8?ERRvm4rc%6T{= zv#aHgll?K&;@9Fg{Ic@*9Sfn~v=c9N*zu|}%|cC^?Rj|HHak146%dtu!6F;K#c_Q; zF>8Lgnd9{>N&4DH_N=Y1$q?mLD7oL|-*;A{IV$g|%{#xyT1C*B8?_<{v)1d0s=))m8cJy<3{-i;^y}6FXAYuBRn#aCK5fGjVhtsdI4(khMtWrH%<2>@#Xdos@25;vr!#C@v!xZ2Rr!#yysb@9Wcz~ zM|BPZ9RkS7^q8ZgTeNTK|CxG6EIw=A2?0re$z;toQ;*k6?hv(Kqw|pV5(n2O=o)1f zzBo}0n0(nfjW1`#Vx5a3g+~q#{G5lMD}@h}jAKL2c%KSSvs!Z0p{+`3qcE<5J>~9HD$bc(6Mz? zh8_3pU}989`Gx&`58A|Xh-{s#!V=?XEY};h39CS zF?Q$>J)lgi-Gw6m^k!x(b>2#7VRZ(dX3|MLCb}yg%gWATcCrGmaG@PWXe=mlF+~3B znXd|)wco2Dx&e%tcAgPKucv?w2+7TpW5o z+%a&6-}i@N)~mA@+Tu25&_rC=g5n0Y{Tnm=jG53`aBrTN*BOja_3B0$Lrb2<7=xJm z7k5SSfKbfOmeJi9C1%?*C#K&86;)f2LlEIfZ!_prthAeS_3nt>tsj=*uH3k#~L=L?RN0JOe!$rlP zD{|PcK>Ez}{@+H7_w>tzwLTXJiJ$y(;tuG1jJ(t0b{mQ?oRZkX3{`bet8>F05Mo*SH= z${*p~llI1EW8EHYh0P6DQ^m{z&fb>xEEU@JbpFF?Hv)^`f&H~|neOvq^7~?ih5wKf zQ*+GY^7dd$4Qtu`f?~(G#gRI{Y5kK=S^|IWUQoGG+`jjrnpODaYXU*f16u-5Ygjh2 zMX)&!k_}qcm<1f!FFSgA9z$n5Vyu9k`W@kft{saQu?UzE5B^Q~sDwVui?UAZBVa~@n(n#Vmvuc#vz)B`nVYwC;y7Q|2}gwP2AA15moB`xt01m zQ>itkDmDL!aqXXXYpPO%7IXK6W{+zkE1Y*ST0ivko7xI(>2Q0o=6IP)1+pkCZ&mA4Czh ztvjN$!Uio^G!L;|Qrt;Qs_hyy00cbzuE5+A?)}(z1;aAU@kmtSe^YKY3O&x6Y3)Biu;!N1ECwk5fnbqpA7J4cSW^TGSB>EqS3sO=U$ zlD!o|r2c|!)ZL+5B8lTW$FsP`JBF1-*`H_Sob zdL#GJiU+nmMU+*IhwS!eui11WzCtqQ#9RBY;(9*u$FV*=HIZFbC-z*b<=`O)X70Uyxby(m?){rjK6|1wS6( zULGWLH!MCfa+N{5LX!d8_Lu7KFTRS`{=Sj#paec&aJM=K8PhkvbYEVDY9taEh}@)lGHainhh-M|@FxH#jb5T^rgr-}TK6 zE9c6!)_O8#_j@jBJGG)p&4HhDP2I~>YW1l~Ez&)%)Z4C3RqFDKQk53Y25lquIU4>? z4CD$@OH)2HJDuk-DWy9L;@3{xL4!j>G@PNH)t3OyY`(atDpHNbgJtCwTz^P)f#RZ^%x2%v zs1YPTRQz}9PJnBMqS}Z=W6ORDE12f^|C_=JiKPV>+ROv4yHzwO$|~@$nwFZD!~ek^ z|D8gXe7=Vp?oAxVUh#80z4}FZ$lJ_foU@&9sqyva<+^H$@dkZoXeVAL%pW0+SCcKnk*wAHCB{w$A8ywSRz8{0+SMYP&p zGV5O#V2>5)7urVsv{0IbUY5T4j(1T{c0sqRMo0>y!)Q}Pv(MT!TP|0LbGY2rP9GW$ z0Y{zbPo<*uT)GPbyhON;mDh4_uhbaY7k`i1wI#yos*@YipIdAmO=gRWBU{ilyDiOTbg{HrsajS%wfZPG_q*4_ zr5)l)U1JKl64Co7>NuZWGtLq1zJvfa)Us!w$ zTz~IwX42OGKBrpJQ2#eaUE7FHCO@LUbE?Fh^LjD{t5k=AhXP;P*ZD_X;mG~sJ4}T8 zB4^&$1oJd%rF?CE%6P$^({|{p?1~GTca$oI1h;Yd@&#Vlt62DT^lDM6q=4zS57%jf zYl_tNswH%LIIQdoQA}N^VH1AlyUT?dJBd>!_YbTUZMA;+pylEAhdU~lMI`H{-_R1* ziQk9xo>a?89wIVUHmCiLDm4#O>K8ESfVSz`4^ZLNhbpya?^LB0H)ATbFbMtg%v7cB z@$|t5cVRz+^jybubHjorPc?Dj*2@)=E7kX37zjpkwuYh$81RF;}v|I zrFQ4sTsw@C@XMQExAMHHVC(Q?)lBZ?^g79tX7LtICktB+RK{;H2m z#E-KfN(yiR2J`^hW+a3Qm%-%Lc^%^167;XsJFaacbXqUG1FW?TK(CX4&8UBp6PD9Ynmx)6E3+NRSLqwdwuq-(-7y1s>U9gKu@4dNQ7YdS39$(KaO zQx5Be9>UA=fUz0#Hp=9cdx8XU^pj_%{M%(;E_5z_*t={|^}to?x*bG|foEww#U3uv zY7U7l9frz!Z_L$BE)gAx*zi2eLtrs?|BE4k6T9!Hh>CtW- zE{1WvKum7u;i!6PA}{+_R;<~L2Ne%CUKc&p^txfirNcTEZpwbcCs;|wDfLbFqVp~r zo7<^*Eh;YZ8nQolK|2fZFBNl>wYpO#+owCQaU{NkFRlt3(4`~il(ETzSX2J2L{@#0 zPrt0Z{#I-^WvEVHS9>4`ul`cDSg1l#XN!HU3~MG!)mLZRRv{7F;1j4%Z$qE{b*h|C zU{;O{#!|_xkjI^qSf|Tkmd?HrB3>KD0%u4#w9R1MTA%xKAfl{`9cSH5X%tZa&eHs! zB&l7Z^ZeOmL-WChZLycUYZ5I3TTp{6BiK2HAeKIx!Al~{A;5i}i6M*`WDdar_iG5# zi3Zks4WLA$c~2!8xi#kgrTHk)epebN+IbRQ;ifcm2tVg!*y2cg!Z+*9CUn8?A1ik# z`5ilm8oUJ!j?cmjgQ{MGOoP*yy$HuN$O}SiuTM3IB3e7*fI6H2w9S|W>AkpFoiZQ2 z*$TcBbO!)7K@>wH!WoHLIcdshIP1utHq z(O_g>$I#J459+g5P4a z%~&sJQlBx4Ww29~KXpG-`C;K4i`!J?M`J@2H4UQv^*Vv;#cW@7A6~D*)Fik9o>Vwj z0_%}1cu_VsbUf~wST+4bge%~Q!nrX-be-_Ca86CU*t{giQR3XA)5#Fl3TbUl&SY$pIm2JcUNjp z{OaIG68U>8WJ)_k1cJTa>@v>3uw7f>N&e}g6WfFjY)*GP)y&SgD3M*c;Uzsr%k8&F zleeY8qF)A2?T@)5BSz}?{$Mqlx1{3i;f42Ovw$xd^G1{uZCP2{t@{2Idm=_jhPF`m zk<-g%tMVkm`NirCx(!oxKYa2^EWBdehD&dgE`+XlxYt4$v#-Aaj7Z8bBw zQ{cFwMDvHB|00zy`fbI&JI_(RKCnw=(^7vzx}MN?H+P$BVfve_)st%1Ot@V+uFX#j zx_Zp#wpo4R6>8O&K7TI!cgiDLt?W6^Msyy2dqtS5aIO~R%k`Jj_>#mk>T8S^&dKpj z;Y-dWEuH*qqDT;6Tj8nvRNj# z$Hu%oI^q^;C>$F4>o+0_{?~+={Po?$>^cmnCJN`q455e$Z4-zy-dt4&J?2atn>G_L zmhg2#3+H$hf6c@Kyu!Ua6ft@d;>^`OD>7>zoyOcdrH>b&GZvtTi4YMj!$aGQK1w?< V|3k+0?AiK=mf|_W4d$`qe*hjM$^QTV literal 0 HcmV?d00001 diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro b/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro new file mode 100644 index 0000000000..db9c6cd9da --- /dev/null +++ b/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro @@ -0,0 +1,37 @@ +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run proxy-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=proxy-1 bro %INPUT" +# @TEST-EXEC: btest-bg-run proxy-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=proxy-2 bro %INPUT" +# @TEST-EXEC: sleep 1 +# @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/missing-intermediate.pcap %INPUT" +# @TEST-EXEC: btest-bg-run worker-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/missing-intermediate.pcap %INPUT" +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: cat manager-1/ssl*.log > ssl.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-file-ids btest-diff ssl.log +# + +redef Log::default_rotation_interval = 0secs; + +@TEST-START-FILE cluster-layout.bro +redef Cluster::nodes = { + ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")], + ["proxy-1"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37758/tcp, $manager="manager-1", $workers=set("worker-1", "worker-2")], + ["proxy-2"] = [$node_type=Cluster::PROXY, $ip=127.0.0.1, $p=37759/tcp, $manager="manager-1", $workers=set("worker-2")], + ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth0"], + ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $proxy="proxy-1", $interface="eth1"], +}; +@TEST-END-FILE + +event terminate_me() { + terminate(); +} + +event remote_connection_closed(p: event_peer) { + schedule 1sec { terminate_me() }; +} + + +@load base/frameworks/cluster +@load protocols/ssl/validate-certs.bro diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro b/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro index 56408483f0..19fca8cb89 100644 --- a/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro +++ b/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro @@ -1,4 +1,7 @@ # @TEST-EXEC: bro -r $TRACES/tls/tls-expired-cert.trace %INPUT -# @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: cat ssl.log > ssl-all.log +# @TEST-EXEC: bro -C -r $TRACES/tls/missing-intermediate.pcap %INPUT +# @TEST-EXEC: cat ssl.log >> ssl-all.log +# @TEST-EXEC: btest-diff ssl-all.log -@load protocols/ssl/validate-certs +@load protocols/ssl/validate-certs.bro From 144302d3e7cb07e2001a10a9c0c4009112254578 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 9 Mar 2015 12:53:17 -0700 Subject: [PATCH 02/22] add knob to revert to old validation behavior --- .../policy/protocols/ssl/validate-certs.bro | 19 +++++++++++++++++-- .../ssl.log | 12 ++++++++++++ .../protocols/ssl/validate-certs-no-cache.bro | 6 ++++++ 3 files changed, 35 insertions(+), 2 deletions(-) create mode 100644 testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-no-cache/ssl.log create mode 100644 testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro diff --git a/scripts/policy/protocols/ssl/validate-certs.bro b/scripts/policy/protocols/ssl/validate-certs.bro index d2b3befaed..09000164aa 100644 --- a/scripts/policy/protocols/ssl/validate-certs.bro +++ b/scripts/policy/protocols/ssl/validate-certs.bro @@ -26,6 +26,18 @@ export { global recently_validated_certs: table[string] of string = table() &read_expire=5mins &redef; + ## Use intermediate CA certificate caching when trying to validate + ## certificates. When this is enabled, Bro keeps track of all valid + ## intermediate CA certificates that it has seen in the past. When + ## encountering a host-certificate that cannot be validated because + ## of missing intermediate CA certificate, the cached list is used + ## to try to validate the cert. This is similar to how Firefox is + ## doing certificate validation. + ## Disabling this will usually greatly increase the number of validation + ## warnings that you encounter. Only disable if you want to find misconfigured + ## servers. + global ssl_cache_intermediate_ca: bool = T &redef; + ## Event from a worker to the manager that it has encountered a new ## valid intermediate global intermediate_add: event(key: string, value: vector of opaque of x509); @@ -88,7 +100,10 @@ function cache_validate(chain: vector of opaque of x509): string # if we have a working chain where we did not store the intermediate certs # in our cache yet - do so - if ( result$result_string == "ok" && result?$chain_certs && |result$chain_certs| > 2 ) + if ( ssl_cache_intermediate_ca && + result$result_string == "ok" && + result?$chain_certs && + |result$chain_certs| > 2 ) { local result_chain = result$chain_certs; local icert = x509_parse(result_chain[1]); @@ -121,7 +136,7 @@ event ssl_established(c: connection) &priority=3 # look if we already have a working chain for the issuer of this cert. # If yes, try this chain first instead of using the chain supplied from # the server. - if ( issuer in intermediate_cache ) + if ( ssl_cache_intermediate_ca && issuer in intermediate_cache ) { intermediate_chain[0] = c$ssl$cert_chain[0]$x509$handle; for ( i in intermediate_cache[issuer] ) diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-no-cache/ssl.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-no-cache/ssl.log new file mode 100644 index 0000000000..9f33703649 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs-no-cache/ssl.log @@ -0,0 +1,12 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path ssl +#open 2015-03-09-19-51-25 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1417039703.224578 CXWv6p3arKYeMETxOg 192.168.4.149 58529 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FghNi02cFL9n6ttuMa (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate +1417039705.820093 CjhGID4nQcgTWjvg4c 192.168.4.149 58530 72.167.102.91 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 secp256r1 - F - - T Fz7gr4fSm2T2sEyDl,FhjNBG25vvoBO6CS79,FQFHJA20WL56NP6LXk (empty) CN=valid.sfig2.catest.starfieldtech.com,O=Starfield Technologies\, LLC,L=Scottsdale,ST=Arizona,C=US,serialNumber=R-1724741-6,businessCategory=Private Organization,jurisdictionST=Arizona,jurisdictionC=US CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - ok +1417039710.349578 CCvvfg3TEfuqmmG4bh 192.168.4.149 58532 128.32.169.140 443 TLSv10 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FRcFYq3e3hgYkZ8dS1 (empty) CN=www.cviis.org,OU=Domain Control Validated CN=Starfield Secure Certificate Authority - G2,OU=http://certs.starfieldtech.com/repository/,O=Starfield Technologies\, Inc.,L=Scottsdale,ST=Arizona,C=US - - unable to get local issuer certificate +#close 2015-03-09-19-51-25 diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro b/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro new file mode 100644 index 0000000000..1bca5b5c50 --- /dev/null +++ b/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro @@ -0,0 +1,6 @@ +# @TEST-EXEC: bro -C -r $TRACES/tls/missing-intermediate.pcap %INPUT +# @TEST-EXEC: btest-diff ssl.log + +@load protocols/ssl/validate-certs.bro + +redef SSL::ssl_cache_intermediate_ca = F; From d208c95e9a030142971c1982b101db70677e52f3 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 9 Mar 2015 12:56:55 -0700 Subject: [PATCH 03/22] and still use the hash for notice suppression. --- scripts/policy/protocols/ssl/validate-certs.bro | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/policy/protocols/ssl/validate-certs.bro b/scripts/policy/protocols/ssl/validate-certs.bro index 09000164aa..6e4aba704b 100644 --- a/scripts/policy/protocols/ssl/validate-certs.bro +++ b/scripts/policy/protocols/ssl/validate-certs.bro @@ -131,6 +131,7 @@ event ssl_established(c: connection) &priority=3 local intermediate_chain: vector of opaque of x509 = vector(); local issuer = c$ssl$cert_chain[0]$x509$certificate$issuer; + local hash = c$ssl$cert_chain[0]$sha1; local result: string; # look if we already have a working chain for the issuer of this cert. @@ -168,6 +169,6 @@ event ssl_established(c: connection) &priority=3 local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status); NOTICE([$note=Invalid_Server_Cert, $msg=message, $sub=c$ssl$subject, $conn=c, - $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status)]); + $identifier=cat(c$id$resp_h,c$id$resp_p,hash,c$ssl$validation_status)]); } } From 2d82cab9989bda7723b390df3df556e55610eba1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 11 Mar 2015 16:48:38 -0500 Subject: [PATCH 04/22] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 1a49b0e3d2..25cf717eca 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 1a49b0e3d23fdfe8da3187dddb310883b641e4a3 +Subproject commit 25cf717ecad9b3012dbf48a5d5102e89da246753 From 9bb00639bab056f6b257ac815dc1dc8b10f30d39 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 11 Mar 2015 17:01:13 -0500 Subject: [PATCH 05/22] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 25cf717eca..78b8d909fa 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 25cf717ecad9b3012dbf48a5d5102e89da246753 +Subproject commit 78b8d909fadc66dd20ef89bc62b52b4e7c4b6f5f From 0a31fd7a69779850d14f7123c2f7dc4aa4bcaedf Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Mar 2015 10:16:12 -0500 Subject: [PATCH 06/22] Give broker python bindings default install path within --prefix. --- CHANGES | 5 +++++ VERSION | 2 +- aux/broker | 2 +- configure | 6 ++++++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 836733370a..c93a21c4d3 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-533 | 2015-03-12 10:18:53 -0500 + + * Give broker python bindings default install path within --prefix. + (Jon Siwek) + 2.3-530 | 2015-03-10 13:22:39 -0500 * Fix broker data stores in absence of --enable-debug. (Jon Siwek) diff --git a/VERSION b/VERSION index 4a351a524e..781ee30b74 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-530 +2.3-533 diff --git a/aux/broker b/aux/broker index 78b8d909fa..11fd5761a6 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 78b8d909fadc66dd20ef89bc62b52b4e7c4b6f5f +Subproject commit 11fd5761a651d18d5ab80d7da545a1980c642e6d diff --git a/configure b/configure index 3f7295711c..b139ee2bec 100755 --- a/configure +++ b/configure @@ -149,6 +149,10 @@ while [ $# -ne 0 ]; do append_cache_entry CMAKE_INSTALL_PREFIX PATH $optarg append_cache_entry BRO_ROOT_DIR PATH $optarg append_cache_entry PY_MOD_INSTALL_DIR PATH $optarg/lib/broctl + + if [ -n "$user_enabled_broker" ]; then + append_cache_entry BROKER_PYTHON_HOME PATH $prefix + fi ;; --scriptdir=*) append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg @@ -189,6 +193,8 @@ while [ $# -ne 0 ]; do --enable-broker) append_cache_entry ENABLE_CXX11 BOOL true append_cache_entry ENABLE_BROKER BOOL true + append_cache_entry BROKER_PYTHON_HOME PATH $prefix + user_enabled_broker="true" ;; --disable-broccoli) append_cache_entry INSTALL_BROCCOLI BOOL false From ccd5387a9fcba63f854b1d19f0b55986d8e89311 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Mar 2015 10:59:49 -0500 Subject: [PATCH 07/22] Update NEWS file. BIT-1338 #close --- CHANGES | 4 ++++ NEWS | 11 +++++++---- VERSION | 2 +- aux/broker | 2 +- 4 files changed, 13 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index c93a21c4d3..627aade078 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-534 | 2015-03-12 10:59:49 -0500 + + * Update NEWS file. (Jon Siwek) + 2.3-533 | 2015-03-12 10:18:53 -0500 * Give broker python bindings default install path within --prefix. diff --git a/NEWS b/NEWS index ec94bd10fe..50e5ddd265 100644 --- a/NEWS +++ b/NEWS @@ -46,11 +46,9 @@ New Functionality TODO: Extend with some more information on Broker. Broker support is by default off for now; it can be enabled at - configure time with --enable-broker. It requires CAF + configure time with --enable-broker. It requires CAF version 0.13+ (https://github.com/actor-framework/actor-framework) as well as a - C++11 compiler. - - TODO: Add minumim version for CAF. + C++11 compiler (e.g. GCC 4.8+ or Clang 3.3+). Broker will become a mandatory dependency in future Bro versions. @@ -75,6 +73,11 @@ Changed Functionality have been added which contain the same information. The ``mime_type`` field of ``Files::Info`` also still has this info. + * The earliest point that new mime type information is available is + in the ``file_mime_type`` event which comes after the ``file_new`` + and ``file_over_new_connection`` events. Scripts which inspected + mime type info within those events will need to be adapted. + * Removed ``Files::add_analyzers_for_mime_type`` function. * Removed ``offset`` parameter of the ``file_extraction_limit`` diff --git a/VERSION b/VERSION index 781ee30b74..724cf738c6 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-533 +2.3-534 diff --git a/aux/broker b/aux/broker index 11fd5761a6..0aa02aa696 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 11fd5761a651d18d5ab80d7da545a1980c642e6d +Subproject commit 0aa02aa6964e75de75af08ad71067f29cd8d2641 From b47376b8e4de7cc8f44bd746ceeb51b3e65469b7 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Mar 2015 13:09:44 -0500 Subject: [PATCH 08/22] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 0aa02aa696..1a2ab9ee7c 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 0aa02aa6964e75de75af08ad71067f29cd8d2641 +Subproject commit 1a2ab9ee7c80ca905e86a2a11283e7c0477341a9 From c56df225b051edc1a98e23a0917206744d2ab8e3 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 12 Mar 2015 16:16:24 -0500 Subject: [PATCH 09/22] Fix Broker leak tests. Forgot to update Broker module names when they changed. --- CHANGES | 4 + VERSION | 2 +- .../clone.clone.out | 5 + .../bro..stdout | 32 +-- .../bro..stdout | 14 ++ .../recv.recv.out | 0 .../send.send.out | 2 +- .../recv.recv.out | 0 .../recv.test.log | 0 .../send.send.out | 1 + .../send.test.log | 0 .../recv.recv.out | 0 .../send.send.out | 2 +- .../clone.clone.out | 5 - .../core.leaks.comm.master_store/bro..stdout | 14 -- .../core.leaks.comm.remote_log/send.send.out | 1 - .../btest/core/leaks/broker/clone_store.bro | 113 +++++++++ testing/btest/core/leaks/broker/data.bro | 233 ++++++++++++++++++ .../leaks/{comm => broker}/master_store.bro | 46 ++-- .../leaks/{comm => broker}/remote_event.test | 32 +-- .../leaks/{comm => broker}/remote_log.test | 16 +- .../leaks/{comm => broker}/remote_print.test | 28 +-- testing/btest/core/leaks/comm/clone_store.bro | 113 --------- testing/btest/core/leaks/comm/data.bro | 233 ------------------ 24 files changed, 450 insertions(+), 446 deletions(-) create mode 100644 testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out rename testing/btest/Baseline/{core.leaks.comm.data => core.leaks.broker.data}/bro..stdout (73%) create mode 100644 testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout rename testing/btest/Baseline/{core.leaks.comm.remote_event => core.leaks.broker.remote_event}/recv.recv.out (100%) rename testing/btest/Baseline/{core.leaks.comm.remote_event => core.leaks.broker.remote_event}/send.send.out (79%) rename testing/btest/Baseline/{core.leaks.comm.remote_log => core.leaks.broker.remote_log}/recv.recv.out (100%) rename testing/btest/Baseline/{core.leaks.comm.remote_log => core.leaks.broker.remote_log}/recv.test.log (100%) create mode 100644 testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out rename testing/btest/Baseline/{core.leaks.comm.remote_log => core.leaks.broker.remote_log}/send.test.log (100%) rename testing/btest/Baseline/{core.leaks.comm.remote_print => core.leaks.broker.remote_print}/recv.recv.out (100%) rename testing/btest/Baseline/{core.leaks.comm.remote_print => core.leaks.broker.remote_print}/send.send.out (62%) delete mode 100644 testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out delete mode 100644 testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout delete mode 100644 testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out create mode 100644 testing/btest/core/leaks/broker/clone_store.bro create mode 100644 testing/btest/core/leaks/broker/data.bro rename testing/btest/core/leaks/{comm => broker}/master_store.bro (62%) rename testing/btest/core/leaks/{comm => broker}/remote_event.test (66%) rename testing/btest/core/leaks/{comm => broker}/remote_log.test (80%) rename testing/btest/core/leaks/{comm => broker}/remote_print.test (65%) delete mode 100644 testing/btest/core/leaks/comm/clone_store.bro delete mode 100644 testing/btest/core/leaks/comm/data.bro diff --git a/CHANGES b/CHANGES index 627aade078..926b30c9c0 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-536 | 2015-03-12 16:16:24 -0500 + + * Fix Broker leak tests. (Jon Siwek) + 2.3-534 | 2015-03-12 10:59:49 -0500 * Update NEWS file. (Jon Siwek) diff --git a/VERSION b/VERSION index 724cf738c6..c168eac2bd 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-534 +2.3-536 diff --git a/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out b/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out new file mode 100644 index 0000000000..017537fea9 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.broker.clone_store/clone.clone.out @@ -0,0 +1,5 @@ +clone keys, [status=BrokerStore::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] +lookup, one, [status=BrokerStore::SUCCESS, result=[d=broker::data{111}]] +lookup, two, [status=BrokerStore::SUCCESS, result=[d=broker::data{222}]] +lookup, myset, [status=BrokerStore::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup, myvec, [status=BrokerStore::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] diff --git a/testing/btest/Baseline/core.leaks.comm.data/bro..stdout b/testing/btest/Baseline/core.leaks.broker.data/bro..stdout similarity index 73% rename from testing/btest/Baseline/core.leaks.comm.data/bro..stdout rename to testing/btest/Baseline/core.leaks.broker.data/bro..stdout index eea78d39a2..628870144a 100644 --- a/testing/btest/Baseline/core.leaks.comm.data/bro..stdout +++ b/testing/btest/Baseline/core.leaks.broker.data/bro..stdout @@ -1,18 +1,18 @@ -Comm::BOOL -Comm::INT -Comm::COUNT -Comm::DOUBLE -Comm::STRING -Comm::ADDR -Comm::SUBNET -Comm::PORT -Comm::TIME -Comm::INTERVAL -Comm::ENUM -Comm::SET -Comm::TABLE -Comm::VECTOR -Comm::RECORD +BrokerComm::BOOL +BrokerComm::INT +BrokerComm::COUNT +BrokerComm::DOUBLE +BrokerComm::STRING +BrokerComm::ADDR +BrokerComm::SUBNET +BrokerComm::PORT +BrokerComm::TIME +BrokerComm::INTERVAL +BrokerComm::ENUM +BrokerComm::SET +BrokerComm::TABLE +BrokerComm::VECTOR +BrokerComm::RECORD *************************** T F @@ -29,7 +29,7 @@ hello 22/tcp 42.0 180.0 -Comm::BOOL +BrokerComm::BOOL *************************** { two, diff --git a/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout b/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout new file mode 100644 index 0000000000..4208503151 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.broker.master_store/bro..stdout @@ -0,0 +1,14 @@ +lookup(two): [status=BrokerStore::SUCCESS, result=[d=broker::data{222}]] +lookup(four): [status=BrokerStore::SUCCESS, result=[d=]] +lookup(myset): [status=BrokerStore::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup(one): [status=BrokerStore::SUCCESS, result=[d=broker::data{111}]] +lookup(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +exists(one): [status=BrokerStore::SUCCESS, result=[d=broker::data{1}]] +exists(two): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] +exists(myset): [status=BrokerStore::SUCCESS, result=[d=broker::data{1}]] +exists(four): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] +pop_right(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{omega}]] +pop_left(myvec): [status=BrokerStore::SUCCESS, result=[d=broker::data{delta}]] +keys: [status=BrokerStore::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] +size: [status=BrokerStore::SUCCESS, result=[d=broker::data{3}]] +size (after clear): [status=BrokerStore::SUCCESS, result=[d=broker::data{0}]] diff --git a/testing/btest/Baseline/core.leaks.comm.remote_event/recv.recv.out b/testing/btest/Baseline/core.leaks.broker.remote_event/recv.recv.out similarity index 100% rename from testing/btest/Baseline/core.leaks.comm.remote_event/recv.recv.out rename to testing/btest/Baseline/core.leaks.broker.remote_event/recv.recv.out diff --git a/testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out b/testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out similarity index 79% rename from testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out rename to testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out index 0e529e08fc..a29c1ecd1e 100644 --- a/testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out +++ b/testing/btest/Baseline/core.leaks.broker.remote_event/send.send.out @@ -1,4 +1,4 @@ -Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp +BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp got event msg, pong, 0 got auto event msg, ping, 0 got event msg, pong, 1 diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/recv.recv.out b/testing/btest/Baseline/core.leaks.broker.remote_log/recv.recv.out similarity index 100% rename from testing/btest/Baseline/core.leaks.comm.remote_log/recv.recv.out rename to testing/btest/Baseline/core.leaks.broker.remote_log/recv.recv.out diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/recv.test.log b/testing/btest/Baseline/core.leaks.broker.remote_log/recv.test.log similarity index 100% rename from testing/btest/Baseline/core.leaks.comm.remote_log/recv.test.log rename to testing/btest/Baseline/core.leaks.broker.remote_log/recv.test.log diff --git a/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out b/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out new file mode 100644 index 0000000000..d97ef33af1 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.broker.remote_log/send.send.out @@ -0,0 +1 @@ +BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/send.test.log b/testing/btest/Baseline/core.leaks.broker.remote_log/send.test.log similarity index 100% rename from testing/btest/Baseline/core.leaks.comm.remote_log/send.test.log rename to testing/btest/Baseline/core.leaks.broker.remote_log/send.test.log diff --git a/testing/btest/Baseline/core.leaks.comm.remote_print/recv.recv.out b/testing/btest/Baseline/core.leaks.broker.remote_print/recv.recv.out similarity index 100% rename from testing/btest/Baseline/core.leaks.comm.remote_print/recv.recv.out rename to testing/btest/Baseline/core.leaks.broker.remote_print/recv.recv.out diff --git a/testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out b/testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out similarity index 62% rename from testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out rename to testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out index 777afdc0d2..65d8ee79b7 100644 --- a/testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out +++ b/testing/btest/Baseline/core.leaks.broker.remote_print/send.send.out @@ -1,4 +1,4 @@ -Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp +BrokerComm::outgoing_connection_established, 127.0.0.1, 9999/tcp got print msg, pong 0 got print msg, pong 1 got print msg, pong 2 diff --git a/testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out b/testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out deleted file mode 100644 index 8a7c89a19b..0000000000 --- a/testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out +++ /dev/null @@ -1,5 +0,0 @@ -clone keys, [status=Store::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] -lookup, one, [status=Store::SUCCESS, result=[d=broker::data{111}]] -lookup, two, [status=Store::SUCCESS, result=[d=broker::data{222}]] -lookup, myset, [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]] -lookup, myvec, [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] diff --git a/testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout b/testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout deleted file mode 100644 index defdc9a3e1..0000000000 --- a/testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout +++ /dev/null @@ -1,14 +0,0 @@ -lookup(two): [status=Store::SUCCESS, result=[d=broker::data{222}]] -lookup(four): [status=Store::SUCCESS, result=[d=]] -lookup(myset): [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]] -lookup(one): [status=Store::SUCCESS, result=[d=broker::data{111}]] -lookup(myvec): [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] -exists(one): [status=Store::SUCCESS, result=[d=broker::data{1}]] -exists(two): [status=Store::SUCCESS, result=[d=broker::data{0}]] -exists(myset): [status=Store::SUCCESS, result=[d=broker::data{1}]] -exists(four): [status=Store::SUCCESS, result=[d=broker::data{0}]] -pop_right(myvec): [status=Store::SUCCESS, result=[d=broker::data{omega}]] -pop_left(myvec): [status=Store::SUCCESS, result=[d=broker::data{delta}]] -keys: [status=Store::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] -size: [status=Store::SUCCESS, result=[d=broker::data{3}]] -size (after clear): [status=Store::SUCCESS, result=[d=broker::data{0}]] diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out b/testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out deleted file mode 100644 index e2415290d6..0000000000 --- a/testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out +++ /dev/null @@ -1 +0,0 @@ -Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/core/leaks/broker/clone_store.bro b/testing/btest/core/leaks/broker/clone_store.bro new file mode 100644 index 0000000000..06df81e1d5 --- /dev/null +++ b/testing/btest/core/leaks/broker/clone_store.bro @@ -0,0 +1,113 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leak + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run clone "bro -m -b ../clone.bro broker_port=$BROKER_PORT >clone.out" +# @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out" + +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone/clone.out + +@TEST-START-FILE clone.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of BrokerStore::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout"; } + } + +event ready() + { + h = BrokerStore::create_clone("mystore"); + + when ( local res = BrokerStore::keys(h) ) + { + print "clone keys", res; + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0))); + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1))); + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2))); + do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + BrokerComm::enable(); + BrokerComm::listen(broker_port, "127.0.0.1"); + BrokerComm::subscribe_to_events("bro/event/ready"); + } + +@TEST-END-FILE + +@TEST-START-FILE master.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of BrokerStore::Handle; + +function dv(d: BrokerComm::Data): BrokerComm::DataVector + { + local rval: BrokerComm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event BrokerComm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event BrokerComm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); + BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); + BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); + BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); + BrokerStore::increment(h, BrokerComm::data("one")); + BrokerStore::decrement(h, BrokerComm::data("two")); + BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); + BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); + BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); + BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); + + when ( local res = BrokerStore::size(h) ) + { event ready(); } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + BrokerComm::enable(); + h = BrokerStore::create_master("mystore"); + BrokerComm::connect("127.0.0.1", broker_port, 1secs); + BrokerComm::auto_event("bro/event/ready", ready); + } + +@TEST-END-FILE diff --git a/testing/btest/core/leaks/broker/data.bro b/testing/btest/core/leaks/broker/data.bro new file mode 100644 index 0000000000..d4f6402ae3 --- /dev/null +++ b/testing/btest/core/leaks/broker/data.bro @@ -0,0 +1,233 @@ +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leaks + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: btest-diff bro/.stdout + +type bro_set: set[string]; +type bro_table: table[string] of count; +type bro_vector: vector of string; + +type bro_record : record { + a: string &optional; + b: string &default = "bee"; + c: count; +}; + +function comm_record_to_bro_record_recurse(it: opaque of BrokerComm::RecordIterator, + rval: bro_record, + idx: count): bro_record + { + if ( BrokerComm::record_iterator_last(it) ) + return rval; + + local field_value = BrokerComm::record_iterator_value(it); + + if ( field_value?$d ) + switch ( idx ) { + case 0: + rval$a = BrokerComm::refine_to_string(field_value); + break; + case 1: + rval$b = BrokerComm::refine_to_string(field_value); + break; + case 2: + rval$c = BrokerComm::refine_to_count(field_value); + break; + }; + + ++idx; + BrokerComm::record_iterator_next(it); + return comm_record_to_bro_record_recurse(it, rval, idx); + } + +function comm_record_to_bro_record(d: BrokerComm::Data): bro_record + { + return comm_record_to_bro_record_recurse(BrokerComm::record_iterator(d), + bro_record($c = 0), 0); + } + +function +comm_set_to_bro_set_recurse(it: opaque of BrokerComm::SetIterator, + rval: bro_set): bro_set + { + if ( BrokerComm::set_iterator_last(it) ) + return rval; + + add rval[BrokerComm::refine_to_string(BrokerComm::set_iterator_value(it))]; + BrokerComm::set_iterator_next(it); + return comm_set_to_bro_set_recurse(it, rval); + } + + +function comm_set_to_bro_set(d: BrokerComm::Data): bro_set + { + return comm_set_to_bro_set_recurse(BrokerComm::set_iterator(d), bro_set()); + } + +function +comm_table_to_bro_table_recurse(it: opaque of BrokerComm::TableIterator, + rval: bro_table): bro_table + { + if ( BrokerComm::table_iterator_last(it) ) + return rval; + + local item = BrokerComm::table_iterator_value(it); + rval[BrokerComm::refine_to_string(item$key)] = BrokerComm::refine_to_count(item$val); + BrokerComm::table_iterator_next(it); + return comm_table_to_bro_table_recurse(it, rval); + } + +function comm_table_to_bro_table(d: BrokerComm::Data): bro_table + { + return comm_table_to_bro_table_recurse(BrokerComm::table_iterator(d), + bro_table()); + } + +function comm_vector_to_bro_vector_recurse(it: opaque of BrokerComm::VectorIterator, + rval: bro_vector): bro_vector + { + if ( BrokerComm::vector_iterator_last(it) ) + return rval; + + rval[|rval|] = BrokerComm::refine_to_string(BrokerComm::vector_iterator_value(it)); + BrokerComm::vector_iterator_next(it); + return comm_vector_to_bro_vector_recurse(it, rval); + } + +function comm_vector_to_bro_vector(d: BrokerComm::Data): bro_vector + { + return comm_vector_to_bro_vector_recurse(BrokerComm::vector_iterator(d), + bro_vector()); + } + +event bro_init() + { +BrokerComm::enable(); + } + +global did_it = F; + +event new_connection(c: connection) + { +if ( did_it ) return; +did_it = T; +print BrokerComm::data_type(BrokerComm::data(T)); +print BrokerComm::data_type(BrokerComm::data(+1)); +print BrokerComm::data_type(BrokerComm::data(1)); +print BrokerComm::data_type(BrokerComm::data(1.1)); +print BrokerComm::data_type(BrokerComm::data("1 (how creative)")); +print BrokerComm::data_type(BrokerComm::data(1.1.1.1)); +print BrokerComm::data_type(BrokerComm::data(1.1.1.1/1)); +print BrokerComm::data_type(BrokerComm::data(1/udp)); +print BrokerComm::data_type(BrokerComm::data(double_to_time(1))); +print BrokerComm::data_type(BrokerComm::data(1sec)); +print BrokerComm::data_type(BrokerComm::data(BrokerComm::BOOL)); +local s: bro_set = bro_set("one", "two", "three"); +local t: bro_table = bro_table(["one"] = 1, ["two"] = 2, ["three"] = 3); +local v: bro_vector = bro_vector("zero", "one", "two"); +local r: bro_record = bro_record($c = 1); +print BrokerComm::data_type(BrokerComm::data(s)); +print BrokerComm::data_type(BrokerComm::data(t)); +print BrokerComm::data_type(BrokerComm::data(v)); +print BrokerComm::data_type(BrokerComm::data(r)); + +print "***************************"; + +print BrokerComm::refine_to_bool(BrokerComm::data(T)); +print BrokerComm::refine_to_bool(BrokerComm::data(F)); +print BrokerComm::refine_to_int(BrokerComm::data(+1)); +print BrokerComm::refine_to_int(BrokerComm::data(+0)); +print BrokerComm::refine_to_int(BrokerComm::data(-1)); +print BrokerComm::refine_to_count(BrokerComm::data(1)); +print BrokerComm::refine_to_count(BrokerComm::data(0)); +print BrokerComm::refine_to_double(BrokerComm::data(1.1)); +print BrokerComm::refine_to_double(BrokerComm::data(-11.1)); +print BrokerComm::refine_to_string(BrokerComm::data("hello")); +print BrokerComm::refine_to_addr(BrokerComm::data(1.2.3.4)); +print BrokerComm::refine_to_subnet(BrokerComm::data(192.168.1.1/16)); +print BrokerComm::refine_to_port(BrokerComm::data(22/tcp)); +print BrokerComm::refine_to_time(BrokerComm::data(double_to_time(42))); +print BrokerComm::refine_to_interval(BrokerComm::data(3min)); +print BrokerComm::refine_to_enum_name(BrokerComm::data(BrokerComm::BOOL)); + +print "***************************"; + +local cs = BrokerComm::data(s); +print comm_set_to_bro_set(cs); +cs = BrokerComm::set_create(); +print BrokerComm::set_size(cs); +print BrokerComm::set_insert(cs, BrokerComm::data("hi")); +print BrokerComm::set_size(cs); +print BrokerComm::set_contains(cs, BrokerComm::data("hi")); +print BrokerComm::set_contains(cs, BrokerComm::data("bye")); +print BrokerComm::set_insert(cs, BrokerComm::data("bye")); +print BrokerComm::set_size(cs); +print BrokerComm::set_remove(cs, BrokerComm::data("hi")); +print BrokerComm::set_size(cs); +print BrokerComm::set_remove(cs, BrokerComm::data("hi")); +print comm_set_to_bro_set(cs); +BrokerComm::set_clear(cs); +print BrokerComm::set_size(cs); + +print "***************************"; + +local ct = BrokerComm::data(t); +print comm_table_to_bro_table(ct); +ct = BrokerComm::table_create(); +print BrokerComm::table_size(ct); +print BrokerComm::table_insert(ct, BrokerComm::data("hi"), BrokerComm::data(42)); +print BrokerComm::table_size(ct); +print BrokerComm::table_contains(ct, BrokerComm::data("hi")); +print BrokerComm::refine_to_count(BrokerComm::table_lookup(ct, BrokerComm::data("hi"))); +print BrokerComm::table_contains(ct, BrokerComm::data("bye")); +print BrokerComm::table_insert(ct, BrokerComm::data("bye"), BrokerComm::data(7)); +print BrokerComm::table_size(ct); +print BrokerComm::table_insert(ct, BrokerComm::data("bye"), BrokerComm::data(37)); +print BrokerComm::table_size(ct); +print BrokerComm::refine_to_count(BrokerComm::table_lookup(ct, BrokerComm::data("bye"))); +print BrokerComm::table_remove(ct, BrokerComm::data("hi")); +print BrokerComm::table_size(ct); + +print "***************************"; + +local cv = BrokerComm::data(v); +print comm_vector_to_bro_vector(cv); +cv = BrokerComm::vector_create(); +print BrokerComm::vector_size(cv); +print BrokerComm::vector_insert(cv, BrokerComm::data("hi"), 0); +print BrokerComm::vector_insert(cv, BrokerComm::data("hello"), 1); +print BrokerComm::vector_insert(cv, BrokerComm::data("greetings"), 2); +print BrokerComm::vector_insert(cv, BrokerComm::data("salutations"), 1); +print comm_vector_to_bro_vector(cv); +print BrokerComm::vector_size(cv); +print BrokerComm::vector_replace(cv, BrokerComm::data("bah"), 2); +print BrokerComm::vector_lookup(cv, 2); +print BrokerComm::vector_lookup(cv, 0); +print comm_vector_to_bro_vector(cv); +print BrokerComm::vector_remove(cv, 2); +print comm_vector_to_bro_vector(cv); +print BrokerComm::vector_size(cv); + +print "***************************"; + +local cr = BrokerComm::data(r); +print comm_record_to_bro_record(cr); +r$a = "test"; +cr = BrokerComm::data(r); +print comm_record_to_bro_record(cr); +r$b = "testagain"; +cr = BrokerComm::data(r); +print comm_record_to_bro_record(cr); +cr = BrokerComm::record_create(3); +print BrokerComm::record_size(cr); +print BrokerComm::record_assign(cr, BrokerComm::data("hi"), 0); +print BrokerComm::record_assign(cr, BrokerComm::data("hello"), 1); +print BrokerComm::record_assign(cr, BrokerComm::data(37), 2); +print BrokerComm::record_lookup(cr, 0); +print BrokerComm::record_lookup(cr, 1); +print BrokerComm::record_lookup(cr, 2); +print BrokerComm::record_size(cr); +} diff --git a/testing/btest/core/leaks/comm/master_store.bro b/testing/btest/core/leaks/broker/master_store.bro similarity index 62% rename from testing/btest/core/leaks/comm/master_store.bro rename to testing/btest/core/leaks/broker/master_store.bro index a5c1063e6f..19c63236f5 100644 --- a/testing/btest/core/leaks/comm/master_store.bro +++ b/testing/btest/core/leaks/broker/master_store.bro @@ -8,7 +8,7 @@ redef exit_only_after_terminate = T; -global h: opaque of Store::Handle; +global h: opaque of BrokerStore::Handle; global lookup_count = 0; const lookup_expect_count = 5; global exists_count = 0; @@ -20,13 +20,13 @@ global test_size: event(where: string &default = ""); event test_clear() { - Store::clear(h); + BrokerStore::clear(h); event test_size("after clear"); } event test_size(where: string) { - when ( local res = Store::size(h) ) + when ( local res = BrokerStore::size(h) ) { if ( where == "" ) { @@ -45,7 +45,7 @@ event test_size(where: string) event test_keys() { - when ( local res = Store::keys(h) ) + when ( local res = BrokerStore::keys(h) ) { print fmt("keys: %s", res); event test_size(); @@ -56,7 +56,7 @@ event test_keys() event test_pop(key: string) { - when ( local lres = Store::pop_left(h, Comm::data(key)) ) + when ( local lres = BrokerStore::pop_left(h, BrokerComm::data(key)) ) { print fmt("pop_left(%s): %s", key, lres); ++pop_count; @@ -67,7 +67,7 @@ event test_pop(key: string) timeout 10sec { print "timeout"; } - when ( local rres = Store::pop_right(h, Comm::data(key)) ) + when ( local rres = BrokerStore::pop_right(h, BrokerComm::data(key)) ) { print fmt("pop_right(%s): %s", key, rres); ++pop_count; @@ -81,7 +81,7 @@ event test_pop(key: string) function do_exists(key: string) { - when ( local res = Store::exists(h, Comm::data(key)) ) + when ( local res = BrokerStore::exists(h, BrokerComm::data(key)) ) { print fmt("exists(%s): %s", key, res); ++exists_count; @@ -95,7 +95,7 @@ function do_exists(key: string) event test_erase() { - Store::erase(h, Comm::data("two")); + BrokerStore::erase(h, BrokerComm::data("two")); do_exists("one"); do_exists("two"); do_exists("myset"); @@ -104,7 +104,7 @@ event test_erase() function do_lookup(key: string) { - when ( local res = Store::lookup(h, Comm::data(key)) ) + when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) ) { print fmt("lookup(%s): %s", key, res); ++lookup_count; @@ -116,9 +116,9 @@ function do_lookup(key: string) { print "timeout"; } } -function dv(d: Comm::Data): Comm::DataVector +function dv(d: BrokerComm::Data): BrokerComm::DataVector { - local rval: Comm::DataVector; + local rval: BrokerComm::DataVector; rval[0] = d; return rval; } @@ -127,8 +127,8 @@ global did_it = F; event bro_init() { - Comm::enable(); - h = Store::create_master("master"); + BrokerComm::enable(); + h = BrokerStore::create_master("master"); } event new_connection(c: connection) @@ -137,16 +137,16 @@ event new_connection(c: connection) did_it = T; local myset: set[string] = {"a", "b", "c"}; local myvec: vector of string = {"alpha", "beta", "gamma"}; - Store::insert(h, Comm::data("one"), Comm::data(110)); - Store::insert(h, Comm::data("two"), Comm::data(223)); - Store::insert(h, Comm::data("myset"), Comm::data(myset)); - Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); - Store::increment(h, Comm::data("one")); - Store::decrement(h, Comm::data("two")); - Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); - Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); - Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); - Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110)); + BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223)); + BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset)); + BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec)); + BrokerStore::increment(h, BrokerComm::data("one")); + BrokerStore::decrement(h, BrokerComm::data("two")); + BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d")); + BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b")); + BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta"))); + BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega"))); do_lookup("one"); do_lookup("two"); do_lookup("myset"); diff --git a/testing/btest/core/leaks/comm/remote_event.test b/testing/btest/core/leaks/broker/remote_event.test similarity index 66% rename from testing/btest/core/leaks/comm/remote_event.test rename to testing/btest/core/leaks/broker/remote_event.test index a329b527db..243d3b04d3 100644 --- a/testing/btest/core/leaks/comm/remote_event.test +++ b/testing/btest/core/leaks/broker/remote_event.test @@ -20,10 +20,10 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { - Comm::enable(); - Comm::listen(broker_port, "127.0.0.1"); - Comm::subscribe_to_events("bro/event/"); - Comm::auto_event("bro/event/my_topic", auto_event_handler); + BrokerComm::enable(); + BrokerComm::listen(broker_port, "127.0.0.1"); + BrokerComm::subscribe_to_events("bro/event/"); + BrokerComm::auto_event("bro/event/my_topic", auto_event_handler); } global event_count = 0; @@ -41,8 +41,8 @@ event event_handler(msg: string, n: count) } event auto_event_handler(msg, n); - local args = Comm::event_args(event_handler, "pong", n); - Comm::event("bro/event/my_topic", args); + local args = BrokerComm::event_args(event_handler, "pong", n); + BrokerComm::event("bro/event/my_topic", args); } @TEST-END-FILE @@ -57,24 +57,24 @@ global auto_event_handler: event(msg: string, c: count); event bro_init() { - Comm::enable(); - Comm::subscribe_to_events("bro/event/my_topic"); - Comm::connect("127.0.0.1", broker_port, 1secs); + BrokerComm::enable(); + BrokerComm::subscribe_to_events("bro/event/my_topic"); + BrokerComm::connect("127.0.0.1", broker_port, 1secs); } global event_count = 0; -event Comm::outgoing_connection_established(peer_address: string, +event BrokerComm::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "Comm::outgoing_connection_established", peer_address, peer_port; - local args = Comm::event_args(event_handler, "ping", event_count); - Comm::event("bro/event/hi", args); + print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + local args = BrokerComm::event_args(event_handler, "ping", event_count); + BrokerComm::event("bro/event/hi", args); ++event_count; } -event Comm::outgoing_connection_broken(peer_address: string, +event BrokerComm::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); @@ -83,8 +83,8 @@ event Comm::outgoing_connection_broken(peer_address: string, event event_handler(msg: string, n: count) { print "got event msg", msg, n; - local args = Comm::event_args(event_handler, "ping", event_count); - Comm::event("bro/event/hi", args); + local args = BrokerComm::event_args(event_handler, "ping", event_count); + BrokerComm::event("bro/event/hi", args); ++event_count; } diff --git a/testing/btest/core/leaks/comm/remote_log.test b/testing/btest/core/leaks/broker/remote_log.test similarity index 80% rename from testing/btest/core/leaks/comm/remote_log.test rename to testing/btest/core/leaks/broker/remote_log.test index 6f20bf8cd4..f6c0c41fda 100644 --- a/testing/btest/core/leaks/comm/remote_log.test +++ b/testing/btest/core/leaks/broker/remote_log.test @@ -29,7 +29,7 @@ export { event bro_init() &priority=5 { - Comm::enable(); + BrokerComm::enable(); Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); } @@ -42,8 +42,8 @@ redef exit_only_after_terminate = T; event bro_init() { - Comm::listen(broker_port, "127.0.0.1"); - Comm::subscribe_to_logs("bro/log/"); + BrokerComm::listen(broker_port, "127.0.0.1"); + BrokerComm::subscribe_to_logs("bro/log/"); } event Test::log_test(rec: Test::Info) @@ -63,8 +63,8 @@ redef exit_only_after_terminate = T; event bro_init() { - Comm::enable_remote_logs(Test::LOG); - Comm::connect("127.0.0.1", broker_port, 1secs); + BrokerComm::enable_remote_logs(Test::LOG); + BrokerComm::connect("127.0.0.1", broker_port, 1secs); } global n = 0; @@ -81,15 +81,15 @@ event do_write() } } -event Comm::outgoing_connection_established(peer_address: string, +event BrokerComm::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "Comm::outgoing_connection_established", peer_address, peer_port; + print "BrokerComm::outgoing_connection_established", peer_address, peer_port; event do_write(); } -event Comm::outgoing_connection_broken(peer_address: string, +event BrokerComm::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); diff --git a/testing/btest/core/leaks/comm/remote_print.test b/testing/btest/core/leaks/broker/remote_print.test similarity index 65% rename from testing/btest/core/leaks/comm/remote_print.test rename to testing/btest/core/leaks/broker/remote_print.test index 43fe50b632..e77881c694 100644 --- a/testing/btest/core/leaks/comm/remote_print.test +++ b/testing/btest/core/leaks/broker/remote_print.test @@ -17,16 +17,16 @@ redef exit_only_after_terminate = T; event bro_init() { - Comm::enable(); - Comm::listen(broker_port, "127.0.0.1"); - Comm::subscribe_to_prints("bro/print/"); + BrokerComm::enable(); + BrokerComm::listen(broker_port, "127.0.0.1"); + BrokerComm::subscribe_to_prints("bro/print/"); } global messages_to_recv = 6; global messages_sent = 0; global messages_recv = 0; -event Comm::print_handler(msg: string) +event BrokerComm::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; @@ -37,7 +37,7 @@ event Comm::print_handler(msg: string) return; } - Comm::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + BrokerComm::print("bro/print/my_topic", fmt("pong %d", messages_sent)); ++messages_sent; } @@ -50,35 +50,35 @@ redef exit_only_after_terminate = T; event bro_init() { - Comm::enable(); - Comm::subscribe_to_prints("bro/print/my_topic"); - Comm::connect("127.0.0.1", broker_port, 1secs); + BrokerComm::enable(); + BrokerComm::subscribe_to_prints("bro/print/my_topic"); + BrokerComm::connect("127.0.0.1", broker_port, 1secs); } global messages_sent = 0; global messages_recv = 0; global peer_disconnected = F; -event Comm::outgoing_connection_established(peer_address: string, +event BrokerComm::outgoing_connection_established(peer_address: string, peer_port: port, peer_name: string) { - print "Comm::outgoing_connection_established", peer_address, peer_port; - Comm::print("bro/print/hi", fmt("ping %d", messages_sent)); + print "BrokerComm::outgoing_connection_established", peer_address, peer_port; + BrokerComm::print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } -event Comm::outgoing_connection_broken(peer_address: string, +event BrokerComm::outgoing_connection_broken(peer_address: string, peer_port: port) { terminate(); } -event Comm::print_handler(msg: string) +event BrokerComm::print_handler(msg: string) { ++messages_recv; print "got print msg", msg; - Comm::print("bro/print/hi", fmt("ping %d", messages_sent)); + BrokerComm::print("bro/print/hi", fmt("ping %d", messages_sent)); ++messages_sent; } diff --git a/testing/btest/core/leaks/comm/clone_store.bro b/testing/btest/core/leaks/comm/clone_store.bro deleted file mode 100644 index 2a75bfa62f..0000000000 --- a/testing/btest/core/leaks/comm/clone_store.bro +++ /dev/null @@ -1,113 +0,0 @@ -# @TEST-SERIALIZE: brokercomm -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt -# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks -# @TEST-GROUP: leak - -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run clone "bro -m -b ../clone.bro broker_port=$BROKER_PORT >clone.out" -# @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out" - -# @TEST-EXEC: btest-bg-wait 45 -# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone/clone.out - -@TEST-START-FILE clone.bro - -const broker_port: port &redef; -redef exit_only_after_terminate = T; - -global h: opaque of Store::Handle; -global expected_key_count = 4; -global key_count = 0; - -function do_lookup(key: string) - { - when ( local res = Store::lookup(h, Comm::data(key)) ) - { - ++key_count; - print "lookup", key, res; - - if ( key_count == expected_key_count ) - terminate(); - } - timeout 10sec - { print "timeout"; } - } - -event ready() - { - h = Store::create_clone("mystore"); - - when ( local res = Store::keys(h) ) - { - print "clone keys", res; - do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0))); - do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1))); - do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2))); - do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3))); - } - timeout 10sec - { print "timeout"; } - } - -event bro_init() - { - Comm::enable(); - Comm::listen(broker_port, "127.0.0.1"); - Comm::subscribe_to_events("bro/event/ready"); - } - -@TEST-END-FILE - -@TEST-START-FILE master.bro - -const broker_port: port &redef; -redef exit_only_after_terminate = T; - -global h: opaque of Store::Handle; - -function dv(d: Comm::Data): Comm::DataVector - { - local rval: Comm::DataVector; - rval[0] = d; - return rval; - } - -global ready: event(); - -event Comm::outgoing_connection_broken(peer_address: string, - peer_port: port) - { - terminate(); - } - -event Comm::outgoing_connection_established(peer_address: string, - peer_port: port, - peer_name: string) - { - local myset: set[string] = {"a", "b", "c"}; - local myvec: vector of string = {"alpha", "beta", "gamma"}; - Store::insert(h, Comm::data("one"), Comm::data(110)); - Store::insert(h, Comm::data("two"), Comm::data(223)); - Store::insert(h, Comm::data("myset"), Comm::data(myset)); - Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); - Store::increment(h, Comm::data("one")); - Store::decrement(h, Comm::data("two")); - Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); - Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); - Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); - Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); - - when ( local res = Store::size(h) ) - { event ready(); } - timeout 10sec - { print "timeout"; } - } - -event bro_init() - { - Comm::enable(); - h = Store::create_master("mystore"); - Comm::connect("127.0.0.1", broker_port, 1secs); - Comm::auto_event("bro/event/ready", ready); - } - -@TEST-END-FILE diff --git a/testing/btest/core/leaks/comm/data.bro b/testing/btest/core/leaks/comm/data.bro deleted file mode 100644 index bf614a2092..0000000000 --- a/testing/btest/core/leaks/comm/data.bro +++ /dev/null @@ -1,233 +0,0 @@ -# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt -# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks -# @TEST-GROUP: leaks - -# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/http/get.trace %INPUT -# @TEST-EXEC: btest-bg-wait 45 -# @TEST-EXEC: btest-diff bro/.stdout - -type bro_set: set[string]; -type bro_table: table[string] of count; -type bro_vector: vector of string; - -type bro_record : record { - a: string &optional; - b: string &default = "bee"; - c: count; -}; - -function comm_record_to_bro_record_recurse(it: opaque of Comm::RecordIterator, - rval: bro_record, - idx: count): bro_record - { - if ( Comm::record_iterator_last(it) ) - return rval; - - local field_value = Comm::record_iterator_value(it); - - if ( field_value?$d ) - switch ( idx ) { - case 0: - rval$a = Comm::refine_to_string(field_value); - break; - case 1: - rval$b = Comm::refine_to_string(field_value); - break; - case 2: - rval$c = Comm::refine_to_count(field_value); - break; - }; - - ++idx; - Comm::record_iterator_next(it); - return comm_record_to_bro_record_recurse(it, rval, idx); - } - -function comm_record_to_bro_record(d: Comm::Data): bro_record - { - return comm_record_to_bro_record_recurse(Comm::record_iterator(d), - bro_record($c = 0), 0); - } - -function -comm_set_to_bro_set_recurse(it: opaque of Comm::SetIterator, - rval: bro_set): bro_set - { - if ( Comm::set_iterator_last(it) ) - return rval; - - add rval[Comm::refine_to_string(Comm::set_iterator_value(it))]; - Comm::set_iterator_next(it); - return comm_set_to_bro_set_recurse(it, rval); - } - - -function comm_set_to_bro_set(d: Comm::Data): bro_set - { - return comm_set_to_bro_set_recurse(Comm::set_iterator(d), bro_set()); - } - -function -comm_table_to_bro_table_recurse(it: opaque of Comm::TableIterator, - rval: bro_table): bro_table - { - if ( Comm::table_iterator_last(it) ) - return rval; - - local item = Comm::table_iterator_value(it); - rval[Comm::refine_to_string(item$key)] = Comm::refine_to_count(item$val); - Comm::table_iterator_next(it); - return comm_table_to_bro_table_recurse(it, rval); - } - -function comm_table_to_bro_table(d: Comm::Data): bro_table - { - return comm_table_to_bro_table_recurse(Comm::table_iterator(d), - bro_table()); - } - -function comm_vector_to_bro_vector_recurse(it: opaque of Comm::VectorIterator, - rval: bro_vector): bro_vector - { - if ( Comm::vector_iterator_last(it) ) - return rval; - - rval[|rval|] = Comm::refine_to_string(Comm::vector_iterator_value(it)); - Comm::vector_iterator_next(it); - return comm_vector_to_bro_vector_recurse(it, rval); - } - -function comm_vector_to_bro_vector(d: Comm::Data): bro_vector - { - return comm_vector_to_bro_vector_recurse(Comm::vector_iterator(d), - bro_vector()); - } - -event bro_init() - { -Comm::enable(); - } - -global did_it = F; - -event new_connection(c: connection) - { -if ( did_it ) return; -did_it = T; -print Comm::data_type(Comm::data(T)); -print Comm::data_type(Comm::data(+1)); -print Comm::data_type(Comm::data(1)); -print Comm::data_type(Comm::data(1.1)); -print Comm::data_type(Comm::data("1 (how creative)")); -print Comm::data_type(Comm::data(1.1.1.1)); -print Comm::data_type(Comm::data(1.1.1.1/1)); -print Comm::data_type(Comm::data(1/udp)); -print Comm::data_type(Comm::data(double_to_time(1))); -print Comm::data_type(Comm::data(1sec)); -print Comm::data_type(Comm::data(Comm::BOOL)); -local s: bro_set = bro_set("one", "two", "three"); -local t: bro_table = bro_table(["one"] = 1, ["two"] = 2, ["three"] = 3); -local v: bro_vector = bro_vector("zero", "one", "two"); -local r: bro_record = bro_record($c = 1); -print Comm::data_type(Comm::data(s)); -print Comm::data_type(Comm::data(t)); -print Comm::data_type(Comm::data(v)); -print Comm::data_type(Comm::data(r)); - -print "***************************"; - -print Comm::refine_to_bool(Comm::data(T)); -print Comm::refine_to_bool(Comm::data(F)); -print Comm::refine_to_int(Comm::data(+1)); -print Comm::refine_to_int(Comm::data(+0)); -print Comm::refine_to_int(Comm::data(-1)); -print Comm::refine_to_count(Comm::data(1)); -print Comm::refine_to_count(Comm::data(0)); -print Comm::refine_to_double(Comm::data(1.1)); -print Comm::refine_to_double(Comm::data(-11.1)); -print Comm::refine_to_string(Comm::data("hello")); -print Comm::refine_to_addr(Comm::data(1.2.3.4)); -print Comm::refine_to_subnet(Comm::data(192.168.1.1/16)); -print Comm::refine_to_port(Comm::data(22/tcp)); -print Comm::refine_to_time(Comm::data(double_to_time(42))); -print Comm::refine_to_interval(Comm::data(3min)); -print Comm::refine_to_enum_name(Comm::data(Comm::BOOL)); - -print "***************************"; - -local cs = Comm::data(s); -print comm_set_to_bro_set(cs); -cs = Comm::set_create(); -print Comm::set_size(cs); -print Comm::set_insert(cs, Comm::data("hi")); -print Comm::set_size(cs); -print Comm::set_contains(cs, Comm::data("hi")); -print Comm::set_contains(cs, Comm::data("bye")); -print Comm::set_insert(cs, Comm::data("bye")); -print Comm::set_size(cs); -print Comm::set_remove(cs, Comm::data("hi")); -print Comm::set_size(cs); -print Comm::set_remove(cs, Comm::data("hi")); -print comm_set_to_bro_set(cs); -Comm::set_clear(cs); -print Comm::set_size(cs); - -print "***************************"; - -local ct = Comm::data(t); -print comm_table_to_bro_table(ct); -ct = Comm::table_create(); -print Comm::table_size(ct); -print Comm::table_insert(ct, Comm::data("hi"), Comm::data(42)); -print Comm::table_size(ct); -print Comm::table_contains(ct, Comm::data("hi")); -print Comm::refine_to_count(Comm::table_lookup(ct, Comm::data("hi"))); -print Comm::table_contains(ct, Comm::data("bye")); -print Comm::table_insert(ct, Comm::data("bye"), Comm::data(7)); -print Comm::table_size(ct); -print Comm::table_insert(ct, Comm::data("bye"), Comm::data(37)); -print Comm::table_size(ct); -print Comm::refine_to_count(Comm::table_lookup(ct, Comm::data("bye"))); -print Comm::table_remove(ct, Comm::data("hi")); -print Comm::table_size(ct); - -print "***************************"; - -local cv = Comm::data(v); -print comm_vector_to_bro_vector(cv); -cv = Comm::vector_create(); -print Comm::vector_size(cv); -print Comm::vector_insert(cv, Comm::data("hi"), 0); -print Comm::vector_insert(cv, Comm::data("hello"), 1); -print Comm::vector_insert(cv, Comm::data("greetings"), 2); -print Comm::vector_insert(cv, Comm::data("salutations"), 1); -print comm_vector_to_bro_vector(cv); -print Comm::vector_size(cv); -print Comm::vector_replace(cv, Comm::data("bah"), 2); -print Comm::vector_lookup(cv, 2); -print Comm::vector_lookup(cv, 0); -print comm_vector_to_bro_vector(cv); -print Comm::vector_remove(cv, 2); -print comm_vector_to_bro_vector(cv); -print Comm::vector_size(cv); - -print "***************************"; - -local cr = Comm::data(r); -print comm_record_to_bro_record(cr); -r$a = "test"; -cr = Comm::data(r); -print comm_record_to_bro_record(cr); -r$b = "testagain"; -cr = Comm::data(r); -print comm_record_to_bro_record(cr); -cr = Comm::record_create(3); -print Comm::record_size(cr); -print Comm::record_assign(cr, Comm::data("hi"), 0); -print Comm::record_assign(cr, Comm::data("hello"), 1); -print Comm::record_assign(cr, Comm::data(37), 2); -print Comm::record_lookup(cr, 0); -print Comm::record_lookup(cr, 1); -print Comm::record_lookup(cr, 2); -print Comm::record_size(cr); -} From 51010eccd4d2746571ae95979a6058d8ee29b3da Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Mar 2015 13:00:29 -0500 Subject: [PATCH 10/22] Add Connection class getter methods for flow labels. BIT-1309 #close --- src/Conn.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Conn.h b/src/Conn.h index 966c77a9f8..20e60d2617 100644 --- a/src/Conn.h +++ b/src/Conn.h @@ -263,6 +263,9 @@ public: void CheckFlowLabel(bool is_orig, uint32 flow_label); + uint32 GetOrigFlowLabel() { return orig_flow_label; } + uint32 GetRespFlowLabel() { return resp_flow_label; } + protected: Connection() { persistent = 0; } From 6fbceb6a987ebbee0321712217f7c4e4d5e52a48 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Mar 2015 13:01:57 -0500 Subject: [PATCH 11/22] Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP. BIT-1311 #close --- NEWS | 3 +++ src/Sessions.cc | 6 +++++- src/TunnelEncapsulation.h | 9 ++++++--- src/types.bif | 1 + .../btest/Baseline/core.tunnels.gre-in-gre/tunnel.log | 4 ++-- testing/btest/Baseline/core.tunnels.gre/tunnel.log | 2 +- 6 files changed, 18 insertions(+), 7 deletions(-) diff --git a/NEWS b/NEWS index 50e5ddd265..981af20370 100644 --- a/NEWS +++ b/NEWS @@ -94,6 +94,9 @@ Changed Functionality - conn.log gained a new field local_resp that works like local_orig, just for the responder address of the connection. +- GRE tunnels are now identified as ``Tunnel::GRE`` instead of + ``Tunnel::IP``. + - [TODO] Add changed BroControl features. Deprecated Functionality diff --git a/src/Sessions.cc b/src/Sessions.cc index ffc2baf944..086216e93d 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -466,6 +466,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, id.src_addr = ip_hdr->SrcAddr(); id.dst_addr = ip_hdr->DstAddr(); Dictionary* d = 0; + BifEnum::Tunnel::Type tunnel_type = BifEnum::Tunnel::IP; switch ( proto ) { case IPPROTO_TCP: @@ -606,6 +607,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, // Treat GRE tunnel like IP tunnels, fallthrough to logic below now // that GRE header is stripped and only payload packet remains. + // The only thing different is the tunnel type enum value to use. + tunnel_type = BifEnum::Tunnel::GRE; } case IPPROTO_IPV4: @@ -653,7 +656,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( it == ip_tunnels.end() ) { - EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr()); + EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr(), + tunnel_type); ip_tunnels[tunnel_idx] = TunnelActivity(ec, network_time); timer_mgr->Add(new IPTunnelTimer(network_time, tunnel_idx)); } diff --git a/src/TunnelEncapsulation.h b/src/TunnelEncapsulation.h index 23f8966ee7..419a3000b4 100644 --- a/src/TunnelEncapsulation.h +++ b/src/TunnelEncapsulation.h @@ -37,10 +37,12 @@ public: * * @param s The tunnel source address, likely taken from an IP header. * @param d The tunnel destination address, likely taken from an IP header. + * @param t The type of IP tunnel. */ - EncapsulatingConn(const IPAddr& s, const IPAddr& d) + EncapsulatingConn(const IPAddr& s, const IPAddr& d, + BifEnum::Tunnel::Type t = BifEnum::Tunnel::IP) : src_addr(s), dst_addr(d), src_port(0), dst_port(0), - proto(TRANSPORT_UNKNOWN), type(BifEnum::Tunnel::IP), + proto(TRANSPORT_UNKNOWN), type(t), uid(Bro::UID(bits_per_uid)) { } @@ -85,7 +87,8 @@ public: if ( ec1.type != ec2.type ) return false; - if ( ec1.type == BifEnum::Tunnel::IP ) + if ( ec1.type == BifEnum::Tunnel::IP || + ec1.type == BifEnum::Tunnel::GRE ) // Reversing endpoints is still same tunnel. return ec1.uid == ec2.uid && ec1.proto == ec2.proto && ((ec1.src_addr == ec2.src_addr && ec1.dst_addr == ec2.dst_addr) || diff --git a/src/types.bif b/src/types.bif index 99df67c9d5..73443a3fd7 100644 --- a/src/types.bif +++ b/src/types.bif @@ -172,6 +172,7 @@ enum Type %{ SOCKS, GTPv1, HTTP, + GRE, %} type EncapsulatingConn: record; diff --git a/testing/btest/Baseline/core.tunnels.gre-in-gre/tunnel.log b/testing/btest/Baseline/core.tunnels.gre-in-gre/tunnel.log index 277d1df679..ad7154d756 100644 --- a/testing/btest/Baseline/core.tunnels.gre-in-gre/tunnel.log +++ b/testing/btest/Baseline/core.tunnels.gre-in-gre/tunnel.log @@ -6,6 +6,6 @@ #open 2014-01-16-21-51-36 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action #types time string addr port addr port enum enum -1341436424.204043 CXWv6p3arKYeMETxOg 72.205.54.70 0 86.106.164.150 0 Tunnel::IP Tunnel::DISCOVER -1341436424.204043 CjhGID4nQcgTWjvg4c 10.10.11.2 0 10.10.13.2 0 Tunnel::IP Tunnel::DISCOVER +1341436424.204043 CXWv6p3arKYeMETxOg 72.205.54.70 0 86.106.164.150 0 Tunnel::GRE Tunnel::DISCOVER +1341436424.204043 CjhGID4nQcgTWjvg4c 10.10.11.2 0 10.10.13.2 0 Tunnel::GRE Tunnel::DISCOVER #close 2014-01-16-21-51-36 diff --git a/testing/btest/Baseline/core.tunnels.gre/tunnel.log b/testing/btest/Baseline/core.tunnels.gre/tunnel.log index f0d87f4964..066e1fe151 100644 --- a/testing/btest/Baseline/core.tunnels.gre/tunnel.log +++ b/testing/btest/Baseline/core.tunnels.gre/tunnel.log @@ -6,5 +6,5 @@ #open 2014-01-16-21-51-12 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action #types time string addr port addr port enum enum -1055289968.793044 CXWv6p3arKYeMETxOg 172.27.1.66 0 66.59.109.137 0 Tunnel::IP Tunnel::DISCOVER +1055289968.793044 CXWv6p3arKYeMETxOg 172.27.1.66 0 66.59.109.137 0 Tunnel::GRE Tunnel::DISCOVER #close 2014-01-16-21-51-12 From 0b957cbe752df7773dad0d5549e9654f92e18cf9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Mar 2015 14:16:04 -0500 Subject: [PATCH 12/22] Include timestamp in default extracted file names. And add a policy script to extract all files. BIT-1335 #close --- CHANGES | 11 +++++++++ NEWS | 3 +++ VERSION | 2 +- scripts/base/files/extract/main.bro | 3 ++- .../frameworks/files/extract-all-files.bro | 8 +++++++ scripts/test-all-policy.bro | 1 + testing/btest/Baseline/plugins.hooks/output | 24 +++++++++---------- .../policy/frameworks/files/extract-all.bro | 2 ++ 8 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 scripts/policy/frameworks/files/extract-all-files.bro create mode 100644 testing/btest/scripts/policy/frameworks/files/extract-all.bro diff --git a/CHANGES b/CHANGES index 926b30c9c0..84f64034ea 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,15 @@ +2.3-539 | 2015-03-13 14:19:27 -0500 + + * BIT-1335: Include timestamp in default extracted file names. + And add a policy script to extract all files. (Jon Siwek) + + * BIT-1311: Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP. + (Jon Siwek) + + * BIT-1309: Add Connection class getter methods for flow labels. + (Jon Siwek) + 2.3-536 | 2015-03-12 16:16:24 -0500 * Fix Broker leak tests. (Jon Siwek) diff --git a/NEWS b/NEWS index 981af20370..4d1539b33c 100644 --- a/NEWS +++ b/NEWS @@ -97,6 +97,9 @@ Changed Functionality - GRE tunnels are now identified as ``Tunnel::GRE`` instead of ``Tunnel::IP``. +- The default name for extracted files changed from extract-protocol-id + to extract-timestamp-protocol-id. + - [TODO] Add changed BroControl features. Deprecated Functionality diff --git a/VERSION b/VERSION index c168eac2bd..64cd9fa66f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-536 +2.3-539 diff --git a/scripts/base/files/extract/main.bro b/scripts/base/files/extract/main.bro index 765263a4d8..7f68a8bcce 100644 --- a/scripts/base/files/extract/main.bro +++ b/scripts/base/files/extract/main.bro @@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool function on_add(f: fa_file, args: Files::AnalyzerArgs) { if ( ! args?$extract_filename ) - args$extract_filename = cat("extract-", f$source, "-", f$id); + args$extract_filename = cat("extract-", f$last_active, "-", f$source, + "-", f$id); f$info$extracted = args$extract_filename; args$extract_filename = build_path_compressed(prefix, args$extract_filename); diff --git a/scripts/policy/frameworks/files/extract-all-files.bro b/scripts/policy/frameworks/files/extract-all-files.bro new file mode 100644 index 0000000000..7bd7b300e9 --- /dev/null +++ b/scripts/policy/frameworks/files/extract-all-files.bro @@ -0,0 +1,8 @@ +##! Extract all files to disk. + +@load base/files/extract + +event file_new(f: fa_file) + { + Files::add_analyzer(f, Files::ANALYZER_EXTRACT); + } diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 0fb74f91cf..dc85986172 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -28,6 +28,7 @@ @load frameworks/intel/seen/where-locations.bro @load frameworks/intel/seen/x509.bro @load frameworks/files/detect-MHR.bro +#@load frameworks/files/extract-all-files.bro @load frameworks/files/hash-all-files.bro @load frameworks/packet-filter/shunt.bro @load frameworks/software/version-changes.bro diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 6956f013bc..63f0a87742 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -124,7 +124,7 @@ 0.000000 MetaHookPost CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> 0.000000 MetaHookPost CallFunction(Cluster::is_enabled, , ()) -> -0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> +0.000000 MetaHookPost CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])) -> 0.000000 MetaHookPost CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])) -> @@ -192,7 +192,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -286,8 +286,8 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, , ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) -> @@ -669,7 +669,7 @@ 0.000000 MetaHookPre CallFunction(Analyzer::register_for_ports, , (Analyzer::ANALYZER_TEREDO, {3544/udp})) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) 0.000000 MetaHookPre CallFunction(Cluster::is_enabled, , ()) -0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) +0.000000 MetaHookPre CallFunction(Files::register_analyzer_add_callback, , (Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)})) 0.000000 MetaHookPre CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}])) 0.000000 MetaHookPre CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}])) 0.000000 MetaHookPre CallFunction(Files::register_protocol, , (Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}])) @@ -737,7 +737,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -831,8 +831,8 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::default_path_func, , (PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, , ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, , (ip or not ip, and, )) @@ -1213,7 +1213,7 @@ 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_SYSLOG, {514/udp}) 0.000000 | HookCallFunction Analyzer::register_for_ports(Analyzer::ANALYZER_TEREDO, {3544/udp}) 0.000000 | HookCallFunction Cluster::is_enabled() -0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}) +0.000000 | HookCallFunction Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, FileExtract::on_add{ if (!FileExtract::args?$extract_filename) FileExtract::args$extract_filename = cat(extract-, FileExtract::f$last_active, -, FileExtract::f$source, -, FileExtract::f$id)FileExtract::f$info$extracted = FileExtract::args$extract_filenameFileExtract::args$extract_filename = build_path_compressed(FileExtract::prefix, FileExtract::args$extract_filename)mkdir(FileExtract::prefix)}) 0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_FTP_DATA, [get_file_handle=FTP::get_file_handle{ if (!FTP::c$id$resp_h, FTP::c$id$resp_p in FTP::ftp_data_expected) return ()return (cat(Analyzer::ANALYZER_FTP_DATA, FTP::c$start_time, FTP::c$id, FTP::is_orig))}, describe=FTP::describe_file{ FTP::cid{ if (FTP::f$source != FTP) return ()for ([FTP::cid] in FTP::f$conns) { if (FTP::f$conns[FTP::cid]?$ftp) return (FTP::describe(FTP::f$conns[FTP::cid]$ftp))}return ()}}]) 0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_HTTP, [get_file_handle=HTTP::get_file_handle{ if (!HTTP::c?$http) return ()if (HTTP::c$http$range_request && !HTTP::is_orig) { return (cat(Analyzer::ANALYZER_HTTP, HTTP::is_orig, HTTP::c$id$orig_h, HTTP::build_url(HTTP::c$http)))}else{ HTTP::mime_depth = HTTP::is_orig ? HTTP::c$http$orig_mime_depth : HTTP::c$http$resp_mime_depthreturn (cat(Analyzer::ANALYZER_HTTP, HTTP::c$start_time, HTTP::is_orig, HTTP::c$http$trans_depth, HTTP::mime_depth, id_string(HTTP::c$id)))}}, describe=HTTP::describe_file{ HTTP::cid{ if (HTTP::f$source != HTTP) return ()for ([HTTP::cid] in HTTP::f$conns) { if (HTTP::f$conns[HTTP::cid]?$http) return (HTTP::build_url_http(HTTP::f$conns[HTTP::cid]$http))}return ()}}]) 0.000000 | HookCallFunction Files::register_protocol(Analyzer::ANALYZER_IRC_DATA, [get_file_handle=IRC::get_file_handle{ return (cat(Analyzer::ANALYZER_IRC_DATA, IRC::c$start_time, IRC::c$id, IRC::is_orig))}, describe=anonymous-function{ return ()}]) @@ -1281,7 +1281,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1375,8 +1375,8 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1425596289.27327, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1426273629.648148, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) diff --git a/testing/btest/scripts/policy/frameworks/files/extract-all.bro b/testing/btest/scripts/policy/frameworks/files/extract-all.bro new file mode 100644 index 0000000000..f54b2e299d --- /dev/null +++ b/testing/btest/scripts/policy/frameworks/files/extract-all.bro @@ -0,0 +1,2 @@ +# @TEST-EXEC: bro -r $TRACES/http/get.trace frameworks/files/extract-all-files +# @TEST-EXEC: grep -q EXTRACT files.log From 46f7d238889af63dedf86f6db45caabb1396386f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Mar 2015 14:53:11 -0500 Subject: [PATCH 13/22] Fix Broxygen coverage. --- scripts/broxygen/__load__.bro | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/broxygen/__load__.bro b/scripts/broxygen/__load__.bro index 8db4a7c1b8..3b78ba8619 100644 --- a/scripts/broxygen/__load__.bro +++ b/scripts/broxygen/__load__.bro @@ -5,6 +5,7 @@ @load frameworks/communication/listen.bro @load frameworks/control/controllee.bro @load frameworks/control/controller.bro +@load frameworks/files/extract-all-files.bro @load policy/misc/dump-events.bro @load ./example.bro From 778b37b5d0cf3612ebbedeb3cb90e9d18fa4adc9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Mar 2015 14:54:46 -0500 Subject: [PATCH 14/22] Deprecate &rotate_interval, &rotate_size, &encrypt, &mergeable. Addresses BIT-1305. --- doc/script-reference/attributes.rst | 7 ------ src/scan.l | 35 ++++++++++++++++++++++++----- 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/doc/script-reference/attributes.rst b/doc/script-reference/attributes.rst index ef6c6a54a1..40646f64f4 100644 --- a/doc/script-reference/attributes.rst +++ b/doc/script-reference/attributes.rst @@ -43,8 +43,6 @@ The Bro scripting language supports the following attributes. +-----------------------------+-----------------------------------------------+ | :bro:attr:`&mergeable` |Prefer set union for synchronized state. | +-----------------------------+-----------------------------------------------+ -| :bro:attr:`&group` |Group event handlers to activate/deactivate. | -+-----------------------------+-----------------------------------------------+ | :bro:attr:`&error_handler` |Used internally for reporter framework events. | +-----------------------------+-----------------------------------------------+ | :bro:attr:`&type_column` |Used by input framework for "port" type. | @@ -198,11 +196,6 @@ Here is a more detailed explanation of each attribute: inconsistencies and can be avoided by unifying the two sets, rather than merely overwriting the old value. -.. bro:attr:: &group - - Groups event handlers such that those in the same group can be - jointly activated or deactivated. - .. bro:attr:: &error_handler Internally set on the events that are associated with the reporter diff --git a/src/scan.l b/src/scan.l index b13215e4b8..896264581b 100644 --- a/src/scan.l +++ b/src/scan.l @@ -56,6 +56,11 @@ char last_tok[128]; if ( ((result = fread(buf, 1, max_size, yyin)) == 0) && ferror(yyin) ) \ reporter->Error("read failed with \"%s\"", strerror(errno)); +static void deprecated_attr(const char* attr) + { + reporter->Warning("Use of deprecated attribute: %s", attr); + } + static string find_relative_file(const string& filename, const string& ext) { if ( filename.empty() ) @@ -263,20 +268,38 @@ when return TOK_WHEN; &delete_func return TOK_ATTR_DEL_FUNC; &deprecated return TOK_ATTR_DEPRECATED; &raw_output return TOK_ATTR_RAW_OUTPUT; -&encrypt return TOK_ATTR_ENCRYPT; +&encrypt { + deprecated_attr(yytext); + return TOK_ATTR_ENCRYPT; + } &error_handler return TOK_ATTR_ERROR_HANDLER; &expire_func return TOK_ATTR_EXPIRE_FUNC; &log return TOK_ATTR_LOG; -&mergeable return TOK_ATTR_MERGEABLE; +&mergeable { + deprecated_attr(yytext); + return TOK_ATTR_MERGEABLE; + } &optional return TOK_ATTR_OPTIONAL; -&persistent return TOK_ATTR_PERSISTENT; +&persistent { + //deprecated_attr(yytext); + return TOK_ATTR_PERSISTENT; + } &priority return TOK_ATTR_PRIORITY; &type_column return TOK_ATTR_TYPE_COLUMN; &read_expire return TOK_ATTR_EXPIRE_READ; &redef return TOK_ATTR_REDEF; -&rotate_interval return TOK_ATTR_ROTATE_INTERVAL; -&rotate_size return TOK_ATTR_ROTATE_SIZE; -&synchronized return TOK_ATTR_SYNCHRONIZED; +&rotate_interval { + deprecated_attr(yytext); + return TOK_ATTR_ROTATE_INTERVAL; + } +&rotate_size { + deprecated_attr(yytext); + return TOK_ATTR_ROTATE_SIZE; + } +&synchronized { + //deprecated_attr(yytext); + return TOK_ATTR_SYNCHRONIZED; + } &write_expire return TOK_ATTR_EXPIRE_WRITE; @DEBUG return TOK_DEBUG; // marks input for debugger From 5e2defebe5da4ba98b0dce5a1aa3460a999f4c0f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 13 Mar 2015 15:44:08 -0500 Subject: [PATCH 15/22] Make INSTALL a symlink to doc/install/install.rst BIT-1275 #close --- CHANGES | 6 ++++++ INSTALL | 4 +--- VERSION | 2 +- 3 files changed, 8 insertions(+), 4 deletions(-) mode change 100644 => 120000 INSTALL diff --git a/CHANGES b/CHANGES index 84f64034ea..d491a666e8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.3-541 | 2015-03-13 15:44:08 -0500 + + * Make INSTALL a symlink to doc/install/install.rst (Jon siwek) + + * Fix Broxygen coverage. (Jon Siwek) + 2.3-539 | 2015-03-13 14:19:27 -0500 * BIT-1335: Include timestamp in default extracted file names. diff --git a/INSTALL b/INSTALL deleted file mode 100644 index 385dac93df..0000000000 --- a/INSTALL +++ /dev/null @@ -1,3 +0,0 @@ - -See doc/install/install.rst for installation instructions. - diff --git a/INSTALL b/INSTALL new file mode 120000 index 0000000000..95fcc60eda --- /dev/null +++ b/INSTALL @@ -0,0 +1 @@ +doc/install/install.rst \ No newline at end of file diff --git a/VERSION b/VERSION index 64cd9fa66f..711f7a5631 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-539 +2.3-541 From c09411bc8b191574ee7e0f2910ea486586b55455 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 16 Mar 2015 15:12:48 -0500 Subject: [PATCH 16/22] BIT-1077: fix HTTP::log_server_header_names. Before, it just re-logged fields from the client side. --- .../policy/protocols/http/header-names.bro | 27 +++++++++++-------- .../http.log | 23 ++++++++++++++++ .../policy/protocols/http/header-names.bro | 5 ++++ 3 files changed, 44 insertions(+), 11 deletions(-) create mode 100644 testing/btest/Baseline/scripts.policy.protocols.http.header-names/http.log create mode 100644 testing/btest/scripts/policy/protocols/http/header-names.bro diff --git a/scripts/policy/protocols/http/header-names.bro b/scripts/policy/protocols/http/header-names.bro index 5aefdad538..ed3f9380a7 100644 --- a/scripts/policy/protocols/http/header-names.bro +++ b/scripts/policy/protocols/http/header-names.bro @@ -26,20 +26,25 @@ export { event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 { - if ( ! is_orig || ! c?$http ) + if ( ! c?$http ) return; - - if ( log_client_header_names ) + + if ( is_orig ) { - if ( ! c$http?$client_header_names ) - c$http$client_header_names = vector(); - c$http$client_header_names[|c$http$client_header_names|] = name; + if ( log_client_header_names ) + { + if ( ! c$http?$client_header_names ) + c$http$client_header_names = vector(); + c$http$client_header_names[|c$http$client_header_names|] = name; + } } - - if ( log_server_header_names ) + else { - if ( ! c$http?$server_header_names ) - c$http$server_header_names = vector(); - c$http$server_header_names[|c$http$server_header_names|] = name; + if ( log_server_header_names ) + { + if ( ! c$http?$server_header_names ) + c$http$server_header_names = vector(); + c$http$server_header_names[|c$http$server_header_names|] = name; + } } } diff --git a/testing/btest/Baseline/scripts.policy.protocols.http.header-names/http.log b/testing/btest/Baseline/scripts.policy.protocols.http.header-names/http.log new file mode 100644 index 0000000000..ca510300c2 --- /dev/null +++ b/testing/btest/Baseline/scripts.policy.protocols.http.header-names/http.log @@ -0,0 +1,23 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open 2015-03-16-20-10-52 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer user_agent request_body_len response_body_len status_code status_msg info_code info_msg filename tags username password proxied orig_fuids orig_mime_types resp_fuids resp_mime_types client_header_names server_header_names +#types time string addr port addr port count string string string string string count count count string count string string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] +1300475168.784020 CRJuHdVW0XPVINV8a 141.142.220.118 48649 208.80.152.118 80 1 GET bits.wikimedia.org /skins-1.5/monobook/main.css http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,VIA,X-VARNISH,LAST-MODIFIED,ETAG,VARY,CONNECTION +1300475168.916018 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/6/63/Wikipedia-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.916183 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/b/bb/Wikipedia_wordmark.svg/174px-Wikipedia_wordmark.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.918358 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/b/bd/Bookshelf-40x201_6.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.952307 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/8/8a/Wikinews-logo.png/35px-Wikinews-logo.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.952296 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/4/4a/Wiktionary-logo-en-35px.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.954820 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 1 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikiquote-logo.svg/35px-Wikiquote-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.962687 Cn78a440HlxuyZKs6f 141.142.220.118 35642 208.80.152.2 80 1 GET meta.wikimedia.org /images/wikimedia-button.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,EXPIRES,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.975934 CJ3xTn1c4Zw9TmAE05 141.142.220.118 49997 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/f/fa/Wikibooks-logo.svg/35px-Wikibooks-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.976436 C7XEbhP654jzLoe3a 141.142.220.118 49996 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/d/df/Wikispecies-logo.svg/35px-Wikispecies-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475168.979264 C3SfNE4BWaU4aSuwkc 141.142.220.118 49998 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4c/Wikisource-logo.svg/35px-Wikisource-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475169.014619 CyAhVIzHqb7t7kv28 141.142.220.118 50000 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/4/4a/Commons-logo.svg/35px-Commons-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475169.014593 CzA03V1VcgagLjnO92 141.142.220.118 49999 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/9/91/Wikiversity-logo.svg/35px-Wikiversity-logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +1300475169.014927 CkDsfG2YIeWJmXWNWj 141.142.220.118 50001 208.80.152.3 80 2 GET upload.wikimedia.org /wikipedia/commons/thumb/7/75/Wikimedia_Community_Logo.svg/35px-Wikimedia_Community_Logo.svg.png http://www.wikipedia.org/ Mozilla/5.0 (X11; U; Linux x86_64; en-US; rv:1.9.2.15) Gecko/20110303 Ubuntu/10.04 (lucid) Firefox/3.6.15 0 0 304 Not Modified - - - (empty) - - - - - - - HOST,USER-AGENT,ACCEPT,ACCEPT-LANGUAGE,ACCEPT-ENCODING,ACCEPT-CHARSET,KEEP-ALIVE,CONNECTION,REFERER,IF-MODIFIED-SINCE,IF-NONE-MATCH,CACHE-CONTROL DATE,CONTENT-TYPE,LAST-MODIFIED,ETAG,AGE,X-CACHE,X-CACHE-LOOKUP,X-CACHE,X-CACHE-LOOKUP,CONNECTION +#close 2015-03-16-20-10-52 diff --git a/testing/btest/scripts/policy/protocols/http/header-names.bro b/testing/btest/scripts/policy/protocols/http/header-names.bro new file mode 100644 index 0000000000..30b1de7fdb --- /dev/null +++ b/testing/btest/scripts/policy/protocols/http/header-names.bro @@ -0,0 +1,5 @@ +# @TEST-EXEC: bro -r $TRACES/wikipedia.trace %INPUT +# @TEST-EXEC: btest-diff http.log + +@load protocols/http/header-names +redef HTTP::log_server_header_names=T; From 1d40d5c6e93132657c0aaabdfd395cba3846e080 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Mar 2015 09:02:12 -0700 Subject: [PATCH 17/22] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 1a2ab9ee7c..694af9d9ed 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 1a2ab9ee7c80ca905e86a2a11283e7c0477341a9 +Subproject commit 694af9d9edd188a461cc762bfdb7b61688b93ada From 62a3a23a2bbacad4bc3043bf71c0e95f3f106ea0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Mar 2015 09:02:46 -0700 Subject: [PATCH 18/22] Updating submodule(s). [nomail] --- aux/broker | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broker b/aux/broker index 694af9d9ed..1a2ab9ee7c 160000 --- a/aux/broker +++ b/aux/broker @@ -1 +1 @@ -Subproject commit 694af9d9edd188a461cc762bfdb7b61688b93ada +Subproject commit 1a2ab9ee7c80ca905e86a2a11283e7c0477341a9 From e291ccc14a17a2c14afc9ab07d05576d2de95f5f Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 17 Mar 2015 12:51:57 -0700 Subject: [PATCH 19/22] add x509 canonifiers to test to not make it fail on differing openssl versions. --- .../scripts/policy/protocols/ssl/validate-certs-no-cache.bro | 2 +- testing/btest/scripts/policy/protocols/ssl/validate-certs.bro | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro b/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro index 1bca5b5c50..343b2fb196 100644 --- a/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro +++ b/testing/btest/scripts/policy/protocols/ssl/validate-certs-no-cache.bro @@ -1,5 +1,5 @@ # @TEST-EXEC: bro -C -r $TRACES/tls/missing-intermediate.pcap %INPUT -# @TEST-EXEC: btest-diff ssl.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-x509-names btest-diff ssl.log @load protocols/ssl/validate-certs.bro diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro b/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro index 19fca8cb89..40e5e09361 100644 --- a/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro +++ b/testing/btest/scripts/policy/protocols/ssl/validate-certs.bro @@ -2,6 +2,6 @@ # @TEST-EXEC: cat ssl.log > ssl-all.log # @TEST-EXEC: bro -C -r $TRACES/tls/missing-intermediate.pcap %INPUT # @TEST-EXEC: cat ssl.log >> ssl-all.log -# @TEST-EXEC: btest-diff ssl-all.log +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-x509-names btest-diff ssl-all.log @load protocols/ssl/validate-certs.bro From 468e7bbce253428bc2a39dcef36c85fee4e3c2a7 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Mar 2015 15:40:39 -0700 Subject: [PATCH 20/22] Increasing a test timeout to not fail on slower machines. --- CHANGES | 2 +- VERSION | 2 +- .../scripts/policy/protocols/ssl/validate-certs-cluster.bro | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 529e06c643..865052fa1c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,5 +1,5 @@ -2.3-553 | 2015-03-17 14:33:23 -0700 +2.3-554 | 2015-03-17 15:40:39 -0700 * Deprecate &rotate_interval, &rotate_size, &encrypt. Addresses BIT-1305. (Jon Siwek) diff --git a/VERSION b/VERSION index 78bd132a6d..1b240f1fbf 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-553 +2.3-554 diff --git a/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro b/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro index db9c6cd9da..795aa78c40 100644 --- a/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro +++ b/testing/btest/scripts/policy/protocols/ssl/validate-certs-cluster.bro @@ -7,7 +7,7 @@ # @TEST-EXEC: sleep 1 # @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/missing-intermediate.pcap %INPUT" # @TEST-EXEC: btest-bg-run worker-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/missing-intermediate.pcap %INPUT" -# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-bg-wait 20 # @TEST-EXEC: cat manager-1/ssl*.log > ssl.log # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-file-ids btest-diff ssl.log # From d3afe97f83a10f2395621de8b4c0966ad426e904 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 17 Mar 2015 15:52:17 -0700 Subject: [PATCH 21/22] Splitting test-all target into Bro tests and test-aux. Also making failure of one sub-suite non-fatal. --- CHANGES | 5 +++++ Makefile | 14 ++++++++------ VERSION | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/CHANGES b/CHANGES index 865052fa1c..ecc7ae90d6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-555 | 2015-03-17 15:57:13 -0700 + + * Splitting test-all Makefile target into Bro tests and test-aux. + (Robin Sommer) + 2.3-554 | 2015-03-17 15:40:39 -0700 * Deprecate &rotate_interval, &rotate_size, &encrypt. Addresses diff --git a/Makefile b/Makefile index 207ce72780..3efddc4dbc 100644 --- a/Makefile +++ b/Makefile @@ -51,13 +51,15 @@ distclean: $(MAKE) -C testing $@ test: - @( cd testing && make ) + -@( cd testing && make ) -test-all: test - test -d aux/broctl && ( cd aux/broctl && make test-all ) - test -d aux/btest && ( cd aux/btest && make test ) - test -d aux/bro-aux && ( cd aux/bro-aux && make test ) - test -d aux/plugins && ( cd aux/plugins && make test-all ) +test-aux: + -test -d aux/broctl && ( cd aux/broctl && make test-all ) + -test -d aux/btest && ( cd aux/btest && make test ) + -test -d aux/bro-aux && ( cd aux/bro-aux && make test ) + -test -d aux/plugins && ( cd aux/plugins && make test-all ) + +test-all: test test-aux configured: @test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 ) diff --git a/VERSION b/VERSION index 1b240f1fbf..5195f911b3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-554 +2.3-555 From 567073ac09fcf2d5fa6fa6bd83f24549a467c1bc Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 18 Mar 2015 08:46:56 -0700 Subject: [PATCH 22/22] Updating submodule(s). [nomail] --- aux/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/plugins b/aux/plugins index 71d820e9d8..172e0559ec 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 71d820e9d8ca753fea8fb34ea3987993b28d79e4 +Subproject commit 172e0559ec508c86abb81b371ee28e79130faec6