From 83783c5ca75dc94e6c8b12fc6c3ee318862d292d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Sun, 11 Sep 2011 20:36:58 -0700 Subject: [PATCH 01/12] Bugfix for log writer. It didn't escape binary stuff in some situations. Closes #585. --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- src/Desc.cc | 28 ++++++++++-------- .../ssh.log | Bin 139 -> 131 bytes .../http.log | 5 ++++ .../ssh.log | Bin 413 -> 397 bytes testing/btest/Traces/www-odd-url.trace | Bin 0 -> 1947 bytes .../logging/ascii-escape-odd-url.bro | 4 +++ 10 files changed, 29 insertions(+), 16 deletions(-) create mode 100644 testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log create mode 100644 testing/btest/Traces/www-odd-url.trace create mode 100644 testing/btest/scripts/base/frameworks/logging/ascii-escape-odd-url.bro diff --git a/aux/binpac b/aux/binpac index 032b4e0f02..a3a9410ded 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 032b4e0f028a08257be0c703b27a7559e57bd40a +Subproject commit a3a9410dedc842f6bb9859642f334ed354633b57 diff --git a/aux/bro-aux b/aux/bro-aux index 04d149a194..d68b98bb99 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 04d149a194e06ed5410ea3af924ff48b9129cd3b +Subproject commit d68b98bb995a105b257f805ec4ff22c4929c7476 diff --git a/aux/broccoli b/aux/broccoli index 89620cc8e5..03e6d398ed 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 89620cc8e500855fb763281000cbe2a24290a829 +Subproject commit 03e6d398edf422140ba9f50e6fabbec33ee2f3cb diff --git a/aux/broctl b/aux/broctl index c7499ee54f..ad8dfaba0c 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit c7499ee54f50bca65606dc3edc1aff132d93af80 +Subproject commit ad8dfaba0c0c784060aa6f0c5e1fcf62244b1a51 diff --git a/src/Desc.cc b/src/Desc.cc index 454de549ef..d7106a5b6a 100644 --- a/src/Desc.cc +++ b/src/Desc.cc @@ -232,35 +232,39 @@ static const char* find_first_unprintable(ODesc* d, const char* bytes, unsigned void ODesc::AddBytes(const void* bytes, unsigned int n) { + if ( ! escape ) + { + AddBytesRaw(bytes, n); + return; + } + const char* s = (const char*) bytes; const char* e = (const char*) bytes + n; while ( s < e ) { - const char* t1 = escape ? (const char*) memchr(s, escape[0], e - s) : e; - const char* t2 = find_first_unprintable(this, s, t1 ? e - t1 : e - s); + const char* t1 = (const char*) memchr(s, escape[0], e - s); - if ( t2 && (t2 < t1 || ! t1) ) + if ( ! t1 ) + t1 = e; + + const char* t2 = find_first_unprintable(this, s, t1 - s); + + if ( t2 && t2 < t1 ) { AddBytesRaw(s, t2 - s); char hex[6] = "\\x00"; hex[2] = hex_chars[((*t2) & 0xf0) >> 4]; hex[3] = hex_chars[(*t2) & 0x0f]; - AddBytesRaw(hex, sizeof(hex)); + AddBytesRaw(hex, 4); s = t2 + 1; continue; } - if ( ! escape ) - break; - - if ( ! t1 ) - break; - if ( memcmp(t1, escape, escape_len) != 0 ) - break; + break; AddBytesRaw(s, t1 - s); @@ -269,7 +273,7 @@ void ODesc::AddBytes(const void* bytes, unsigned int n) char hex[5] = "\\x00"; hex[2] = hex_chars[((*t1) & 0xf0) >> 4]; hex[3] = hex_chars[(*t1) & 0x0f]; - AddBytesRaw(hex, sizeof(hex)); + AddBytesRaw(hex, 4); ++t1; } diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log index fb68b42aefc7d0d4011f247326aa46b0c192f867..7f9371f9395336bd86fd7e1b9c5ec8306ce1c91c 100644 GIT binary patch delta 47 scmeBXY-XI`$r4kMmNqd^Od%yTt;WSM#L>}H(c>BPVgQ<0X&z>t!fR^#Fr;%LN`n3Nn-VV;a#9HbO1i%<^$5UUk8 diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log new file mode 100644 index 0000000000..e76a706ebe --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-escape-odd-url/http.log @@ -0,0 +1,5 @@ +#separator \x09 +#path http +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p method host uri referrer user_agent request_content_length response_content_length status_code status_msg filename tags username password proxied mime_type md5 extraction_file +#types time string addr port addr port string string string string string count count count string string table string string table string string file +1315799856.264750 UWkUyAuUGXf 10.0.1.104 64216 193.40.5.162 80 GET lepo.it.da.ut.ee /~cect/teoreetilised seminarid_2010/arheoloogia_uurimisr\xfchma_seminar/Joyce et al - The Languages of Archaeology ~ Dialogue, Narrative and Writing.pdf - Wget/1.12 (darwin10.8.0) - 346 404 Not Found - - - - - text/html - - diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.types/ssh.log index d1e6086fc415230acdb544817626b865d9aaba69..ffd579c224ccdaad8af8ce46546f0859d536358c 100644 GIT binary patch delta 106 zcmbQs+{--SHLrz%p{22*o{_n+nW5RlAI?G53NaN1iJ55%8U_kB3cCDt)&l;L)qx zstu$A8QeKTNcX9yZ*nrXK(zh6#~Z(0M+?ukUf9-t-a$W+`XIA2&dCRsa4+EJzQ1uM zK!(s7A9cQmzv<+@Zx}VcKSK+twX@~&FTYpmNY=OoPLzZI$zf{vs3rPlzesW77BnPY zD3^z>9VcELNrSwap&2GKYB`iLqhPo+dnu&2v}l;RV`S$-imJ%Elc!d}vT}w#S1LJ1 z(Quthqr>^4K6h|ZzGkgtDBh$DKY9T_K^*Bk-%IIcuB7Lvi*suMKM5z3*Eun_f~!>= zNkhj~iT2=pQgS|v?xqbXOTKlNZ5KNPvwvllGF%vPMgMZ7Zk1Roa+{z@7}PfIW?7% z6{%n9vmu$dC5J#8)8q)h)mu6K_8qJPQX-YCGOSK8Ma>+`N1-=-SriV_8wF+*sa0YF zksc`O%LA%{RsFA0;C=;`Ua>JKMwgkKXT`!5z*cfx$_5PA3ikCy`hzO3TdX~KV>msl zjbOt}q(GYN7AFRoJ=RH|AZt(mPaN#;zEq`;;}r{v$$XeE716{jK<{Z zrVx`WhhlG8*%f|IjeA+gzf|0^ Date: Mon, 12 Sep 2011 12:12:47 -0700 Subject: [PATCH 02/12] Updating baselines and testing helper script. --- .../ssh.log | Bin 131 -> 129 bytes .../ssh.log | Bin 472 -> 468 bytes testing/external/scripts/update-traces | 4 +++- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log b/testing/btest/Baseline/scripts.base.frameworks.logging.ascii-binary/ssh.log index 7f9371f9395336bd86fd7e1b9c5ec8306ce1c91c..b236cb818b259921aadf57d071d0d4516b72160a 100644 GIT binary patch delta 20 bcmZo>Y-F4eH!)t4DFwj(TEEGL&^pT delta 23 ecmZolo!Go)*OrR2K;i<}x%kG(}gfpcqqOo(#gJ dMX5D4q25?k$tf3?!qlY}7uVD{1z}aE1pt{!C*%MC delta 139 zcmcb@e1mzy0$xKia|2T&JxgOVQw#HnYmLza-s+(XC@9BNm?txUiPECfnwn5=E< Date: Mon, 12 Sep 2011 12:16:14 -0700 Subject: [PATCH 03/12] Updating submodule(s). --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index a3a9410ded..4fc13f7c69 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit a3a9410dedc842f6bb9859642f334ed354633b57 +Subproject commit 4fc13f7c6987b4163609e3df7a31f38501411cb7 diff --git a/aux/bro-aux b/aux/bro-aux index d68b98bb99..86990f1640 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit d68b98bb995a105b257f805ec4ff22c4929c7476 +Subproject commit 86990f1640d986e39d5bb1287dbeb03b59a464f0 diff --git a/aux/broccoli b/aux/broccoli index 03e6d398ed..6df97331bb 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 03e6d398edf422140ba9f50e6fabbec33ee2f3cb +Subproject commit 6df97331bb74d02ef2252138b301e4ca14523962 diff --git a/aux/broctl b/aux/broctl index ad8dfaba0c..870ee782bf 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit ad8dfaba0c0c784060aa6f0c5e1fcf62244b1a51 +Subproject commit 870ee782bfeb3a60bac40fce4273436e5f2d280b diff --git a/aux/btest b/aux/btest index d1c620d98c..ab78a66dd7 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit d1c620d98ce9d9c0b203314108b413784965d2ed +Subproject commit ab78a66dd782f165ddf921faaf1f065b2f987481 From eda8632d62ae1f4a4a9c489d0e940178598a51eb Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 12 Sep 2011 15:50:04 -0400 Subject: [PATCH 04/12] Small updates for a warning from the cluster framework. --- scripts/base/frameworks/cluster/main.bro | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 6a0d29f838..4c31992ea1 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -86,15 +86,15 @@ function local_node_type(): NodeType return is_enabled() ? nodes[node]$node_type : NONE; } -event remote_connection_handshake_done(p: event_peer) +event remote_connection_handshake_done(p: event_peer) &priority=5 { - if ( nodes[p$descr]$node_type == WORKER ) + if ( p$descr in nodes && nodes[p$descr]$node_type == WORKER ) ++worker_count; } -event remote_connection_closed(p: event_peer) +event remote_connection_closed(p: event_peer) &priority=5 { - if ( nodes[p$descr]$node_type == WORKER ) + if ( p$descr in nodes && nodes[p$descr]$node_type == WORKER ) --worker_count; } From c87704cc255a31b9f2b3fb3147632ebab6150736 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 12 Sep 2011 16:14:28 -0400 Subject: [PATCH 05/12] Small usability and correctness updates. - Removed an notice definition from the base SSL scripts. - Moved a logging stream ID into the export section for known-services and bumped priority for creating the stream. - Adding configuration knobs for the SQL injection attack detection script and renaming the HTTP::SQL_Injection_Attack notice to HTTP::SQL_Injection_Attack_Against - Bumped priority when creating Known::CERTS_LOG. --- scripts/base/protocols/ssl/main.bro | 5 --- .../policy/protocols/conn/known-services.bro | 8 ++-- scripts/policy/protocols/http/detect-sqli.bro | 37 ++++++++++++------- scripts/policy/protocols/ssl/known-certs.bro | 2 +- 4 files changed, 29 insertions(+), 23 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 99dc846903..f5ada491cb 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -1,15 +1,10 @@ @load ./consts -@load base/frameworks/notice module SSL; export { redef enum Log::ID += { LOG }; - redef enum Notice::Type += { - Self_Signed_Cert - }; - type Info: record { ts: time &log; uid: string &log; diff --git a/scripts/policy/protocols/conn/known-services.bro b/scripts/policy/protocols/conn/known-services.bro index 843c8bbfa1..3c73c07aad 100644 --- a/scripts/policy/protocols/conn/known-services.bro +++ b/scripts/policy/protocols/conn/known-services.bro @@ -7,9 +7,9 @@ module Known; -redef enum Log::ID += { SERVICES_LOG }; - export { + redef enum Log::ID += { SERVICES_LOG }; + type Info: record { ts: time &log; host: addr &log; @@ -33,10 +33,10 @@ redef record connection += { known_services_watch: bool &default=F; }; -event bro_init() +event bro_init() &priority=5 { Log::create_stream(Known::SERVICES_LOG, [$columns=Info, - $ev=log_known_services]); + $ev=log_known_services]); } function known_services_done(c: connection) diff --git a/scripts/policy/protocols/http/detect-sqli.bro b/scripts/policy/protocols/http/detect-sqli.bro index 470615458e..89a37728f4 100644 --- a/scripts/policy/protocols/http/detect-sqli.bro +++ b/scripts/policy/protocols/http/detect-sqli.bro @@ -1,19 +1,19 @@ ##! SQL injection detection in HTTP. -@load base/frameworks/notice/main -@load base/frameworks/metrics/main -@load base/protocols/http/main +@load base/frameworks/notice +@load base/frameworks/metrics +@load base/protocols/http module HTTP; export { redef enum Notice::Type += { SQL_Injection_Attacker, - SQL_Injection_Attack, + SQL_Injection_Attack_Against, }; redef enum Metrics::ID += { - SQL_ATTACKS, + SQL_ATTACKER, SQL_ATTACKS_AGAINST, }; @@ -26,6 +26,16 @@ export { ## Indicator of a cookie based SQL injection attack. Not implemented yet. COOKIE_SQLI, }; + + ## This defines the threshold that determines if an SQL injection attack + ## is ongoing based on the number of requests that appear to be SQL + ## injection attacks. + const sqli_requests_threshold = 50 &redef; + + ## Interval at which to watch for the :bro:id:`sqli_requests_threshold` + ## variable to be crossed. At the end of each interval the counter is + ## reset. + const sqli_requests_interval = 5min &redef; ## This regular expression is used to match URI based SQL injections const match_sql_injection_uri = @@ -37,15 +47,16 @@ export { | /\/\*![[:digit:]]{5}.*?\*\// &redef; } -event bro_init() +event bro_init() &priority=3 { - Metrics::add_filter(SQL_ATTACKS, [$log=F, - $break_interval=5mins, - $note=SQL_Injection_Attacker]); + Metrics::add_filter(SQL_ATTACKER, [$log=F, + $notice_threshold=sqli_requests_threshold, + $break_interval=sqli_requests_interval, + $note=SQL_Injection_Attacker]); Metrics::add_filter(SQL_ATTACKS_AGAINST, [$log=F, - $break_interval=5mins, - $note=SQL_Injection_Attack, - $notice_threshold=50]); + $notice_threshold=sqli_requests_threshold, + $break_interval=sqli_requests_interval, + $note=SQL_Injection_Attack_Against]); } event http_request(c: connection, method: string, original_URI: string, @@ -55,7 +66,7 @@ event http_request(c: connection, method: string, original_URI: string, { add c$http$tags[URI_SQLI]; - Metrics::add_data(SQL_ATTACKS, [$host=c$id$orig_h], 1); + Metrics::add_data(SQL_ATTACKER, [$host=c$id$orig_h], 1); Metrics::add_data(SQL_ATTACKS_AGAINST, [$host=c$id$resp_h], 1); } } diff --git a/scripts/policy/protocols/ssl/known-certs.bro b/scripts/policy/protocols/ssl/known-certs.bro index 9a6dfbcd44..8a013123c8 100644 --- a/scripts/policy/protocols/ssl/known-certs.bro +++ b/scripts/policy/protocols/ssl/known-certs.bro @@ -34,7 +34,7 @@ export { global log_known_certs: event(rec: Info); } -event bro_init() +event bro_init() &priority=5 { Log::create_stream(Known::CERTS_LOG, [$columns=Info, $ev=log_known_certs]); } From ee1884ca936206673c03e7a0e011750456fb3b6a Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 13 Sep 2011 09:09:55 -0400 Subject: [PATCH 06/12] Another change to possibly fix the SSL memory consumption problem. - If a protocol violation happens, Bro now logs what it has seen up until the protocol violation and deletes the c$ssl record so that a long lived connection with a protocol violation does continue to hold the memory. --- scripts/base/protocols/ssl/main.bro | 30 ++++++++++++++++++++--------- 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index f5ada491cb..cfc513120d 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -70,6 +70,20 @@ function set_session(c: connection) if ( ! c?$ssl ) c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()]; } + +function finish(c: connection, violation: bool) + { + Log::write(SSL::LOG, c$ssl); + if ( delete_certs_after_logging ) + { + if ( c$ssl?$cert ) + delete c$ssl$cert; + if ( c$ssl?$cert_chain ) + delete c$ssl$cert_chain; + } + if ( violation ) + delete c$ssl; + } event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5 { @@ -120,14 +134,12 @@ event ssl_established(c: connection) &priority=5 event ssl_established(c: connection) &priority=-5 { - Log::write(SSL::LOG, c$ssl); - - if ( delete_certs_after_logging ) - { - if ( c$ssl?$cert ) - delete c$ssl$cert; - if ( c$ssl?$cert_chain ) - delete c$ssl$cert_chain; - } + finish(c, F); } +event protocol_violation(c: connection, atype: count, aid: count, + reason: string) &priority=5 + { + if ( c?$ssl ) + finish(c, T); + } \ No newline at end of file From fa375d3164ab1bfbf92e62edc64c416f8ee9ca04 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 13 Sep 2011 09:10:19 -0400 Subject: [PATCH 07/12] Removing a small "TODO" now that a bug has been fixed. --- scripts/base/frameworks/dpd/main.bro | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/base/frameworks/dpd/main.bro b/scripts/base/frameworks/dpd/main.bro index 5b77ad6d0c..1e71c61f7e 100644 --- a/scripts/base/frameworks/dpd/main.bro +++ b/scripts/base/frameworks/dpd/main.bro @@ -25,8 +25,7 @@ export { ## Disabled analyzer IDs. This is only for internal tracking ## so as to not attempt to disable analyzers multiple times. - # TODO: This is waiting on ticket #460 to remove the '0'. - disabled_aids: set[count] &default=set(0); + disabled_aids: set[count] &default=set(); }; ## Ignore violations which go this many bytes into the connection. From 0a7685bf2927ae7cf9545b1e3c5d6807ddaf9187 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 13 Sep 2011 10:41:25 -0400 Subject: [PATCH 08/12] Not sure what happened here, but the broctl remote was on an old branch. --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 870ee782bf..c7499ee54f 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 870ee782bfeb3a60bac40fce4273436e5f2d280b +Subproject commit c7499ee54f50bca65606dc3edc1aff132d93af80 From af6c7c8b1a2886a7eadc296f4a1c2dc0f2d3f6cb Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 13 Sep 2011 21:34:29 -0400 Subject: [PATCH 09/12] HTTP body size measurement added to http log. - The value of the content-length headers has now been removed but it could be added back locally at an installation by a user. - Added fields to indicate if some parsing interruption happened during the body transfer. - Closes #581 --- scripts/base/protocols/http/main.bro | 40 +++++++++++++++++++--------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/scripts/base/protocols/http/main.bro b/scripts/base/protocols/http/main.bro index 82136e0e37..27bffe5187 100644 --- a/scripts/base/protocols/http/main.bro +++ b/scripts/base/protocols/http/main.bro @@ -30,10 +30,20 @@ export { referrer: string &log &optional; ## The value of the User-Agent header from the client. user_agent: string &log &optional; - ## The value of the Content-Length header from the client. - request_content_length: count &log &optional; - ## The value of the Content-Length header from the server. - response_content_length: count &log &optional; + ## The actual uncompressed content size of the data transferred from + ## the client. + request_body_len: count &log &optional; + ## This indicates whether or not there was an interruption while the + ## request body was being sent. + request_body_interrupted: bool &log &default=F; + ## The actual uncompressed content size of the data transferred from + ## the server. + response_body_len: count &log &optional; + ## This indicates whether or not there was an interruption while the + ## request body was being sent. An interruption could cause hash + ## calculation to fail and a number of other problems since the + ## analyzer may not be able to get back on track with the connection. + response_body_interrupted: bool &log &default=F; ## The status code returned by the server. status_code: count &log &optional; ## The status message returned by the server. @@ -174,9 +184,6 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr # The split is done to remove the occasional port value that shows up here. c$http$host = split1(value, /:/)[1]; - else if ( name == "CONTENT-LENGTH" ) - c$http$request_content_length = extract_count(value); - else if ( name == "USER-AGENT" ) c$http$user_agent = value; @@ -201,7 +208,7 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr } else { - c$http$username = ""; + c$http$username = fmt(" (%s)", value); if ( c$http$capture_password ) c$http$password = userpass; } @@ -212,10 +219,8 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr } else # server headers { - if ( name == "CONTENT-LENGTH" ) - c$http$response_content_length = extract_count(value); - else if ( name == "CONTENT-DISPOSITION" && - /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) + if ( name == "CONTENT-DISPOSITION" && + /[fF][iI][lL][eE][nN][aA][mM][eE]/ in value ) c$http$filename = extract_filename_from_content_disposition(value); } } @@ -223,6 +228,17 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = 5 { set_state(c, F, is_orig); + + if ( is_orig ) + { + c$http$request_body_len = stat$body_length; + c$http$request_body_interrupted = stat$interrupted; + } + else + { + c$http$response_body_len = stat$body_length; + c$http$response_body_interrupted = stat$interrupted; + } } event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) &priority = -5 From f32b567c8558121d017f45662d767fbde7c50826 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Tue, 13 Sep 2011 22:33:26 -0400 Subject: [PATCH 10/12] New script for logging header names and values. - Closes #519. --- .../protocols/http/{headers.bro => header-names.bro} | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) rename scripts/policy/protocols/http/{headers.bro => header-names.bro} (61%) diff --git a/scripts/policy/protocols/http/headers.bro b/scripts/policy/protocols/http/header-names.bro similarity index 61% rename from scripts/policy/protocols/http/headers.bro rename to scripts/policy/protocols/http/header-names.bro index dc3eddcbc0..fdecdfa0e8 100644 --- a/scripts/policy/protocols/http/headers.bro +++ b/scripts/policy/protocols/http/header-names.bro @@ -1,4 +1,6 @@ -##! Extract and include the header keys used for each request in the log. +##! Extract and include the header names used for each request in the HTTP +##! logging stream. The headers in the logging stream will be stored in the +##! same order which they were seen on the wire. @load base/protocols/http/main @@ -8,15 +10,13 @@ export { redef record Info += { ## The vector of HTTP headers. No header values are included here, just ## the header names. - ## TODO: with an empty vector as &default, the vector isn't coerced to the - ## correct type. headers: vector of string &log &optional; }; } -event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=4 +event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 { - if ( ! is_orig ) + if ( ! is_orig || ! c?$http ) return; if ( ! c$http?$headers ) From c8e62556664fd0006183694dc31bf9d597461b37 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 14 Sep 2011 22:44:17 -0400 Subject: [PATCH 11/12] More options for the header-names.bro script. --- .../policy/protocols/http/header-names.bro | 32 +++++++++++++++---- 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/scripts/policy/protocols/http/header-names.bro b/scripts/policy/protocols/http/header-names.bro index fdecdfa0e8..284db78351 100644 --- a/scripts/policy/protocols/http/header-names.bro +++ b/scripts/policy/protocols/http/header-names.bro @@ -8,10 +8,20 @@ module HTTP; export { redef record Info += { - ## The vector of HTTP headers. No header values are included here, just - ## the header names. - headers: vector of string &log &optional; + ## The vector of HTTP header names sent by the client. No header + ## values are included here, just the header names. + client_header_names: vector of string &log &optional; + + ## The vector of HTTP header names sent by the server. No header + ## values are included here, just the header names. + server_headers_names: vector of string &log &optional; }; + + ## A boolean value to determine if client header names are to be logged. + const log_client_header_names = T &redef; + + ## A boolean value to determine if server header names are to be logged. + const log_server_header_names = F &redef; } event http_header(c: connection, is_orig: bool, name: string, value: string) &priority=3 @@ -19,7 +29,17 @@ event http_header(c: connection, is_orig: bool, name: string, value: string) &pr if ( ! is_orig || ! c?$http ) return; - if ( ! c$http?$headers ) - c$http$headers = vector(); - c$http$headers[|c$http$headers|] = name; + if ( log_client_header_names ) + { + if ( ! c$http?$client_header_names ) + c$http$client_header_names = vector(); + c$http$client_header_names[|c$http$client_header_names|] = name; + } + + if ( log_server_header_names ) + { + if ( ! c$http?$server_header_names ) + c$http$server_header_names = vector(); + c$http$server_header_names[|c$http$server_header_names|] = name; + } } From d656e2a8c49445beeddee3ad996aecb581901995 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 14 Sep 2011 22:51:52 -0400 Subject: [PATCH 12/12] Fixing a major memory utilization issue with the x509_verify bif. - The check for the root_certs variable was recreating the certificate store over and over again which would eventually consume lots of memory in the x509_stores internal (c++) variable. The index check uses the Val pointer value for comparison now. --- src/bro.bif | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/bro.bif b/src/bro.bif index ef0cef5229..5629805ec5 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3423,7 +3423,7 @@ function unique_id_from%(pool: int, prefix: string%) : string #include // This is the indexed map of X509 certificate stores. -static map x509_stores; +static map x509_stores; // ### NOTE: while d2i_X509 does not take a const u_char** pointer, // here we assume d2i_X509 does not write to , so it is safe to @@ -3448,9 +3448,8 @@ function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: tabl int i = 0; // If this certificate store was built previously, just reuse the old one. - BroString* s = convert_index_to_string(root_certs); - if ( x509_stores.count(*s) > 0 ) - ctx = x509_stores[*s]; + if ( x509_stores.count(root_certs) > 0 ) + ctx = x509_stores[root_certs]; if ( ! ctx ) // lookup to see if we have this one built already! { @@ -3475,9 +3474,8 @@ function x509_verify%(der_cert: string, cert_stack: string_vec, root_certs: tabl delete idxs; // Save the newly constructed certificate store into the cacheing map. - x509_stores[*s] = ctx; + x509_stores[root_certs] = ctx; } - delete s; const uint8 *cert_data = der_cert->Bytes(); X509* cert = d2i_X509_(NULL, &cert_data, der_cert->Len());