From ff8184242aa8383775a42406b6fd1fb01266462d Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 17:03:39 -0800 Subject: [PATCH 01/49] Enhance ssl.log with information from notary. This commit brings enhances each log line with the data from the notary when available. The added fields include: - notary.first_seen - notary.last_seen - notary.times_seen - notary.valid The semantics of these fields map 1-to-1 to the corresponding fields in DNS TXT lookups from the notary. The implementation of this feature required a bit plumbing: when Bro finishes the analysis, the log record is copied into table indexed by connection ID where it remains until either Bro terminates or the answer of the notary arrives. The script accummulates requests for a given digest into a "waitlist," to avoid multiple redundant lookups for high-profile websites who receive a large chunk of traffic. When a DNS reply arrives asynchronously, the when handler clears the waitlist and assigns the information to all records in the buffered. The script also adds Each log entry into a double-ended queue to make sure the records arrive on disk in the same way Bro sees them. Each reply also triggers a sweep through this deque which flushes the buffer up to the first outstanding reply. Here is an example from the public M57 trace from 2009: % bro-cut ts id.orig_h id.resp_h server_name notary.first_seen notary.last_seen notary.times_seen notary.valid < ssl.log 1258562650.121682 192.168.1.104 208.97.132.223 mail.m57.biz - - - - 1258535660.267128 192.168.1.104 65.55.184.16 - - - - - 1258561662.604948 192.168.1.105 66.235.128.158 - - - - - 1258561885.571010 192.168.1.105 65.55.184.155 www.update.microsoft.com - - - - 1258563578.455331 192.168.1.103 208.97.132.223 - - - - - 1258563716.527681 192.168.1.103 96.6.248.124 - - - - - 1258563884.667153 192.168.1.103 66.235.139.152 - - - - - 1258564818.755676 192.168.1.103 12.41.118.177 - - - - - 1258564821.637874 192.168.1.103 12.41.118.177 - - - - - 1258564821.637871 192.168.1.103 12.41.118.177 - - - - - 1258564821.637876 192.168.1.103 12.41.118.177 - - - - - 1258564821.638126 192.168.1.103 12.41.118.177 - - - - - 1258562467.525034 192.168.1.104 208.97.132.223 mail.m57.biz 15392 15695 301 F 1258563063.965975 192.168.1.104 63.245.209.105 aus2.mozilla.org - - - - 1258563064.091396 192.168.1.104 63.245.209.91 addons.mozilla.org - - - - 1258563329.202273 192.168.1.103 208.97.132.223 - 15392 15695 301 F 1258563712.945933 192.168.1.103 65.55.16.121 - - - - - 1258563714.044500 192.168.1.103 65.54.186.79 - - - - - 1258563716.146680 192.168.1.103 96.6.248.124 - - - - - 1258563737.432312 192.168.1.103 96.6.245.186 - - - - - 1258563716.526933 192.168.1.103 96.6.245.186 - - - - - 1258563716.527430 192.168.1.103 96.6.245.186 - - - - - 1258563716.527179 192.168.1.103 96.6.245.186 - - - - - 1258563716.527683 192.168.1.103 96.6.245.186 - - - - - 1258563716.527432 192.168.1.103 96.6.245.186 - - - - - 1258563751.178683 192.168.1.103 66.235.139.152 - - - - - 1258563751.171938 192.168.1.103 65.54.234.75 - - - - - 1258563751.182433 192.168.1.103 65.242.27.35 - - - - - 1258563883.414188 192.168.1.103 65.55.16.121 - - - - - 1258563884.702380 192.168.1.103 65.242.27.35 - - - - - 1258563885.678766 192.168.1.103 65.54.186.79 - - - - - 1258563886.124987 192.168.1.103 65.54.186.79 - - - - - 1258564027.877525 192.168.1.103 65.54.234.75 - - - - - 1258564688.206859 192.168.1.103 65.54.186.107 - - - - - 1258567162.001225 192.168.1.105 208.97.132.223 mail.m57.biz - - - - 1258568040.512840 192.168.1.103 208.97.132.223 - - - - - 1258564688.577376 192.168.1.103 207.46.120.170 - - - - - 1258564723.029005 192.168.1.103 65.54.186.107 - - - - - 1258564723.784032 192.168.1.103 65.55.194.249 - - - - - 1258564748.521756 192.168.1.103 65.54.186.107 - - - - - 1258564817.601152 192.168.1.103 12.41.118.177 - - - - - 1258565684.353653 192.168.1.105 208.97.132.223 mail.m57.biz 15392 15695 301 F 1258565710.188691 192.168.1.105 74.125.155.109 pop.gmail.com - - - - 1258566061.103696 192.168.1.103 208.97.132.223 - 15392 15695 301 F 1258566893.914987 192.168.1.102 208.97.132.223 - 15392 15695 301 F --- scripts/base/protocols/ssl/main.bro | 157 +++++++++++++++++++++++++++- 1 file changed, 156 insertions(+), 1 deletion(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 6b434ae09d..60f17ea7ce 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -8,6 +8,14 @@ module SSL; export { redef enum Log::ID += { LOG }; + ## A response from the ICSI certificate notary. + type NotaryResponse: record { + first_seen: count &log &optional; + last_seen: count &log &optional; + times_seen: count &log &optional; + valid: bool &log &optional; + }; + type Info: record { ## Time when the SSL connection was first detected. ts: time &log; @@ -72,15 +80,48 @@ export { ## utility. const openssl_util = "openssl" &redef; + ## Flag that determines whether to use the ICSI certificate notary to enhance + ## the SSL log records. + const use_notary = T &redef; + ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); } +# TODO: Maybe wrap these in @ifdef's? Otherwise we carry the extra baggage +# around all the time. +redef record Info += { + sha1_digest: string &optional; + notary: NotaryResponse &log &optional; + }; + redef record connection += { ssl: Info &optional; }; +# The DNS cache of notary responses. +global notary_cache: table[string] of NotaryResponse &create_expire = 1 hr; + +# The buffered SSL log records. +global records: table[string] of Info; + +# The records that wait for a notary response identified by the cert digest. +# Each digest refers to a list of connection UIDs which are updated when a DNS +# reply arrives asynchronously. +global waiting: table[string] of vector of string; + +# A double-ended queue that determines the log record order in which logs have +# to written out to disk. +global deque: table[count] of string; + +# The top-most deque index. +global head = 0; + +# The bottom deque index that points to the next record to be flushed as soon +# as the notary response arrives. +global tail = 0; + event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -122,9 +163,103 @@ function set_session(c: connection) $client_cert_chain=vector()]; } +function clear_waitlist(digest: string) + { + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + { + local uid = waiting[digest][i]; + records[uid]$notary = []; + } + delete waiting[digest]; + } + } + +function lookup_cert_hash(uid: string, digest: string) + { + # Add the record ID to the list of waiting IDs for this digest. + local waits_already = digest in waiting; + if ( ! waits_already ) + waiting[digest] = vector(); + waiting[digest][|waiting[digest]|] = uid; + if ( waits_already ) + return; + + local domain = "%s.notary.icsi.berkeley.edu"; + when ( local str = lookup_hostname_txt(fmt(domain, digest)) ) + { + # Cache every response for a digest. + # TODO: should we ignore failing answers? + notary_cache[digest] = []; + + # Parse notary answer. + if ( str == "" ) + { + # TODO: Should we handle NXDOMAIN separately? + clear_waitlist(digest); + return; + } + local fields = split(str, / /); + if ( |fields| != 5 ) # version 1 has 5 fields. + { + clear_waitlist(digest); + return; + } + local version = split(fields[1], /=/)[2]; + if ( version != "1" ) + { + clear_waitlist(digest); + return; + } + local r = notary_cache[digest]; + r$first_seen = to_count(split(fields[2], /=/)[2]); + r$last_seen = to_count(split(fields[3], /=/)[2]); + r$times_seen = to_count(split(fields[4], /=/)[2]); + r$valid = split(fields[5], /=/)[2] == "1"; + + # Assign notary answer to all waiting records. + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + records[waiting[digest][i]]$notary = r; + delete waiting[digest]; + } + + # Flush all records up to the record which still awaits an answer. + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + if ( ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } + } + function finish(c: connection) { - Log::write(SSL::LOG, c$ssl); + if ( use_notary && c$ssl?$sha1_digest) + { + local digest = c$ssl$sha1_digest; + if ( c$ssl$sha1_digest in notary_cache ) + c$ssl$notary = notary_cache[digest]; + else + lookup_cert_hash(c$uid, digest); + records[c$uid] = c$ssl; # Copy the record. + deque[head] = c$uid; + ++head; + } + else + { + Log::write(SSL::LOG, c$ssl); + } + if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); delete c$ssl; @@ -181,6 +316,9 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun c$ssl$issuer_subject = cert$issuer; c$ssl$not_valid_before = cert$not_valid_before; c$ssl$not_valid_after = cert$not_valid_after; + + if ( use_notary ) + c$ssl$sha1_digest = sha1_hash(der_cert); } else { @@ -228,3 +366,20 @@ event protocol_violation(c: connection, atype: count, aid: count, if ( c?$ssl ) finish(c); } + +event bro_done() + { + if ( ! use_notary || |deque| == 0 ) + return; + + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } From 7355a0089ae4c1d61f4e9f842763e8689514769a Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 17:17:58 -0800 Subject: [PATCH 02/49] Adhere to Bro coding style guidelines. --- scripts/base/protocols/ssl/main.bro | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 60f17ea7ce..e36fbd90e6 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -208,10 +208,10 @@ function lookup_cert_hash(uid: string, digest: string) } local version = split(fields[1], /=/)[2]; if ( version != "1" ) - { + { clear_waitlist(digest); return; - } + } local r = notary_cache[digest]; r$first_seen = to_count(split(fields[2], /=/)[2]); r$last_seen = to_count(split(fields[3], /=/)[2]); @@ -220,24 +220,24 @@ function lookup_cert_hash(uid: string, digest: string) # Assign notary answer to all waiting records. if ( digest in waiting ) - { + { for ( i in waiting[digest] ) records[waiting[digest][i]]$notary = r; delete waiting[digest]; - } + } # Flush all records up to the record which still awaits an answer. local current: string; for ( unused_index in deque ) { - current = deque[tail]; - local info = records[current]; - if ( ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; + current = deque[tail]; + local info = records[current]; + if ( ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; } } } From 382262e286bb923ca38dd156217e14988656bc2f Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 17:56:31 -0800 Subject: [PATCH 03/49] Factor notary code into separte file. There exists one complication: the new file notary.bro requires the definition of the SSL::Info record, but as does main.bro. Because I did not really know where to put the common code (it's not a constant, so ssl/const.bro does not really fit), I put it into __load.bro__ so that it sticks out for now. If anybody has an idea how to solve this elegantly, please let me know. --- scripts/base/protocols/ssl/__load__.bro | 64 ++++++- scripts/base/protocols/ssl/main.bro | 218 +----------------------- scripts/base/protocols/ssl/notary.bro | 162 ++++++++++++++++++ 3 files changed, 232 insertions(+), 212 deletions(-) create mode 100644 scripts/base/protocols/ssl/notary.bro diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro index eaaa13cd76..e97d1681fc 100644 --- a/scripts/base/protocols/ssl/__load__.bro +++ b/scripts/base/protocols/ssl/__load__.bro @@ -1,3 +1,65 @@ @load ./consts + +module SSL; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + ## Time when the SSL connection was first detected. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + ## SSL/TLS version that the server offered. + version: string &log &optional; + ## SSL/TLS cipher suite that the server chose. + cipher: string &log &optional; + ## Value of the Server Name Indicator SSL/TLS extension. It + ## indicates the server name that the client was requesting. + server_name: string &log &optional; + ## Session ID offered by the client for session resumption. + session_id: string &log &optional; + ## Subject of the X.509 certificate offered by the server. + subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the server. + issuer_subject: string &log &optional; + ## NotValidBefore field value from the server certificate. + not_valid_before: time &log &optional; + ## NotValidAfter field value from the server certificate. + not_valid_after: time &log &optional; + ## Last alert that was seen during the connection. + last_alert: string &log &optional; + + ## Subject of the X.509 certificate offered by the client. + client_subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the client. + client_issuer_subject: string &log &optional; + + ## Full binary server certificate stored in DER format. + cert: string &optional; + ## Chain of certificates offered by the server to validate its + ## complete signing chain. + cert_chain: vector of string &optional; + + ## Full binary client certificate stored in DER format. + client_cert: string &optional; + ## Chain of certificates offered by the client to validate its + ## complete signing chain. + client_cert_chain: vector of string &optional; + + ## The analyzer ID used for the analyzer instance attached + ## to each connection. It is not used for logging since it's a + ## meaningless arbitrary number. + analyzer_id: count &optional; + }; +} + +redef record connection += { + ssl: Info &optional; +}; + +@load ./notary @load ./main -@load ./mozilla-ca-list \ No newline at end of file +@load ./mozilla-ca-list diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index e36fbd90e6..f02602235c 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -6,66 +6,6 @@ module SSL; export { - redef enum Log::ID += { LOG }; - - ## A response from the ICSI certificate notary. - type NotaryResponse: record { - first_seen: count &log &optional; - last_seen: count &log &optional; - times_seen: count &log &optional; - valid: bool &log &optional; - }; - - type Info: record { - ## Time when the SSL connection was first detected. - ts: time &log; - ## Unique ID for the connection. - uid: string &log; - ## The connection's 4-tuple of endpoint addresses/ports. - id: conn_id &log; - ## SSL/TLS version that the server offered. - version: string &log &optional; - ## SSL/TLS cipher suite that the server chose. - cipher: string &log &optional; - ## Value of the Server Name Indicator SSL/TLS extension. It - ## indicates the server name that the client was requesting. - server_name: string &log &optional; - ## Session ID offered by the client for session resumption. - session_id: string &log &optional; - ## Subject of the X.509 certificate offered by the server. - subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the server. - issuer_subject: string &log &optional; - ## NotValidBefore field value from the server certificate. - not_valid_before: time &log &optional; - ## NotValidAfter field value from the server certificate. - not_valid_after: time &log &optional; - ## Last alert that was seen during the connection. - last_alert: string &log &optional; - - ## Subject of the X.509 certificate offered by the client. - client_subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the client. - client_issuer_subject: string &log &optional; - - ## Full binary server certificate stored in DER format. - cert: string &optional; - ## Chain of certificates offered by the server to validate its - ## complete signing chain. - cert_chain: vector of string &optional; - - ## Full binary client certificate stored in DER format. - client_cert: string &optional; - ## Chain of certificates offered by the client to validate its - ## complete signing chain. - client_cert_chain: vector of string &optional; - - ## The analyzer ID used for the analyzer instance attached - ## to each connection. It is not used for logging since it's a - ## meaningless arbitrary number. - analyzer_id: count &optional; - }; - ## The default root CA bundle. By loading the ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; @@ -80,48 +20,11 @@ export { ## utility. const openssl_util = "openssl" &redef; - ## Flag that determines whether to use the ICSI certificate notary to enhance - ## the SSL log records. - const use_notary = T &redef; - ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); } -# TODO: Maybe wrap these in @ifdef's? Otherwise we carry the extra baggage -# around all the time. -redef record Info += { - sha1_digest: string &optional; - notary: NotaryResponse &log &optional; - }; - -redef record connection += { - ssl: Info &optional; -}; - -# The DNS cache of notary responses. -global notary_cache: table[string] of NotaryResponse &create_expire = 1 hr; - -# The buffered SSL log records. -global records: table[string] of Info; - -# The records that wait for a notary response identified by the cert digest. -# Each digest refers to a list of connection UIDs which are updated when a DNS -# reply arrives asynchronously. -global waiting: table[string] of vector of string; - -# A double-ended queue that determines the log record order in which logs have -# to written out to disk. -global deque: table[count] of string; - -# The top-most deque index. -global head = 0; - -# The bottom deque index that points to the next record to be flushed as soon -# as the notary response arrives. -global tail = 0; - event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -163,102 +66,15 @@ function set_session(c: connection) $client_cert_chain=vector()]; } -function clear_waitlist(digest: string) - { - if ( digest in waiting ) - { - for ( i in waiting[digest] ) - { - local uid = waiting[digest][i]; - records[uid]$notary = []; - } - delete waiting[digest]; - } - } - -function lookup_cert_hash(uid: string, digest: string) - { - # Add the record ID to the list of waiting IDs for this digest. - local waits_already = digest in waiting; - if ( ! waits_already ) - waiting[digest] = vector(); - waiting[digest][|waiting[digest]|] = uid; - if ( waits_already ) - return; - - local domain = "%s.notary.icsi.berkeley.edu"; - when ( local str = lookup_hostname_txt(fmt(domain, digest)) ) - { - # Cache every response for a digest. - # TODO: should we ignore failing answers? - notary_cache[digest] = []; - - # Parse notary answer. - if ( str == "" ) - { - # TODO: Should we handle NXDOMAIN separately? - clear_waitlist(digest); - return; - } - local fields = split(str, / /); - if ( |fields| != 5 ) # version 1 has 5 fields. - { - clear_waitlist(digest); - return; - } - local version = split(fields[1], /=/)[2]; - if ( version != "1" ) - { - clear_waitlist(digest); - return; - } - local r = notary_cache[digest]; - r$first_seen = to_count(split(fields[2], /=/)[2]); - r$last_seen = to_count(split(fields[3], /=/)[2]); - r$times_seen = to_count(split(fields[4], /=/)[2]); - r$valid = split(fields[5], /=/)[2] == "1"; - - # Assign notary answer to all waiting records. - if ( digest in waiting ) - { - for ( i in waiting[digest] ) - records[waiting[digest][i]]$notary = r; - delete waiting[digest]; - } - - # Flush all records up to the record which still awaits an answer. - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - if ( ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } - } - function finish(c: connection) { - if ( use_notary && c$ssl?$sha1_digest) - { - local digest = c$ssl$sha1_digest; - if ( c$ssl$sha1_digest in notary_cache ) - c$ssl$notary = notary_cache[digest]; - else - lookup_cert_hash(c$uid, digest); - records[c$uid] = c$ssl; # Copy the record. - deque[head] = c$uid; - ++head; - } - else - { - Log::write(SSL::LOG, c$ssl); - } +# TODO: This dummy flag merely exists to check whether the notary script is +# loaded. There's probably a better way to incorporate the notary. +@ifdef( Notary::enabled ) + Notary::push(c$ssl); +@else + Log::write(SSL::LOG, c$ssl); +@endif if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); @@ -316,9 +132,6 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun c$ssl$issuer_subject = cert$issuer; c$ssl$not_valid_before = cert$not_valid_before; c$ssl$not_valid_after = cert$not_valid_after; - - if ( use_notary ) - c$ssl$sha1_digest = sha1_hash(der_cert); } else { @@ -366,20 +179,3 @@ event protocol_violation(c: connection, atype: count, aid: count, if ( c?$ssl ) finish(c); } - -event bro_done() - { - if ( ! use_notary || |deque| == 0 ) - return; - - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/base/protocols/ssl/notary.bro new file mode 100644 index 0000000000..6cf687ea90 --- /dev/null +++ b/scripts/base/protocols/ssl/notary.bro @@ -0,0 +1,162 @@ +module Notary; + +export { + # Flag to tell the SSL analysis script that it should buffer logs instead of + # flushing them directly. + const enabled = T; + + ## A response from the ICSI certificate notary. + type Response: record { + first_seen: count &log &optional; + last_seen: count &log &optional; + times_seen: count &log &optional; + valid: bool &log &optional; + }; + + ## Hands over an SSL record to the Notary module. This is an ownership + ## transfer, i.e., the caller does not need to call Log::write on this record + ## anymore. + global push: function(info: SSL::Info); + + ## The notary domain to query. + const domain = "notary.icsi.berkeley.edu" &redef; +} + +redef record SSL::Info += { + sha1_digest: string &optional; + notary: Response &log &optional; + }; + +# The DNS cache of notary responses. +global notary_cache: table[string] of Response &create_expire = 1 hr; + +# The buffered SSL log records. +global records: table[string] of SSL::Info; + +# The records that wait for a notary response identified by the cert digest. +# Each digest refers to a list of connection UIDs which are updated when a DNS +# reply arrives asynchronously. +global waiting: table[string] of vector of string; + +# A double-ended queue that determines the log record order in which logs have +# to written out to disk. +global deque: table[count] of string; + +# The top-most deque index. +global head = 0; + +# The bottom deque index that points to the next record to be flushed as soon +# as the notary response arrives. +global tail = 0; + +function clear_waitlist(digest: string) + { + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + { + local uid = waiting[digest][i]; + records[uid]$notary = []; + } + delete waiting[digest]; + } + } + +function flush(evict_all: bool) + { + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + if ( ! evict_all && ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } + +function lookup_cert_hash(uid: string, digest: string) + { + # Add the record ID to the list of waiting IDs for this digest. + local waits_already = digest in waiting; + if ( ! waits_already ) + waiting[digest] = vector(); + waiting[digest][|waiting[digest]|] = uid; + if ( waits_already ) + return; + + when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) ) + { + # Cache every response for a digest. + notary_cache[digest] = []; + + # Parse notary answer. + if ( str == "" ) + { + # TODO: Should we handle NXDOMAIN separately? + clear_waitlist(digest); + return; + } + local fields = split(str, / /); + if ( |fields| != 5 ) # version 1 has 5 fields. + { + clear_waitlist(digest); + return; + } + local version = split(fields[1], /=/)[2]; + if ( version != "1" ) + { + clear_waitlist(digest); + return; + } + local r = notary_cache[digest]; + r$first_seen = to_count(split(fields[2], /=/)[2]); + r$last_seen = to_count(split(fields[3], /=/)[2]); + r$times_seen = to_count(split(fields[4], /=/)[2]); + r$valid = split(fields[5], /=/)[2] == "1"; + + # Assign notary answer to all waiting records. + if ( digest in waiting ) + { + for ( i in waiting[digest] ) + records[waiting[digest][i]]$notary = r; + delete waiting[digest]; + } + + flush(F); + } + } + +function push(info: SSL::Info) + { + if ( ! info?$sha1_digest ) + return; + + local digest = info$sha1_digest; + if ( info$sha1_digest in notary_cache ) + info$notary = notary_cache[digest]; + else + lookup_cert_hash(info$uid, digest); + records[info$uid] = info; + deque[head] = info$uid; + ++head; + } + +event x509_certificate(c: connection, is_orig: bool, cert: X509, + chain_idx: count, chain_len: count, der_cert: string) + { + if ( is_orig || chain_idx != 0 || ! c?$ssl ) + return; + + c$ssl$sha1_digest = sha1_hash(der_cert); + } + +event bro_done() + { + if ( |deque| == 0 ) + return; + flush(T); + } From 8a569facd623d5d15d5e6d42ce3f549bcf4d94c2 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Fri, 21 Dec 2012 18:04:19 -0800 Subject: [PATCH 04/49] More style tweaks: replace spaces with tabs. --- scripts/base/protocols/ssl/notary.bro | 68 +++++++++++++-------------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/base/protocols/ssl/notary.bro index 6cf687ea90..4ae03fd3f8 100644 --- a/scripts/base/protocols/ssl/notary.bro +++ b/scripts/base/protocols/ssl/notary.bro @@ -13,12 +13,12 @@ export { valid: bool &log &optional; }; - ## Hands over an SSL record to the Notary module. This is an ownership - ## transfer, i.e., the caller does not need to call Log::write on this record - ## anymore. - global push: function(info: SSL::Info); + ## Hands over an SSL record to the Notary module. This is an ownership + ## transfer, i.e., the caller does not need to call Log::write on this record + ## anymore. + global push: function(info: SSL::Info); - ## The notary domain to query. + ## The notary domain to query. const domain = "notary.icsi.berkeley.edu" &redef; } @@ -63,26 +63,26 @@ function clear_waitlist(digest: string) } function flush(evict_all: bool) - { - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - if ( ! evict_all && ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } + { + local current: string; + for ( unused_index in deque ) + { + current = deque[tail]; + local info = records[current]; + if ( ! evict_all && ! info?$notary ) + break; + Log::write(SSL::LOG, info); + delete deque[tail]; + delete records[current]; + ++tail; + } + } function lookup_cert_hash(uid: string, digest: string) - { - # Add the record ID to the list of waiting IDs for this digest. - local waits_already = digest in waiting; - if ( ! waits_already ) + j{ + j# Add the record ID to the list of waiting IDs for this digest. + jlocal waits_already = digest in waiting; + jif ( ! waits_already ) waiting[digest] = vector(); waiting[digest][|waiting[digest]|] = uid; if ( waits_already ) @@ -135,14 +135,14 @@ function push(info: SSL::Info) if ( ! info?$sha1_digest ) return; - local digest = info$sha1_digest; - if ( info$sha1_digest in notary_cache ) - info$notary = notary_cache[digest]; - else - lookup_cert_hash(info$uid, digest); - records[info$uid] = info; - deque[head] = info$uid; - ++head; + local digest = info$sha1_digest; + if ( info$sha1_digest in notary_cache ) + info$notary = notary_cache[digest]; + else + lookup_cert_hash(info$uid, digest); + records[info$uid] = info; + deque[head] = info$uid; + ++head; } event x509_certificate(c: connection, is_orig: bool, cert: X509, @@ -151,12 +151,12 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, if ( is_orig || chain_idx != 0 || ! c?$ssl ) return; - c$ssl$sha1_digest = sha1_hash(der_cert); - } + c$ssl$sha1_digest = sha1_hash(der_cert); + } event bro_done() { if ( |deque| == 0 ) return; - flush(T); + flush(T); } From 9e81342c92d1c56b4030b3a29e97c8c977155348 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Sat, 22 Dec 2012 20:30:17 -0800 Subject: [PATCH 05/49] Implement delay-token style SSL logging. This commit moves the notary script into the policy directory, along with some architectural changes: the main SSL script now has functionality to add and remove tokens for a given record. When adding a token, the script delays the logging until the token has been removed or until the record exceeds a maximum delay time. As before, the base SSL script stores all records sequentially and buffers even non-delayed records for the sake of having an ordered log file. If this turns out to be not so important, we can easily revert to a simpler logic. (This is still WiP, some debuggin statements still linger.) --- scripts/base/protocols/ssl/__load__.bro | 62 ------- scripts/base/protocols/ssl/main.bro | 167 +++++++++++++++++- .../{base => policy}/protocols/ssl/notary.bro | 109 +++--------- scripts/site/local.bro | 4 + 4 files changed, 192 insertions(+), 150 deletions(-) rename scripts/{base => policy}/protocols/ssl/notary.bro (53%) diff --git a/scripts/base/protocols/ssl/__load__.bro b/scripts/base/protocols/ssl/__load__.bro index e97d1681fc..239438047c 100644 --- a/scripts/base/protocols/ssl/__load__.bro +++ b/scripts/base/protocols/ssl/__load__.bro @@ -1,65 +1,3 @@ @load ./consts - -module SSL; - -export { - redef enum Log::ID += { LOG }; - - type Info: record { - ## Time when the SSL connection was first detected. - ts: time &log; - ## Unique ID for the connection. - uid: string &log; - ## The connection's 4-tuple of endpoint addresses/ports. - id: conn_id &log; - ## SSL/TLS version that the server offered. - version: string &log &optional; - ## SSL/TLS cipher suite that the server chose. - cipher: string &log &optional; - ## Value of the Server Name Indicator SSL/TLS extension. It - ## indicates the server name that the client was requesting. - server_name: string &log &optional; - ## Session ID offered by the client for session resumption. - session_id: string &log &optional; - ## Subject of the X.509 certificate offered by the server. - subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the server. - issuer_subject: string &log &optional; - ## NotValidBefore field value from the server certificate. - not_valid_before: time &log &optional; - ## NotValidAfter field value from the server certificate. - not_valid_after: time &log &optional; - ## Last alert that was seen during the connection. - last_alert: string &log &optional; - - ## Subject of the X.509 certificate offered by the client. - client_subject: string &log &optional; - ## Subject of the signer of the X.509 certificate offered by the client. - client_issuer_subject: string &log &optional; - - ## Full binary server certificate stored in DER format. - cert: string &optional; - ## Chain of certificates offered by the server to validate its - ## complete signing chain. - cert_chain: vector of string &optional; - - ## Full binary client certificate stored in DER format. - client_cert: string &optional; - ## Chain of certificates offered by the client to validate its - ## complete signing chain. - client_cert_chain: vector of string &optional; - - ## The analyzer ID used for the analyzer instance attached - ## to each connection. It is not used for logging since it's a - ## meaningless arbitrary number. - analyzer_id: count &optional; - }; -} - -redef record connection += { - ssl: Info &optional; -}; - -@load ./notary @load ./main @load ./mozilla-ca-list diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index f02602235c..642d93eb96 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -6,6 +6,63 @@ module SSL; export { + redef enum Log::ID += { LOG }; + + type Info: record { + ## Time when the SSL connection was first detected. + ts: time &log; + ## Unique ID for the connection. + uid: string &log; + ## The connection's 4-tuple of endpoint addresses/ports. + id: conn_id &log; + ## SSL/TLS version that the server offered. + version: string &log &optional; + ## SSL/TLS cipher suite that the server chose. + cipher: string &log &optional; + ## Value of the Server Name Indicator SSL/TLS extension. It + ## indicates the server name that the client was requesting. + server_name: string &log &optional; + ## Session ID offered by the client for session resumption. + session_id: string &log &optional; + ## Subject of the X.509 certificate offered by the server. + subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the server. + issuer_subject: string &log &optional; + ## NotValidBefore field value from the server certificate. + not_valid_before: time &log &optional; + ## NotValidAfter field value from the server certificate. + not_valid_after: time &log &optional; + ## Last alert that was seen during the connection. + last_alert: string &log &optional; + + ## Subject of the X.509 certificate offered by the client. + client_subject: string &log &optional; + ## Subject of the signer of the X.509 certificate offered by the client. + client_issuer_subject: string &log &optional; + + ## Full binary server certificate stored in DER format. + cert: string &optional; + ## Chain of certificates offered by the server to validate its + ## complete signing chain. + cert_chain: vector of string &optional; + + ## Full binary client certificate stored in DER format. + client_cert: string &optional; + ## Chain of certificates offered by the client to validate its + ## complete signing chain. + client_cert_chain: vector of string &optional; + + ## The analyzer ID used for the analyzer instance attached + ## to each connection. It is not used for logging since it's a + ## meaningless arbitrary number. + analyzer_id: count &optional; + + ## Adding a string "token" to this set will cause the SSL script + ## to delay logging the record until either the token has been removed or + ## the record has been delayed for :bro:id:`SSL::max_log_delay`. + delay_tokens: set[string] &optional; + }; + ## The default root CA bundle. By loading the ## mozilla-ca-list.bro script it will be set to Mozilla's root CA list. const root_certs: table[string] of string = {} &redef; @@ -20,11 +77,24 @@ export { ## utility. const openssl_util = "openssl" &redef; + ## The maximum amount of time a plugin can delay records from being logged. + const max_log_delay = 15secs &redef; + + ## TODO: document. + global add_delayed_record: function(info: Info, token: string); + + ## TODO: document. + global clear_delayed_record: function(uid: string, token: string) : Info; + ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. global log_ssl: event(rec: Info); } +redef record connection += { + ssl: Info &optional; +}; + event bro_init() &priority=5 { Log::create_stream(SSL::LOG, [$columns=Info, $ev=log_ssl]); @@ -59,6 +129,20 @@ redef likely_server_ports += { 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp }; +# The buffered SSL log records. +global records: table[string] of Info; + +# A double-ended queue that determines the log record order in which logs have +# to written out to disk. +global deque: table[count] of string; + +# The top-most deque index. +global head = 0; + +# The bottom deque index that points to the next record to be flushed as soon +# as the notary response arrives. +global tail = 0; + function set_session(c: connection) { if ( ! c?$ssl ) @@ -66,16 +150,69 @@ function set_session(c: connection) $client_cert_chain=vector()]; } +function add_delayed_record(info: Info, token: string) + { + if ( info$uid in records ) + { + print fmt("----- ignoring duplicate %s -----", info$uid); + return; + } + + records[info$uid] = info; + deque[head] = info$uid; + ++head; + + info$delay_tokens = set(); + add info$delay_tokens[token]; + } + +function clear_delayed_record(uid: string, token: string) : Info + { + local info = records[uid]; + delete info$delay_tokens[token]; + return info; + } + +global log_record: function(info: Info); + +event delay_logging(info: Info) + { + log_record(info); + } + +function log_record(info: Info) + { + for ( unused_index in records ) + { + if ( head == tail ) + return; + local uid = deque[tail]; + if ( |records[uid]$delay_tokens| > 0 ) + { + if ( info$ts + max_log_delay > network_time() ) + { + schedule 1sec { delay_logging(info) }; + return; + } + else + { + event reporter_info( + network_time(), + fmt("SSL delay tokens not released in time (%s)", + info$delay_tokens), + ""); + } + } + Log::write(SSL::LOG, records[uid]); + delete records[uid]; + delete deque[tail]; + ++tail; + } + } + function finish(c: connection) { -# TODO: This dummy flag merely exists to check whether the notary script is -# loaded. There's probably a better way to incorporate the notary. -@ifdef( Notary::enabled ) - Notary::push(c$ssl); -@else - Log::write(SSL::LOG, c$ssl); -@endif - + log_record(c$ssl); if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); delete c$ssl; @@ -179,3 +316,17 @@ event protocol_violation(c: connection, atype: count, aid: count, if ( c?$ssl ) finish(c); } + +event bro_done() + { + if ( |records| == 0 ) + return; + for ( unused_index in records ) + { + local uid = deque[tail]; + Log::write(SSL::LOG, records[uid]); + delete records[uid]; + delete deque[tail]; + ++tail; + } + } diff --git a/scripts/base/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro similarity index 53% rename from scripts/base/protocols/ssl/notary.bro rename to scripts/policy/protocols/ssl/notary.bro index 4ae03fd3f8..f720359aa6 100644 --- a/scripts/base/protocols/ssl/notary.bro +++ b/scripts/policy/protocols/ssl/notary.bro @@ -1,10 +1,6 @@ module Notary; export { - # Flag to tell the SSL analysis script that it should buffer logs instead of - # flushing them directly. - const enabled = T; - ## A response from the ICSI certificate notary. type Response: record { first_seen: count &log &optional; @@ -13,83 +9,66 @@ export { valid: bool &log &optional; }; - ## Hands over an SSL record to the Notary module. This is an ownership - ## transfer, i.e., the caller does not need to call Log::write on this record - ## anymore. - global push: function(info: SSL::Info); - ## The notary domain to query. const domain = "notary.icsi.berkeley.edu" &redef; } redef record SSL::Info += { - sha1_digest: string &optional; + sha1: string &log &optional; notary: Response &log &optional; }; # The DNS cache of notary responses. global notary_cache: table[string] of Response &create_expire = 1 hr; -# The buffered SSL log records. -global records: table[string] of SSL::Info; - # The records that wait for a notary response identified by the cert digest. # Each digest refers to a list of connection UIDs which are updated when a DNS # reply arrives asynchronously. global waiting: table[string] of vector of string; -# A double-ended queue that determines the log record order in which logs have -# to written out to disk. -global deque: table[count] of string; - -# The top-most deque index. -global head = 0; - -# The bottom deque index that points to the next record to be flushed as soon -# as the notary response arrives. -global tail = 0; - function clear_waitlist(digest: string) { + print "----- clearing waitlist -----"; if ( digest in waiting ) { for ( i in waiting[digest] ) { - local uid = waiting[digest][i]; - records[uid]$notary = []; + print fmt("----- retrieving %s -----", waiting[digest][i]); + local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); + info$notary = []; } delete waiting[digest]; } } -function flush(evict_all: bool) +event x509_certificate(c: connection, is_orig: bool, cert: X509, + chain_idx: count, chain_len: count, der_cert: string) { - local current: string; - for ( unused_index in deque ) - { - current = deque[tail]; - local info = records[current]; - if ( ! evict_all && ! info?$notary ) - break; - Log::write(SSL::LOG, info); - delete deque[tail]; - delete records[current]; - ++tail; - } - } + if ( is_orig || chain_idx != 0 || ! c?$ssl ) + return; -function lookup_cert_hash(uid: string, digest: string) - j{ - j# Add the record ID to the list of waiting IDs for this digest. - jlocal waits_already = digest in waiting; - jif ( ! waits_already ) + local digest = sha1_hash(der_cert); + c$ssl$sha1 = digest; + + if ( digest in notary_cache ) + { + c$ssl$notary = notary_cache[digest]; + return; + } + + print fmt("----- adding %s -----", c$ssl$uid); + SSL::add_delayed_record(c$ssl, "notary"); + + local waits_already = digest in waiting; + if ( ! waits_already ) waiting[digest] = vector(); - waiting[digest][|waiting[digest]|] = uid; + waiting[digest][|waiting[digest]|] = c$uid; if ( waits_already ) return; when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) ) { + print fmt("----- when for %s: %s -----", digest, str); # Cache every response for a digest. notary_cache[digest] = []; @@ -122,41 +101,11 @@ function lookup_cert_hash(uid: string, digest: string) if ( digest in waiting ) { for ( i in waiting[digest] ) - records[waiting[digest][i]]$notary = r; + { + local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); + info$notary = r; + } delete waiting[digest]; } - - flush(F); } } - -function push(info: SSL::Info) - { - if ( ! info?$sha1_digest ) - return; - - local digest = info$sha1_digest; - if ( info$sha1_digest in notary_cache ) - info$notary = notary_cache[digest]; - else - lookup_cert_hash(info$uid, digest); - records[info$uid] = info; - deque[head] = info$uid; - ++head; - } - -event x509_certificate(c: connection, is_orig: bool, cert: X509, - chain_idx: count, chain_len: count, der_cert: string) - { - if ( is_orig || chain_idx != 0 || ! c?$ssl ) - return; - - c$ssl$sha1_digest = sha1_hash(der_cert); - } - -event bro_done() - { - if ( |deque| == 0 ) - return; - flush(T); - } diff --git a/scripts/site/local.bro b/scripts/site/local.bro index db1a786839..a080300185 100644 --- a/scripts/site/local.bro +++ b/scripts/site/local.bro @@ -56,6 +56,10 @@ redef Software::vulnerable_versions += { # This script enables SSL/TLS certificate validation. @load protocols/ssl/validate-certs +# This script checks each SSL certificate hash against the ICSI certificate +# notary service. +@load protocols/ssl/notary + # If you have libGeoIP support built in, do some geographic detections and # logging for SSH traffic. @load protocols/ssh/geo-data From 7ff15f4599e36c300515955b5f9664cbe5480980 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 24 Dec 2012 22:57:15 -0800 Subject: [PATCH 06/49] Simplify delayed logging of SSL records. --- scripts/base/protocols/ssl/main.bro | 54 +++++++++---------------- scripts/policy/protocols/ssl/notary.bro | 46 +++++++++------------ 2 files changed, 39 insertions(+), 61 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 642d93eb96..249f8b9609 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -80,11 +80,13 @@ export { ## The maximum amount of time a plugin can delay records from being logged. const max_log_delay = 15secs &redef; - ## TODO: document. - global add_delayed_record: function(info: Info, token: string); + ## Delays an SSL record for a specific token: the record will not be logged + ## as longs the token exists or until :bro:id:`SSL::max_log_delay` elapses. + global delay_record: function(info: Info, token: string); - ## TODO: document. - global clear_delayed_record: function(uid: string, token: string) : Info; + ## Undelays an SSL record for a previously inserted token, allowing the + ## record to be logged. + global undelay_record: function(info: Info, token: string); ## Event that can be handled to access the SSL ## record as it is sent on to the logging framework. @@ -129,12 +131,9 @@ redef likely_server_ports += { 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp }; -# The buffered SSL log records. -global records: table[string] of Info; - # A double-ended queue that determines the log record order in which logs have # to written out to disk. -global deque: table[count] of string; +global deque: table[count] of Info; # The top-most deque index. global head = 0; @@ -150,27 +149,19 @@ function set_session(c: connection) $client_cert_chain=vector()]; } -function add_delayed_record(info: Info, token: string) +function delay_record(info: Info, token: string) { - if ( info$uid in records ) - { - print fmt("----- ignoring duplicate %s -----", info$uid); - return; - } - - records[info$uid] = info; - deque[head] = info$uid; - ++head; - info$delay_tokens = set(); add info$delay_tokens[token]; + + deque[head] = info; + ++head; } -function clear_delayed_record(uid: string, token: string) : Info +function undelay_record(info: Info, token: string) { - local info = records[uid]; - delete info$delay_tokens[token]; - return info; + if ( token in info$delay_tokens ) + delete info$delay_tokens[token]; } global log_record: function(info: Info); @@ -182,12 +173,11 @@ event delay_logging(info: Info) function log_record(info: Info) { - for ( unused_index in records ) + for ( unused_index in deque ) { if ( head == tail ) return; - local uid = deque[tail]; - if ( |records[uid]$delay_tokens| > 0 ) + if ( |deque[tail]$delay_tokens| > 0 ) { if ( info$ts + max_log_delay > network_time() ) { @@ -203,8 +193,7 @@ function log_record(info: Info) ""); } } - Log::write(SSL::LOG, records[uid]); - delete records[uid]; + Log::write(SSL::LOG, deque[tail]); delete deque[tail]; ++tail; } @@ -215,7 +204,6 @@ function finish(c: connection) log_record(c$ssl); if ( disable_analyzer_after_detection && c?$ssl && c$ssl?$analyzer_id ) disable_analyzer(c$id, c$ssl$analyzer_id); - delete c$ssl; } event ssl_client_hello(c: connection, version: count, possible_ts: time, session_id: string, ciphers: count_set) &priority=5 @@ -319,13 +307,11 @@ event protocol_violation(c: connection, atype: count, aid: count, event bro_done() { - if ( |records| == 0 ) + if ( |deque| == 0 ) return; - for ( unused_index in records ) + for ( unused_index in deque ) { - local uid = deque[tail]; - Log::write(SSL::LOG, records[uid]); - delete records[uid]; + Log::write(SSL::LOG, deque[tail]); delete deque[tail]; ++tail; } diff --git a/scripts/policy/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro index f720359aa6..40afc977c2 100644 --- a/scripts/policy/protocols/ssl/notary.bro +++ b/scripts/policy/protocols/ssl/notary.bro @@ -24,20 +24,15 @@ global notary_cache: table[string] of Response &create_expire = 1 hr; # The records that wait for a notary response identified by the cert digest. # Each digest refers to a list of connection UIDs which are updated when a DNS # reply arrives asynchronously. -global waiting: table[string] of vector of string; +global wait_list: table[string] of vector of SSL::Info; function clear_waitlist(digest: string) { - print "----- clearing waitlist -----"; - if ( digest in waiting ) + if ( digest in wait_list ) { - for ( i in waiting[digest] ) - { - print fmt("----- retrieving %s -----", waiting[digest][i]); - local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); - info$notary = []; - } - delete waiting[digest]; + for ( i in wait_list[digest] ) + SSL::undelay_record(wait_list[digest][i], "notary"); + delete wait_list[digest]; } } @@ -56,31 +51,27 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, return; } - print fmt("----- adding %s -----", c$ssl$uid); - SSL::add_delayed_record(c$ssl, "notary"); + SSL::delay_record(c$ssl, "notary"); - local waits_already = digest in waiting; + local waits_already = digest in wait_list; if ( ! waits_already ) - waiting[digest] = vector(); - waiting[digest][|waiting[digest]|] = c$uid; + wait_list[digest] = vector(); + wait_list[digest][|wait_list[digest]|] = c$ssl; if ( waits_already ) return; when ( local str = lookup_hostname_txt(fmt("%s.%s", digest, domain)) ) { - print fmt("----- when for %s: %s -----", digest, str); - # Cache every response for a digest. notary_cache[digest] = []; # Parse notary answer. - if ( str == "" ) + if ( str == "" ) # NXDOMAIN { - # TODO: Should we handle NXDOMAIN separately? clear_waitlist(digest); return; } local fields = split(str, / /); - if ( |fields| != 5 ) # version 1 has 5 fields. + if ( |fields| != 5 ) # version 1 has 5 fields. { clear_waitlist(digest); return; @@ -97,15 +88,16 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, r$times_seen = to_count(split(fields[4], /=/)[2]); r$valid = split(fields[5], /=/)[2] == "1"; - # Assign notary answer to all waiting records. - if ( digest in waiting ) + # Assign notary answer to all records waiting for this digest. + if ( digest in wait_list ) { - for ( i in waiting[digest] ) - { - local info = SSL::clear_delayed_record(waiting[digest][i], "notary"); + for ( i in wait_list[digest] ) + { + local info = wait_list[digest][i]; + SSL::undelay_record(info, "notary"); info$notary = r; - } - delete waiting[digest]; + } + delete wait_list[digest]; } } } From 32a0ead698c467a9a89f5b58687db2f7e2812d3c Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Mon, 24 Dec 2012 23:06:56 -0800 Subject: [PATCH 07/49] Give log buffer the correct name. --- scripts/base/protocols/ssl/main.bro | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 249f8b9609..e798695948 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -131,15 +131,13 @@ redef likely_server_ports += { 989/tcp, 990/tcp, 992/tcp, 993/tcp, 995/tcp, 5223/tcp }; -# A double-ended queue that determines the log record order in which logs have -# to written out to disk. -global deque: table[count] of Info; +# A queue that buffers log records. +global queue: table[count] of Info; -# The top-most deque index. +# The top queue index where records are added. global head = 0; -# The bottom deque index that points to the next record to be flushed as soon -# as the notary response arrives. +# The bottom queue index that points to the next record to be flushed. global tail = 0; function set_session(c: connection) @@ -154,7 +152,7 @@ function delay_record(info: Info, token: string) info$delay_tokens = set(); add info$delay_tokens[token]; - deque[head] = info; + queue[head] = info; ++head; } @@ -173,11 +171,11 @@ event delay_logging(info: Info) function log_record(info: Info) { - for ( unused_index in deque ) + for ( unused_index in queue ) { if ( head == tail ) return; - if ( |deque[tail]$delay_tokens| > 0 ) + if ( |queue[tail]$delay_tokens| > 0 ) { if ( info$ts + max_log_delay > network_time() ) { @@ -193,8 +191,8 @@ function log_record(info: Info) ""); } } - Log::write(SSL::LOG, deque[tail]); - delete deque[tail]; + Log::write(SSL::LOG, queue[tail]); + delete queue[tail]; ++tail; } } @@ -307,12 +305,12 @@ event protocol_violation(c: connection, atype: count, aid: count, event bro_done() { - if ( |deque| == 0 ) + if ( |queue| == 0 ) return; - for ( unused_index in deque ) + for ( unused_index in queue ) { - Log::write(SSL::LOG, deque[tail]); - delete deque[tail]; + Log::write(SSL::LOG, queue[tail]); + delete queue[tail]; ++tail; } } From 8695053e278158ce40cb2a3d6aa04ff60653ca0e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 11 Jan 2013 11:43:15 -0600 Subject: [PATCH 08/49] Disable automatic case fallthrough in switch stmts. Addresses #754. Case bodies now don't require a "break" statement to prevent fallthrough to case bodies below. Empty case bodies generate an error message at parse-time to help indicate the absence of automatic fallthrough; to associate multiple values with a case, use "case 1, 2:" instead of "case 1: case 2:". --- src/Stmt.cc | 22 +++++++------------ src/Stmt.h | 6 +++++ testing/btest/core/leaks/switch-statement.bro | 17 ++++++-------- testing/btest/language/switch-statement.bro | 17 ++++++-------- 4 files changed, 28 insertions(+), 34 deletions(-) diff --git a/src/Stmt.cc b/src/Stmt.cc index 2cd7117ddb..3e37256338 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -701,6 +701,9 @@ SwitchStmt::SwitchStmt(Expr* index, case_list* arg_cases) : const Case* c = (*cases)[i]; const ListExpr* le = c->Cases(); + if ( ! c->Body() || c->Body()->AsStmtList()->Stmts().length() == 0 ) + c->Error("empty case label body does nothing"); + if ( le ) { if ( ! le->Type()->AsTypeList()->AllMatch(e->Type(), false) ) @@ -795,22 +798,13 @@ Val* SwitchStmt::DoExec(Frame* f, Val* v, stmt_flow_type& flow) const if ( matching_label_idx == -1 ) return 0; - for ( int i = matching_label_idx; i < cases->length(); ++i ) - { - const Case* c = (*cases)[i]; + const Case* c = (*cases)[matching_label_idx]; + flow = FLOW_NEXT; + rval = c->Body()->Exec(f, flow); + + if ( flow == FLOW_BREAK ) flow = FLOW_NEXT; - rval = c->Body()->Exec(f, flow); - - if ( flow == FLOW_BREAK ) - { - flow = FLOW_NEXT; - break; - } - - if ( flow == FLOW_RETURN ) - break; - } return rval; } diff --git a/src/Stmt.h b/src/Stmt.h index 497d7c97b1..32be7f33fc 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -46,6 +46,12 @@ public: return (StmtList*) this; } + const StmtList* AsStmtList() const + { + CHECK_TAG(tag, STMT_LIST, "Stmt::AsStmtList", stmt_name) + return (const StmtList*) this; + } + ForStmt* AsForStmt() { CHECK_TAG(tag, STMT_FOR, "Stmt::AsForStmt", stmt_name) diff --git a/testing/btest/core/leaks/switch-statement.bro b/testing/btest/core/leaks/switch-statement.bro index 24829006b5..6fbdb0d54a 100644 --- a/testing/btest/core/leaks/switch-statement.bro +++ b/testing/btest/core/leaks/switch-statement.bro @@ -157,6 +157,7 @@ function switch_break(v: count): string case 2: rval += "testing"; break; + rval += "ERROR"; case 3: rval += "tested"; } @@ -251,17 +252,17 @@ event new_connection(c: connection) test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testtestingreturn" ); + test_switch( switch_break(1) , "testreturn" ); test_switch( switch_break(2) , "testingreturn" ); test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "123dr" ); - test_switch( switch_default(2) , "23dr" ); - test_switch( switch_default(3) , "3dr" ); + test_switch( switch_default(1) , "1r" ); + test_switch( switch_default(2) , "2r" ); + test_switch( switch_default(3) , "3r" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1d2r" ); + test_switch( switch_default_placement(1) , "1r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "d2r" ); + test_switch( switch_default_placement(4) , "dr" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -271,16 +272,12 @@ event new_connection(c: connection) switch ( v[i] ) { case 1, 2: expect = "1,2"; - break; case 3, 4, 5: expect = "3,4,5"; - break; case 6, 7, 8, 9: expect = "6,7,8,9"; - break; default: expect = "n/a"; - break; } test_switch( switch_case_list(v[i]) , expect ); } diff --git a/testing/btest/language/switch-statement.bro b/testing/btest/language/switch-statement.bro index b8c34f77dc..dcf2a4c041 100644 --- a/testing/btest/language/switch-statement.bro +++ b/testing/btest/language/switch-statement.bro @@ -152,6 +152,7 @@ function switch_break(v: count): string case 2: rval += "testing"; break; + rval += "ERROR"; case 3: rval += "tested"; } @@ -246,17 +247,17 @@ event bro_init() test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testtestingreturn" ); + test_switch( switch_break(1) , "testreturn" ); test_switch( switch_break(2) , "testingreturn" ); test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "123dr" ); - test_switch( switch_default(2) , "23dr" ); - test_switch( switch_default(3) , "3dr" ); + test_switch( switch_default(1) , "1r" ); + test_switch( switch_default(2) , "2r" ); + test_switch( switch_default(3) , "3r" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1d2r" ); + test_switch( switch_default_placement(1) , "1r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "d2r" ); + test_switch( switch_default_placement(4) , "dr" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -266,16 +267,12 @@ event bro_init() switch ( v[i] ) { case 1, 2: expect = "1,2"; - break; case 3, 4, 5: expect = "3,4,5"; - break; case 6, 7, 8, 9: expect = "6,7,8,9"; - break; default: expect = "n/a"; - break; } test_switch( switch_case_list(v[i]) , expect ); } From a635f96518ddeefe553ef4fd98213bc07ad63226 Mon Sep 17 00:00:00 2001 From: Matthias Vallentin Date: Tue, 15 Jan 2013 14:56:02 -0800 Subject: [PATCH 09/49] Small cosmetic changes. --- scripts/policy/protocols/ssl/notary.bro | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/scripts/policy/protocols/ssl/notary.bro b/scripts/policy/protocols/ssl/notary.bro index 40afc977c2..0dc96e3570 100644 --- a/scripts/policy/protocols/ssl/notary.bro +++ b/scripts/policy/protocols/ssl/notary.bro @@ -24,15 +24,15 @@ global notary_cache: table[string] of Response &create_expire = 1 hr; # The records that wait for a notary response identified by the cert digest. # Each digest refers to a list of connection UIDs which are updated when a DNS # reply arrives asynchronously. -global wait_list: table[string] of vector of SSL::Info; +global waitlist: table[string] of vector of SSL::Info; function clear_waitlist(digest: string) { - if ( digest in wait_list ) + if ( digest in waitlist ) { - for ( i in wait_list[digest] ) - SSL::undelay_record(wait_list[digest][i], "notary"); - delete wait_list[digest]; + for ( i in waitlist[digest] ) + SSL::undelay_record(waitlist[digest][i], "notary"); + delete waitlist[digest]; } } @@ -53,10 +53,10 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, SSL::delay_record(c$ssl, "notary"); - local waits_already = digest in wait_list; + local waits_already = digest in waitlist; if ( ! waits_already ) - wait_list[digest] = vector(); - wait_list[digest][|wait_list[digest]|] = c$ssl; + waitlist[digest] = vector(); + waitlist[digest][|waitlist[digest]|] = c$ssl; if ( waits_already ) return; @@ -89,15 +89,15 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, r$valid = split(fields[5], /=/)[2] == "1"; # Assign notary answer to all records waiting for this digest. - if ( digest in wait_list ) + if ( digest in waitlist ) { - for ( i in wait_list[digest] ) + for ( i in waitlist[digest] ) { - local info = wait_list[digest][i]; + local info = waitlist[digest][i]; SSL::undelay_record(info, "notary"); info$notary = r; } - delete wait_list[digest]; + delete waitlist[digest]; } } } From be71a42f4c9cdde69b74f18203db062dbc18dea2 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 16 Jan 2013 16:17:17 -0600 Subject: [PATCH 10/49] Add "fallthrough" keyword, require a flow statement to end case blocks. Case blocks in switch statements now must end in a break, return, or fallthrough statement to give best mix of safety, readability, and flexibility. The new fallthrough keyword explicitly allows control to be passed to the next case block in a switch statement. Addresses #754. --- src/SerialTypes.h | 1 + src/Stmt.cc | 83 +++++++++++++++++-- src/Stmt.h | 18 +++- src/StmtEnums.h | 4 +- src/parse.y | 12 ++- src/scan.l | 1 + testing/btest/core/leaks/switch-statement.bro | 32 ++++--- testing/btest/language/switch-statement.bro | 32 ++++--- 8 files changed, 150 insertions(+), 33 deletions(-) diff --git a/src/SerialTypes.h b/src/SerialTypes.h index e103c1c40e..723badab1e 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -171,6 +171,7 @@ SERIAL_STMT(EVENT_BODY_LIST, 16) SERIAL_STMT(INIT_STMT, 17) SERIAL_STMT(NULL_STMT, 18) SERIAL_STMT(WHEN_STMT, 19) +SERIAL_STMT(FALLTHROUGH_STMT, 20) #define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE) SERIAL_TYPE(BRO_TYPE, 1) diff --git a/src/Stmt.cc b/src/Stmt.cc index 3e37256338..cc506db985 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t) "print", "event", "expr", "if", "when", "switch", "for", "next", "break", "return", "add", "delete", "list", "bodylist", - "", + "", "fallthrough", "null", }; @@ -584,6 +584,29 @@ bool IfStmt::DoUnserialize(UnserialInfo* info) return s2 != 0; } +static BroStmtTag get_last_stmt_tag(const Stmt* stmt) + { + if ( ! stmt ) return STMT_NULL; + + if ( stmt->Tag() != STMT_LIST ) return stmt->Tag(); + + const StmtList* stmts = stmt->AsStmtList(); + int len = stmts->Stmts().length(); + + if ( len == 0 ) return STMT_LIST; + + return get_last_stmt_tag(stmts->Stmts()[len - 1]); + } + +Case::Case(ListExpr* c, Stmt* arg_s) + : cases(simplify_expr_list(c, SIMPLIFY_GENERAL)), s(arg_s) + { + BroStmtTag t = get_last_stmt_tag(Body()); + + if ( t != STMT_BREAK && t != STMT_FALLTHROUGH && t != STMT_RETURN ) + Error("case block must end in break/fallthrough/return statement"); + } + Case::~Case() { Unref(cases); @@ -701,9 +724,6 @@ SwitchStmt::SwitchStmt(Expr* index, case_list* arg_cases) : const Case* c = (*cases)[i]; const ListExpr* le = c->Cases(); - if ( ! c->Body() || c->Body()->AsStmtList()->Stmts().length() == 0 ) - c->Error("empty case label body does nothing"); - if ( le ) { if ( ! le->Type()->AsTypeList()->AllMatch(e->Type(), false) ) @@ -798,12 +818,18 @@ Val* SwitchStmt::DoExec(Frame* f, Val* v, stmt_flow_type& flow) const if ( matching_label_idx == -1 ) return 0; - const Case* c = (*cases)[matching_label_idx]; + for ( int i = matching_label_idx; i < cases->length(); ++i ) + { + const Case* c = (*cases)[i]; - flow = FLOW_NEXT; - rval = c->Body()->Exec(f, flow); + flow = FLOW_NEXT; + rval = c->Body()->Exec(f, flow); - if ( flow == FLOW_BREAK ) + if ( flow == FLOW_BREAK || flow == FLOW_RETURN ) + break; + } + + if ( flow != FLOW_RETURN ) flow = FLOW_NEXT; return rval; @@ -1461,6 +1487,47 @@ bool BreakStmt::DoUnserialize(UnserialInfo* info) return true; } +Val* FallthroughStmt::Exec(Frame* /* f */, stmt_flow_type& flow) const + { + RegisterAccess(); + flow = FLOW_FALLTHROUGH; + return 0; + } + +int FallthroughStmt::IsPure() const + { + return 1; + } + +void FallthroughStmt::Describe(ODesc* d) const + { + Stmt::Describe(d); + Stmt::DescribeDone(d); + } + +TraversalCode FallthroughStmt::Traverse(TraversalCallback* cb) const + { + TraversalCode tc = cb->PreStmt(this); + HANDLE_TC_STMT_PRE(tc); + + tc = cb->PostStmt(this); + HANDLE_TC_STMT_POST(tc); + } + +IMPLEMENT_SERIAL(FallthroughStmt, SER_FALLTHROUGH_STMT); + +bool FallthroughStmt::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_FALLTHROUGH_STMT, Stmt); + return true; + } + +bool FallthroughStmt::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(Stmt); + return true; + } + ReturnStmt::ReturnStmt(Expr* arg_e) : ExprStmt(STMT_RETURN, arg_e) { Scope* s = current_scope(); diff --git a/src/Stmt.h b/src/Stmt.h index 32be7f33fc..32b90b4190 100644 --- a/src/Stmt.h +++ b/src/Stmt.h @@ -195,8 +195,7 @@ protected: class Case : public BroObj { public: - Case(ListExpr* c, Stmt* arg_s) : - cases(simplify_expr_list(c,SIMPLIFY_GENERAL)), s(arg_s) { } + Case(ListExpr* c, Stmt* arg_s); ~Case(); const ListExpr* Cases() const { return cases; } @@ -371,6 +370,21 @@ protected: DECLARE_SERIAL(BreakStmt); }; +class FallthroughStmt : public Stmt { +public: + FallthroughStmt() : Stmt(STMT_FALLTHROUGH) { } + + Val* Exec(Frame* f, stmt_flow_type& flow) const; + int IsPure() const; + + void Describe(ODesc* d) const; + + TraversalCode Traverse(TraversalCallback* cb) const; + +protected: + DECLARE_SERIAL(FallthroughStmt); +}; + class ReturnStmt : public ExprStmt { public: ReturnStmt(Expr* e); diff --git a/src/StmtEnums.h b/src/StmtEnums.h index f431e3fea1..1114816a93 100644 --- a/src/StmtEnums.h +++ b/src/StmtEnums.h @@ -16,6 +16,7 @@ typedef enum { STMT_ADD, STMT_DELETE, STMT_LIST, STMT_EVENT_BODY_LIST, STMT_INIT, + STMT_FALLTHROUGH, STMT_NULL #define NUM_STMTS (int(STMT_NULL) + 1) } BroStmtTag; @@ -24,7 +25,8 @@ typedef enum { FLOW_NEXT, // continue on to next statement FLOW_LOOP, // go to top of loop FLOW_BREAK, // break out of loop - FLOW_RETURN // return from function + FLOW_RETURN, // return from function + FLOW_FALLTHROUGH// fall through to next switch case } stmt_flow_type; extern const char* stmt_name(BroStmtTag t); diff --git a/src/parse.y b/src/parse.y index 090786647e..7ce1174595 100644 --- a/src/parse.y +++ b/src/parse.y @@ -8,8 +8,8 @@ %token TOK_ATENDIF TOK_ATELSE TOK_ATIF TOK_ATIFDEF TOK_ATIFNDEF %token TOK_BOOL TOK_BREAK TOK_CASE TOK_CONST %token TOK_CONSTANT TOK_COPY TOK_COUNT TOK_COUNTER TOK_DEFAULT TOK_DELETE -%token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FILE TOK_FOR -%token TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT +%token TOK_DOUBLE TOK_ELSE TOK_ENUM TOK_EVENT TOK_EXPORT TOK_FALLTHROUGH +%token TOK_FILE TOK_FOR TOK_FUNCTION TOK_GLOBAL TOK_HOOK TOK_ID TOK_IF TOK_INT %token TOK_INTERVAL TOK_LIST TOK_LOCAL TOK_MODULE %token TOK_NEXT TOK_OF TOK_OPAQUE TOK_PATTERN TOK_PATTERN_TEXT %token TOK_PORT TOK_PRINT TOK_RECORD TOK_REDEF @@ -1436,6 +1436,14 @@ stmt: brofiler.AddStmt($$); } + | TOK_FALLTHROUGH ';' opt_no_test + { + set_location(@1, @2); + $$ = new FallthroughStmt; + if ( ! $3 ) + brofiler.AddStmt($$); + } + | TOK_RETURN ';' opt_no_test { set_location(@1, @2); diff --git a/src/scan.l b/src/scan.l index efcd273e36..ffbc125728 100644 --- a/src/scan.l +++ b/src/scan.l @@ -282,6 +282,7 @@ else return TOK_ELSE; enum return TOK_ENUM; event return TOK_EVENT; export return TOK_EXPORT; +fallthrough return TOK_FALLTHROUGH; file return TOK_FILE; for return TOK_FOR; function return TOK_FUNCTION; diff --git a/testing/btest/core/leaks/switch-statement.bro b/testing/btest/core/leaks/switch-statement.bro index 6fbdb0d54a..845915ae8a 100644 --- a/testing/btest/core/leaks/switch-statement.bro +++ b/testing/btest/core/leaks/switch-statement.bro @@ -148,18 +148,19 @@ function switch_empty(v: count): string return "n/a"; } -function switch_break(v: count): string +function switch_fallthrough(v: count): string { local rval = ""; switch ( v ) { case 1: rval += "test"; + fallthrough; case 2: rval += "testing"; - break; - rval += "ERROR"; + fallthrough; case 3: rval += "tested"; + break; } return rval + "return"; } @@ -170,12 +171,16 @@ function switch_default(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; case 2: rval += "2"; + break; case 3: rval += "3"; + fallthrough; default: rval += "d"; + break; } return rval + "r"; } @@ -186,13 +191,16 @@ function switch_default_placement(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; default: rval += "d"; + fallthrough; case 2: rval += "2"; break; case 3: rval += "3"; + break; } return rval + "r"; } @@ -252,17 +260,17 @@ event new_connection(c: connection) test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testreturn" ); - test_switch( switch_break(2) , "testingreturn" ); - test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "1r" ); + test_switch( switch_fallthrough(1) , "testtestingtestedreturn" ); + test_switch( switch_fallthrough(2) , "testingtestedreturn" ); + test_switch( switch_fallthrough(3) , "testedreturn" ); + test_switch( switch_default(1) , "12r" ); test_switch( switch_default(2) , "2r" ); - test_switch( switch_default(3) , "3r" ); + test_switch( switch_default(3) , "3dr" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1r" ); + test_switch( switch_default_placement(1) , "1d2r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "dr" ); + test_switch( switch_default_placement(4) , "d2r" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -272,12 +280,16 @@ event new_connection(c: connection) switch ( v[i] ) { case 1, 2: expect = "1,2"; + break; case 3, 4, 5: expect = "3,4,5"; + break; case 6, 7, 8, 9: expect = "6,7,8,9"; + break; default: expect = "n/a"; + break; } test_switch( switch_case_list(v[i]) , expect ); } diff --git a/testing/btest/language/switch-statement.bro b/testing/btest/language/switch-statement.bro index dcf2a4c041..152b14f87d 100644 --- a/testing/btest/language/switch-statement.bro +++ b/testing/btest/language/switch-statement.bro @@ -143,18 +143,19 @@ function switch_empty(v: count): string return "n/a"; } -function switch_break(v: count): string +function switch_fallthrough(v: count): string { local rval = ""; switch ( v ) { case 1: rval += "test"; + fallthrough; case 2: rval += "testing"; - break; - rval += "ERROR"; + fallthrough; case 3: rval += "tested"; + break; } return rval + "return"; } @@ -165,12 +166,16 @@ function switch_default(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; case 2: rval += "2"; + break; case 3: rval += "3"; + fallthrough; default: rval += "d"; + break; } return rval + "r"; } @@ -181,13 +186,16 @@ function switch_default_placement(v: count): string switch ( v ) { case 1: rval += "1"; + fallthrough; default: rval += "d"; + fallthrough; case 2: rval += "2"; break; case 3: rval += "3"; + break; } return rval + "r"; } @@ -247,17 +255,17 @@ event bro_init() test_switch( switch_subnet([fe80::1]/96) , "[fe80::0]" ); test_switch( switch_subnet(192.168.1.100/16) , "192.168.0.0/16" ); test_switch( switch_empty(2) , "n/a" ); - test_switch( switch_break(1) , "testreturn" ); - test_switch( switch_break(2) , "testingreturn" ); - test_switch( switch_break(3) , "testedreturn" ); - test_switch( switch_default(1) , "1r" ); + test_switch( switch_fallthrough(1) , "testtestingtestedreturn" ); + test_switch( switch_fallthrough(2) , "testingtestedreturn" ); + test_switch( switch_fallthrough(3) , "testedreturn" ); + test_switch( switch_default(1) , "12r" ); test_switch( switch_default(2) , "2r" ); - test_switch( switch_default(3) , "3r" ); + test_switch( switch_default(3) , "3dr" ); test_switch( switch_default(4) , "dr" ); - test_switch( switch_default_placement(1) , "1r" ); + test_switch( switch_default_placement(1) , "1d2r" ); test_switch( switch_default_placement(2) , "2r" ); test_switch( switch_default_placement(3) , "3r" ); - test_switch( switch_default_placement(4) , "dr" ); + test_switch( switch_default_placement(4) , "d2r" ); local v = vector(0,1,2,3,4,5,6,7,9,10); local expect: string; @@ -267,12 +275,16 @@ event bro_init() switch ( v[i] ) { case 1, 2: expect = "1,2"; + break; case 3, 4, 5: expect = "3,4,5"; + break; case 6, 7, 8, 9: expect = "6,7,8,9"; + break; default: expect = "n/a"; + break; } test_switch( switch_case_list(v[i]) , expect ); } From 0a69b87f03b18f6c5b4e6952912b5390c9e698b1 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 17 Jan 2013 15:21:50 -0600 Subject: [PATCH 11/49] Fix uninitialized locals in event/hook handlers from having a value. Since values for local variables are referenced by offset within a Frame (not by identifier name), and event/hook handler bodies share a common Frame, the value offsets for local variables in different handlers may overlap. This meant locals in a handler without an initialization may actually end up referring to the value of a previous handler's local that has the same Frame offset. When executing the body, that can possibly result in a type-conflict error or give give unexpected results instead of a "use of uninitialized value" error. This patch makes it so uninitialized locals do always refer to a null value before executing the body of a event/hook handler, so that using them without assigning a value within the body will connsistently give a "use of uninitialized value" error. Addresses #932. --- src/Stmt.cc | 16 +++++++++---- src/Var.cc | 5 +--- .../Baseline/language.uninitialized-local/out | 2 ++ .../btest/language/uninitialized-local.bro | 23 +++++++++++++++++++ 4 files changed, 38 insertions(+), 8 deletions(-) create mode 100644 testing/btest/Baseline/language.uninitialized-local/out create mode 100644 testing/btest/language/uninitialized-local.bro diff --git a/src/Stmt.cc b/src/Stmt.cc index 2cd7117ddb..d7052d0b66 100644 --- a/src/Stmt.cc +++ b/src/Stmt.cc @@ -1789,13 +1789,21 @@ Val* InitStmt::Exec(Frame* f, stmt_flow_type& flow) const ID* aggr = (*inits)[i]; BroType* t = aggr->Type(); - Val* v; - if ( t->Tag() == TYPE_RECORD ) + Val* v = 0; + + switch ( t->Tag() ) { + case TYPE_RECORD: v = new RecordVal(t->AsRecordType()); - else if ( aggr->Type()->Tag() == TYPE_VECTOR ) + break; + case TYPE_VECTOR: v = new VectorVal(t->AsVectorType()); - else + break; + case TYPE_TABLE: v = new TableVal(t->AsTableType(), aggr->Attrs()); + break; + default: + break; + } f->SetElement(aggr->Offset(), v); } diff --git a/src/Var.cc b/src/Var.cc index b4d76097d3..0aadd93e92 100644 --- a/src/Var.cc +++ b/src/Var.cc @@ -243,10 +243,7 @@ Stmt* add_local(ID* id, BroType* t, init_class c, Expr* init, else { - if ( t->Tag() == TYPE_RECORD || t->Tag() == TYPE_TABLE || - t->Tag() == TYPE_VECTOR ) - current_scope()->AddInit(id); - + current_scope()->AddInit(id); return new NullStmt; } } diff --git a/testing/btest/Baseline/language.uninitialized-local/out b/testing/btest/Baseline/language.uninitialized-local/out new file mode 100644 index 0000000000..f803415fe6 --- /dev/null +++ b/testing/btest/Baseline/language.uninitialized-local/out @@ -0,0 +1,2 @@ +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.uninitialized-local/uninitialized-local.bro, line 16: value used but not set (my_string) +Continuing diff --git a/testing/btest/language/uninitialized-local.bro b/testing/btest/language/uninitialized-local.bro new file mode 100644 index 0000000000..e1cc178c0a --- /dev/null +++ b/testing/btest/language/uninitialized-local.bro @@ -0,0 +1,23 @@ +# @TEST-EXEC: bro -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event testit() &priority=10 + { + local my_count: count = 10; + } + +event testit() + { + # my_string's value occupies same Frame offset as my_count's from above + # handler, but execution of this handler body should still "initialize" + # it to a null value instead of referring to a left-over value of my_count. + local my_string: string; + local my_vector: vector of string; + my_vector[0] = my_string; + print "Continuing"; + } + +event bro_init() + { + event testit(); + } From 2823744ea5a5f315b0897731866ba60a37870ad3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 17 Jan 2013 17:40:34 -0800 Subject: [PATCH 12/49] Removing unused class member. --- src/threading/BasicThread.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index e17324e948..100efe8851 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -199,10 +199,6 @@ private: bool terminating; // Set to to true to signal termination. bool killed; // Set to true once forcefully killed. - // Used as a semaphore to tell the pthread thread when it may - // terminate. - pthread_mutex_t terminate; - // For implementing Fmt(). char* buf; unsigned int buf_len; From fdd11428c142868b2856978322f42204db6e1d40 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Jan 2013 12:46:00 -0600 Subject: [PATCH 13/49] Change reporter messages to more reliably print to stderr. Moved this functionality to be internal instead of in the script-layer event handlers. The issue with the later is that bad things can happen between the time a reporter event handler is dispatched and the time it is executed, and if bro crashes in that time, the message may never be seen/logged. Addressed #930 (and revisits #836). --- scripts/base/frameworks/reporter/main.bro | 46 ++++--------------- scripts/base/init-bare.bro | 19 ++++++++ src/Reporter.cc | 37 ++++++++++++--- src/Reporter.h | 6 +++ src/main.cc | 2 + .../core.reporter-error-in-handler/output | 4 +- .../core.reporter-runtime-error/output | 4 +- .../Baseline/core.reporter/logger-test.log | 12 ++--- testing/btest/Baseline/core.reporter/output | 16 ++++--- .../.stderr | 2 +- .../reporter.log | 6 +-- 11 files changed, 90 insertions(+), 64 deletions(-) diff --git a/scripts/base/frameworks/reporter/main.bro b/scripts/base/frameworks/reporter/main.bro index edc5b1779a..249ecdac98 100644 --- a/scripts/base/frameworks/reporter/main.bro +++ b/scripts/base/frameworks/reporter/main.bro @@ -1,10 +1,15 @@ ##! This framework is intended to create an output and filtering path for ##! internal messages/warnings/errors. It should typically be loaded to -##! avoid Bro spewing internal messages to standard error and instead log -##! them to a file in a standard way. Note that this framework deals with -##! the handling of internally-generated reporter messages, for the -##! interface into actually creating reporter messages from the scripting -##! layer, use the built-in functions in :doc:`/scripts/base/reporter.bif`. +##! log such messages to a file in a standard way. For the options to +##! toggle whether messages are additionally written to STDERR, see +##! :bro:see:`Reporter::info_to_stderr`, +##! :bro:see:`Reporter::warnings_to_stderr`, and +##! :bro:see:`Reporter::errors_to_stderr`. +##! +##! Note that this framework deals with the handling of internally generated +##! reporter messages, for the interface in to actually creating interface +##! into actually creating reporter messages from the scripting layer, use +##! the built-in functions in :doc:`/scripts/base/reporter.bif`. module Reporter; @@ -36,26 +41,11 @@ export { ## Not all reporter messages will have locations in them though. location: string &log &optional; }; - - ## Tunable for sending reporter warning messages to STDERR. The option to - ## turn it off is presented here in case Bro is being run by some - ## external harness and shouldn't output anything to the console. - const warnings_to_stderr = T &redef; - - ## Tunable for sending reporter error messages to STDERR. The option to - ## turn it off is presented here in case Bro is being run by some - ## external harness and shouldn't output anything to the console. - const errors_to_stderr = T &redef; } -global stderr: file; - event bro_init() &priority=5 { Log::create_stream(Reporter::LOG, [$columns=Info]); - - if ( errors_to_stderr || warnings_to_stderr ) - stderr = open("/dev/stderr"); } event reporter_info(t: time, msg: string, location: string) &priority=-5 @@ -65,26 +55,10 @@ event reporter_info(t: time, msg: string, location: string) &priority=-5 event reporter_warning(t: time, msg: string, location: string) &priority=-5 { - if ( warnings_to_stderr ) - { - if ( t > double_to_time(0.0) ) - print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location); - else - print stderr, fmt("WARNING: %s (%s)", msg, location); - } - Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]); } event reporter_error(t: time, msg: string, location: string) &priority=-5 { - if ( errors_to_stderr ) - { - if ( t > double_to_time(0.0) ) - print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location); - else - print stderr, fmt("ERROR: %s (%s)", msg, location); - } - Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]); } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index e5365a9428..1a1349dc94 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2857,6 +2857,25 @@ export { } # end export module GLOBAL; +module Reporter; +export { + ## Tunable for sending reporter info messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const info_to_stderr = T &redef; + + ## Tunable for sending reporter warning messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const warnings_to_stderr = T &redef; + + ## Tunable for sending reporter error messages to STDERR. The option to + ## turn it off is presented here in case Bro is being run by some + ## external harness and shouldn't output anything to the console. + const errors_to_stderr = T &redef; +} +module GLOBAL; + ## Number of bytes per packet to capture from live interfaces. const snaplen = 8192 &redef; diff --git a/src/Reporter.cc b/src/Reporter.cc index 18f39ce4af..fc3dde3ca0 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -27,6 +27,13 @@ Reporter::Reporter() via_events = false; in_error_handler = 0; + // Always use stderr at startup/init before scripts have been fully parsed. + // Messages may otherwise be missed if an error occurs that prevents events + // from ever being dispatched. + info_to_stderr = true; + warnings_to_stderr = true; + errors_to_stderr = true; + openlog("bro", 0, LOG_LOCAL5); } @@ -35,11 +42,19 @@ Reporter::~Reporter() closelog(); } +void Reporter::InitOptions() + { + info_to_stderr = internal_const_val("Reporter::info_to_stderr")->AsBool(); + warnings_to_stderr = internal_const_val("Reporter::warnings_to_stderr")->AsBool(); + errors_to_stderr = internal_const_val("Reporter::errors_to_stderr")->AsBool(); + } + void Reporter::Info(const char* fmt, ...) { va_list ap; va_start(ap, fmt); - DoLog("", reporter_info, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = info_to_stderr ? stderr : 0; + DoLog("", reporter_info, out, 0, 0, true, true, 0, fmt, ap); va_end(ap); } @@ -47,7 +62,8 @@ void Reporter::Warning(const char* fmt, ...) { va_list ap; va_start(ap, fmt); - DoLog("warning", reporter_warning, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = warnings_to_stderr ? stderr : 0; + DoLog("warning", reporter_warning, out, 0, 0, true, true, 0, fmt, ap); va_end(ap); } @@ -56,7 +72,8 @@ void Reporter::Error(const char* fmt, ...) ++errors; va_list ap; va_start(ap, fmt); - DoLog("error", reporter_error, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = errors_to_stderr ? stderr : 0; + DoLog("error", reporter_error, out, 0, 0, true, true, 0, fmt, ap); va_end(ap); } @@ -98,7 +115,9 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...) PushLocation(expr->GetLocationInfo()); va_list ap; va_start(ap, fmt); - DoLog("expression error", reporter_error, stderr, 0, 0, true, true, d.Description(), fmt, ap); + FILE* out = errors_to_stderr ? stderr : 0; + DoLog("expression error", reporter_error, out, 0, 0, true, true, + d.Description(), fmt, ap); va_end(ap); PopLocation(); throw InterpreterException(); @@ -122,7 +141,9 @@ void Reporter::InternalWarning(const char* fmt, ...) { va_list ap; va_start(ap, fmt); - DoLog("internal warning", reporter_warning, stderr, 0, 0, true, true, 0, fmt, ap); + FILE* out = warnings_to_stderr ? stderr : 0; + DoLog("internal warning", reporter_warning, out, 0, 0, true, true, 0, fmt, + ap); va_end(ap); } @@ -189,7 +210,9 @@ void Reporter::Weird(const IPAddr& orig, const IPAddr& resp, const char* name) WeirdFlowHelper(orig, resp, "%s", name); } -void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Connection* conn, val_list* addl, bool location, bool time, const char* postfix, const char* fmt, va_list ap) +void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, + Connection* conn, val_list* addl, bool location, bool time, + const char* postfix, const char* fmt, va_list ap) { static char tmp[512]; @@ -298,7 +321,7 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, Conne mgr.QueueEvent(event, vl); } - else + if ( out ) { string s = ""; diff --git a/src/Reporter.h b/src/Reporter.h index e610e1519e..6a205cfd6d 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -43,6 +43,9 @@ public: Reporter(); ~Reporter(); + // Initialize reporter-sepcific options that are defined in script-layer. + void InitOptions(); + // Report an informational message, nothing that needs specific // attention. void Info(const char* fmt, ...) FMT_ATTR; @@ -126,6 +129,9 @@ private: int errors; bool via_events; int in_error_handler; + bool info_to_stderr; + bool warnings_to_stderr; + bool errors_to_stderr; std::list > locations; }; diff --git a/src/main.cc b/src/main.cc index 5999186240..59cdb23241 100644 --- a/src/main.cc +++ b/src/main.cc @@ -824,6 +824,8 @@ int main(int argc, char** argv) exit(1); } + reporter->InitOptions(); + init_general_global_var(); if ( user_pcap_filter ) diff --git a/testing/btest/Baseline/core.reporter-error-in-handler/output b/testing/btest/Baseline/core.reporter-error-in-handler/output index b20b1b2292..2f92afe6b0 100644 --- a/testing/btest/Baseline/core.reporter-error-in-handler/output +++ b/testing/btest/Baseline/core.reporter-error-in-handler/output @@ -1,3 +1,3 @@ -error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) -ERROR: no such index (a[1]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 28: no such index (a[1]) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-error-in-handler/reporter-error-in-handler.bro, line 22: no such index (a[2]) 1st error printed on script level diff --git a/testing/btest/Baseline/core.reporter-runtime-error/output b/testing/btest/Baseline/core.reporter-runtime-error/output index 5a03f5feb2..c2ace6ceb6 100644 --- a/testing/btest/Baseline/core.reporter-runtime-error/output +++ b/testing/btest/Baseline/core.reporter-runtime-error/output @@ -1,2 +1,2 @@ -error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) -ERROR: no such index (a[2]) (/home/jsiwek/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 12: no such index (a[1]) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter-runtime-error/reporter-runtime-error.bro, line 9: no such index (a[2]) diff --git a/testing/btest/Baseline/core.reporter/logger-test.log b/testing/btest/Baseline/core.reporter/logger-test.log index 5afd904b63..4ee0d03341 100644 --- a/testing/btest/Baseline/core.reporter/logger-test.log +++ b/testing/btest/Baseline/core.reporter/logger-test.log @@ -1,6 +1,6 @@ -reporter_info|init test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 -reporter_warning|init test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 -reporter_error|init test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 -reporter_info|done test-info|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 -reporter_warning|done test-warning|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 -reporter_error|done test-error|/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 +reporter_info|init test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8|0.000000 +reporter_warning|init test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9|0.000000 +reporter_error|init test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10|0.000000 +reporter_info|done test-info|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15|0.000000 +reporter_warning|done test-warning|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16|0.000000 +reporter_error|done test-error|/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17|0.000000 diff --git a/testing/btest/Baseline/core.reporter/output b/testing/btest/Baseline/core.reporter/output index f2c59259c2..24a12f9679 100644 --- a/testing/btest/Baseline/core.reporter/output +++ b/testing/btest/Baseline/core.reporter/output @@ -1,7 +1,9 @@ -/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info -warning in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning -error in /home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error -WARNING: init test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9) -ERROR: init test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10) -WARNING: done test-warning (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16) -ERROR: done test-error (/home/jsiwek/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17) +/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 52: pre test-info +warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 53: pre test-warning +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 54: pre test-error +/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 8: init test-info +warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 9: init test-warning +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 10: init test-error +/Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 15: done test-info +warning in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 16: done test-warning +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/core.reporter/reporter.bro, line 17: done test-error diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr index 78af1e7a73..dc5065f5c8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/.stderr @@ -1 +1 @@ -ERROR: no such index (test[3]) (/blah/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9: no such index (test[3]) diff --git a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log index b314bc45c3..391cf77a00 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log +++ b/testing/btest/Baseline/scripts.base.frameworks.reporter.stderr/reporter.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path reporter -#open 2012-08-10-20-09-23 +#open 2013-01-18-18-29-30 #fields ts level message location #types time enum string string -0.000000 Reporter::ERROR no such index (test[3]) /da/home/robin/bro/master/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9 -#close 2012-08-10-20-09-23 +0.000000 Reporter::ERROR no such index (test[3]) /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/scripts.base.frameworks.reporter.stderr/stderr.bro, line 9 +#close 2013-01-18-18-29-30 From 624980b98d7741f9af0cfb417336d149bb21e704 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Jan 2013 12:56:23 -0600 Subject: [PATCH 14/49] Add a null value check in CompositeHash::ComputeHash. Because I guess aborting is nicer than segfaulting. Addresses #930. --- src/CompHash.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/CompHash.cc b/src/CompHash.cc index 86677f9719..306b3ea83e 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -273,6 +273,9 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0, HashKey* CompositeHash::ComputeHash(const Val* v, int type_check) const { + if ( ! v ) + reporter->InternalError("null value given to CompositeHash::ComputeHash"); + if ( is_singleton ) return ComputeSingletonHash(v, type_check); From acafcfafd2cf5a5bd8b88b6256d21a5af6a92591 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 18 Jan 2013 13:15:34 -0600 Subject: [PATCH 15/49] Revert "Trick for parallelizing input framework unit tests." This reverts commit 43ed437daabb4575549e82f53881980353905242. The old way of doing the tests seems more reliable for now. --- .../.stderrwithoutfirstline | 14 +++++++------- .../.stderrwithoutfirstline | 6 +++--- .../bro..stderr | 10 +++++----- .../btest/scripts/base/frameworks/input/basic.bro | 6 +++++- .../scripts/base/frameworks/input/bignumber.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/binary.bro | 7 ++++++- .../base/frameworks/input/empty-values-hashing.bro | 8 +++++++- .../scripts/base/frameworks/input/emptyvals.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/event.bro | 7 ++++++- .../scripts/base/frameworks/input/executeraw.bro | 7 ++++++- .../base/frameworks/input/invalidnumbers.bro | 9 +++++++-- .../scripts/base/frameworks/input/invalidtext.bro | 9 +++++++-- .../scripts/base/frameworks/input/missing-file.bro | 7 ++++++- .../base/frameworks/input/onecolumn-norecord.bro | 7 ++++++- .../base/frameworks/input/onecolumn-record.bro | 7 ++++++- .../scripts/base/frameworks/input/optional.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/port.bro | 7 ++++++- .../base/frameworks/input/predicate-stream.bro | 7 ++++++- .../scripts/base/frameworks/input/predicate.bro | 7 ++++++- .../base/frameworks/input/predicatemodify.bro | 7 ++++++- .../frameworks/input/predicatemodifyandreread.bro | 7 ++++++- .../input/predicaterefusesecondsamerecord.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/raw.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/repeat.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/reread.bro | 6 +++++- .../scripts/base/frameworks/input/rereadraw.bro | 7 ++++++- .../btest/scripts/base/frameworks/input/set.bro | 7 ++++++- .../scripts/base/frameworks/input/setseparator.bro | 7 ++++++- .../base/frameworks/input/setspecialcases.bro | 8 +++++++- .../btest/scripts/base/frameworks/input/stream.bro | 6 +++++- .../scripts/base/frameworks/input/streamraw.bro | 7 ++++++- .../base/frameworks/input/subrecord-event.bro | 6 +++++- .../scripts/base/frameworks/input/subrecord.bro | 6 +++++- .../scripts/base/frameworks/input/tableevent.bro | 7 ++++++- .../scripts/base/frameworks/input/twotables.bro | 6 +++++- .../base/frameworks/input/unsupported_types.bro | 6 +++++- 36 files changed, 210 insertions(+), 50 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline index 54005fb4b8..3ef51e40f2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidnumbers/.stderrwithoutfirstline @@ -1,8 +1,8 @@ -1355266097.683599 error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. -1355266097.683599 error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' to Val. Ignoring line. -1355266097.683599 warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' -1355266097.683599 warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' -1355266097.683599 error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number -1355266097.683599 error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Ignoring line. -1355266097.683599 received termination signal +error: ../input.log/Input::READER_ASCII: Number '12129223372036854775800' out of supported range. +error: ../input.log/Input::READER_ASCII: Could not convert line '12129223372036854775800 121218446744073709551612' to Val. Ignoring line. +warning: ../input.log/Input::READER_ASCII: Number '9223372036854775801TEXTHERE' contained non-numeric trailing characters. Ignored trailing characters 'TEXTHERE' +warning: ../input.log/Input::READER_ASCII: Number '1Justtext' contained non-numeric trailing characters. Ignored trailing characters 'Justtext' +error: ../input.log/Input::READER_ASCII: String 'Justtext' contained no parseable number +error: ../input.log/Input::READER_ASCII: Could not convert line 'Justtext 1' to Val. Ignoring line. +received termination signal >>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline b/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline index f0545daeae..3d8ba5e267 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalidtext/.stderrwithoutfirstline @@ -1,4 +1,4 @@ -1355265853.593476 error: ../input.log/Input::READER_ASCII: String 'l' contained no parseable number -1355265853.593476 error: ../input.log/Input::READER_ASCII: Could not convert line ' l' to Val. Ignoring line. -1355265853.593476 received termination signal +error: ../input.log/Input::READER_ASCII: String 'l' contained no parseable number +error: ../input.log/Input::READER_ASCII: Could not convert line ' l' to Val. Ignoring line. +received termination signal >>> diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr index e61280cdfc..4380007b93 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file/bro..stderr @@ -1,5 +1,5 @@ -1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat -1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: Init failed -1355265996.626106 warning: Stream input is already queued for removal. Ignoring remove. -1355265996.626106 error: does-not-exist.dat/Input::READER_ASCII: terminating thread -1355265996.626106 received termination signal +error: does-not-exist.dat/Input::READER_ASCII: Init: cannot open does-not-exist.dat +error: does-not-exist.dat/Input::READER_ASCII: Init failed +warning: Stream input is already queued for removal. Ignoring remove. +error: does-not-exist.dat/Input::READER_ASCII: terminating thread +received termination signal diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index fbb320e03f..dfac84d062 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro index 098481a518..5b93472551 100644 --- a/testing/btest/scripts/base/frameworks/input/bignumber.bro +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -10,6 +13,8 @@ -9223372036854775800 18446744073709551612 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro index 7caa734d34..8d75abc5a9 100644 --- a/testing/btest/scripts/base/frameworks/input/binary.bro +++ b/testing/btest/scripts/base/frameworks/input/binary.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -22,6 +25,8 @@ abc\xff\x7cdef|DATA2 #end|2012-07-20-01-49-19 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro index 78f3d3a72e..c8760b467e 100644 --- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: btest-bg-wait -k 5 @@ -20,6 +23,9 @@ 2 TEST TEST @TEST-END-FILE +@load frameworks/communication/listen + + module A; type Idx: record { diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro index e5e9bc22e3..94b0f1b620 100644 --- a/testing/btest/scripts/base/frameworks/input/emptyvals.bro +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,8 @@ T 1 - 2 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index 7d553a53e3..ba47d5e3f2 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 09cd920bee..626b9cdfd2 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out # @TEST-EXEC: btest-diff out @@ -14,6 +17,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; type Val: record { diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro index 3c467598ad..1deec605ae 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -1,8 +1,11 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline @TEST-START-FILE input.log #separator \x09 @@ -14,6 +17,8 @@ Justtext 1 9223372036854775800 -18446744073709551612 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/invalidtext.bro b/testing/btest/scripts/base/frameworks/input/invalidtext.bro index 668716d045..75efb1247d 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidtext.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidtext.bro @@ -1,8 +1,11 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out # @TEST-EXEC: sed 1d .stderr > .stderrwithoutfirstline -# @TEST-EXEC: TEST_DIFF_CANONIFIER="$SCRIPTS/diff-remove-abspath | $SCRIPTS/diff-remove-timestamps" btest-diff .stderrwithoutfirstline +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff .stderrwithoutfirstline @TEST-START-FILE input.log #separator \x09 @@ -12,6 +15,8 @@ 5 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro index 7048698221..aa5acf619e 100644 --- a/testing/btest/scripts/base/frameworks/input/missing-file.bro +++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro @@ -1,7 +1,12 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff bro/.stderr +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index 505aa2245d..c08b1420fb 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -10,6 +13,8 @@ T -42 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index e85267b4c3..9e420e75fe 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -10,6 +13,8 @@ T -42 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro index 670f0f48ba..2fe0e5c86f 100644 --- a/testing/btest/scripts/base/frameworks/input/optional.bro +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 2225132395..081c59559b 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -9,6 +12,8 @@ 1.2.3.6 30 unknown @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index b931bbc41e..8cf927e346 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out # @@ -20,6 +23,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 9f2c4362de..8fb33242e8 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro index 4129a47873..17467bbc27 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,8 @@ 2 T test2 idx2 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro index 17e7fa4bf2..5a9e993651 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: sleep 2 @@ -55,6 +58,8 @@ 1 T test1 idx1 @TEST-END-FILE +@load frameworks/communication/listen + redef InputAscii::empty_field = "EMPTY"; module A; diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro index 0df8b14dd6..ba0b468cdc 100644 --- a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro +++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,6 +17,8 @@ need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 95 1342569600 1.228.83. need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 65 1342656000 1.228.83.33 - - 9318 HANARO-AS Hanaro Telecom Inc. 1.224.0.0/13 apnic KR spam infrastructure spamming;malware domain public http://reputation.alienvault.com/reputation.generic @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index 7201e9ad3a..d15aec22bb 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,6 +16,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro index f29061fa8b..a966ac064e 100644 --- a/testing/btest/scripts/base/frameworks/input/repeat.bro +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -10,6 +13,8 @@ 1 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index bfb7b6fd84..11aa873f9d 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 # @TEST-EXEC: cp input2.log input.log # @TEST-EXEC: sleep 2 @@ -56,6 +59,7 @@ F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro index d1db2bb049..2fdcdc8f9e 100644 --- a/testing/btest/scripts/base/frameworks/input/rereadraw.bro +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,6 +16,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro index a460f6788a..b2b5cea323 100644 --- a/testing/btest/scripts/base/frameworks/input/set.bro +++ b/testing/btest/scripts/base/frameworks/input/set.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,6 +16,8 @@ 192.168.17.42 @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro index 156f566d28..b7148d80bd 100644 --- a/testing/btest/scripts/base/frameworks/input/setseparator.bro +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -10,6 +13,8 @@ redef InputAscii::set_separator = "|"; +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro index 86e53feb0a..022eac9731 100644 --- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,6 +16,9 @@ 6 @TEST-END-FILE + +@load frameworks/communication/listen + global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index 684f4ea395..1ecd8a2eb0 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 # @TEST-EXEC: cat input2.log >> input.log # @TEST-EXEC: sleep 3 @@ -22,6 +25,7 @@ F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro index d8e43d6386..3bc06f7dea 100644 --- a/testing/btest/scripts/base/frameworks/input/streamraw.bro +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 # @TEST-EXEC: cat input2.log >> input.log # @TEST-EXEC: sleep 3 @@ -24,6 +27,8 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE +@load frameworks/communication/listen + module A; type Val: record { diff --git a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro index 92ee6dd500..4e7dc1690a 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/subrecord.bro b/testing/btest/scripts/base/frameworks/input/subrecord.bro index 11f247f764..512b8ec58f 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index db39fdf72b..723e519237 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,6 +19,8 @@ 7 T @TEST-END-FILE +@load frameworks/communication/listen + global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro index 5540019971..83ae86cd46 100644 --- a/testing/btest/scripts/base/frameworks/input/twotables.bro +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -1,5 +1,8 @@ +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# # @TEST-EXEC: cp input1.log input.log -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 5 # @TEST-EXEC: cp input3.log input.log # @TEST-EXEC: btest-bg-wait -k 10 @@ -31,6 +34,7 @@ F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro index 0651e0693e..e1350f61a9 100644 --- a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro +++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro @@ -1,4 +1,7 @@ -# @TEST-EXEC: btest-bg-run bro bro -b --pseudo-realtime -r $TRACES/socks.trace %INPUT +# (uses listen.bro just to ensure input sources are more reliably fully-read). +# @TEST-SERIALIZE: comm +# +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -11,6 +14,7 @@ whatever T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.00 @TEST-END-FILE @load base/protocols/ssh +@load frameworks/communication/listen global outfile: file; From cce73a3aadc94215eb78b17d502cc5d66bb9a853 Mon Sep 17 00:00:00 2001 From: Gilbert Clark Date: Fri, 18 Jan 2013 14:23:34 -0500 Subject: [PATCH 16/49] Fixing variable size issues (was uint, should be long) with http response code. --- src/logging/writers/ElasticSearch.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/logging/writers/ElasticSearch.cc b/src/logging/writers/ElasticSearch.cc index ae825ac997..fccf33534e 100644 --- a/src/logging/writers/ElasticSearch.cc +++ b/src/logging/writers/ElasticSearch.cc @@ -402,7 +402,7 @@ bool ElasticSearch::HTTPSend(CURL *handle) case CURLE_OK: { - uint http_code = 0; + long http_code = 0; curl_easy_getinfo(curl_handle, CURLINFO_RESPONSE_CODE, &http_code); if ( http_code == 200 ) // Hopefully everything goes through here. From 4f39470c1bc37cce69c597ba5d5a28e1bb0548e1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 18 Jan 2013 17:34:33 -0800 Subject: [PATCH 17/49] New option exit_only_after_terminate to prevent Bro from exiting. If set, the main loop won't terminate before somebody calls terminate(). This should make input framework testing more reliable I'd hope. --- scripts/base/init-bare.bro | 9 +++++++++ src/Net.cc | 3 ++- src/const.bif | 1 + 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index e5365a9428..a01b52817c 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -2608,6 +2608,15 @@ const gap_report_freq = 1.0 sec &redef; ## .. bro:see:: content_gap gap_report partial_connection const report_gaps_for_partial = F &redef; +## Flag to prevent Bro from exiting automatically when input is exhausted. +## Normally Bro terminates when all packets sources have gone dry +## and communication isn't enabled. If this flag is set, Bro's main loop will +## instead keep idleing until :bro:see::`terminate` is explicitly called. +## +## This is mainly for testing purposes when termination behaviour needs to be +## controlled for reproducing results. +const exit_only_after_terminate = F &redef; + ## The CA certificate file to authorize remote Bros/Broccolis. ## ## .. bro:see:: ssl_private_key ssl_passphrase diff --git a/src/Net.cc b/src/Net.cc index 328998b011..73c618b8af 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -421,7 +421,8 @@ void net_run() set_processing_status("RUNNING", "net_run"); while ( io_sources.Size() || - (packet_sorter && ! packet_sorter->Empty()) ) + (packet_sorter && ! packet_sorter->Empty()) || + (BifConst::exit_only_after_terminate && ! terminating) ) { double ts; IOSource* src = io_sources.FindSoonest(&ts); diff --git a/src/const.bif b/src/const.bif index ea7dc03817..ea84b3363d 100644 --- a/src/const.bif +++ b/src/const.bif @@ -6,6 +6,7 @@ const ignore_keep_alive_rexmit: bool; const skip_http_data: bool; const use_conn_size_analyzer: bool; const report_gaps_for_partial: bool; +const exit_only_after_terminate: bool; const NFS3::return_data: bool; const NFS3::return_data_max: count; From c9412c427162d5c2003eba8114fd4911c7783072 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 14:43:12 -0800 Subject: [PATCH 18/49] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 3585dc9a7a..edf3ad17f2 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 3585dc9a7afe20d70cb77fc2472cc6bce3850b67 +Subproject commit edf3ad17f2d0e3c8892ffdfa306fe62354128fd8 From 0541c49a50d5b1682429bc3dd873fc375fb47378 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 14:43:38 -0800 Subject: [PATCH 19/49] Changing btest call to use "-j" instead of "-j 5". --- testing/btest/Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/testing/btest/Makefile b/testing/btest/Makefile index 93ccc8d5ec..ff63bdb601 100644 --- a/testing/btest/Makefile +++ b/testing/btest/Makefile @@ -6,13 +6,13 @@ all: cleanup btest-verbose coverage # Showing all tests. btest-verbose: - @$(BTEST) -j 5 -f $(DIAG) + @$(BTEST) -j -f $(DIAG) brief: cleanup btest-brief coverage # Brief output showing only failed tests. btest-brief: - @$(BTEST) -j 5 -b -f $(DIAG) + @$(BTEST) -j -b -f $(DIAG) coverage: @../scripts/coverage-calc ".tmp/script-coverage*" coverage.log `pwd`/../../scripts From b6c71f539089587f579f9e4a10b2763d152d789c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 18:11:49 -0800 Subject: [PATCH 20/49] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index edf3ad17f2..6c0f099c52 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit edf3ad17f2d0e3c8892ffdfa306fe62354128fd8 +Subproject commit 6c0f099c52f9e7b5d6f3efafb63581a6cc0861a5 From ed165f22a248cd0cfc4e2af94b8c48ff4c4303aa Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 23 Jan 2013 19:43:27 -0800 Subject: [PATCH 21/49] Making a test portable. --- CHANGES | 4 ++++ VERSION | 2 +- testing/btest/language/switch-incomplete.bro | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index ae5ae96a39..1029b7f639 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.1-304 | 2013-01-23 19:43:27 -0800 + + * Making a test portable. (Robin Sommer) + 2.1-302 | 2013-01-23 16:17:29 -0800 * Refactoring ASCII formatting/parsing from loggers/readers into a diff --git a/VERSION b/VERSION index 7ad39c568b..adf66ac1d3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-302 +2.1-304 diff --git a/testing/btest/language/switch-incomplete.bro b/testing/btest/language/switch-incomplete.bro index c39f3f3688..7ee800b274 100644 --- a/testing/btest/language/switch-incomplete.bro +++ b/testing/btest/language/switch-incomplete.bro @@ -1,5 +1,5 @@ # @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1 -# @TEST-EXEC: btest-diff out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out event bro_init() { From 69afc4a88269e497af4a9890f1a403e55694f934 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Jan 2013 09:48:23 -0600 Subject: [PATCH 22/49] Add an error for record coercions that would orphan a field. These cases should be avoidable by fixing scripts where they occur and they can also help catch typos that would lead to unintentional runtime behavior. Adding this already revealed several scripts where a field in an inlined record was never removed after a code refactor. --- scripts/base/frameworks/signatures/main.bro | 8 +++---- scripts/base/protocols/http/file-hash.bro | 2 +- scripts/base/protocols/http/file-ident.bro | 4 +--- scripts/base/protocols/socks/main.bro | 2 +- scripts/policy/protocols/http/detect-MHR.bro | 2 +- src/Expr.cc | 7 ++++-- .../language.record-ceorce-orphan/out | 2 ++ .../btest/language/record-ceorce-orphan.bro | 22 +++++++++++++++++++ 8 files changed, 36 insertions(+), 13 deletions(-) create mode 100644 testing/btest/Baseline/language.record-ceorce-orphan/out create mode 100644 testing/btest/language/record-ceorce-orphan.bro diff --git a/scripts/base/frameworks/signatures/main.bro b/scripts/base/frameworks/signatures/main.bro index 26f78a68d1..4102214075 100644 --- a/scripts/base/frameworks/signatures/main.bro +++ b/scripts/base/frameworks/signatures/main.bro @@ -148,7 +148,7 @@ function has_signature_matched(id: string, orig: addr, resp: addr): bool event sig_summary(orig: addr, id: string, msg: string) { NOTICE([$note=Signature_Summary, $src=orig, - $filename=id, $msg=fmt("%s: %s", orig, msg), + $msg=fmt("%s: %s", orig, msg), $n=count_per_orig[orig,id] ]); } @@ -209,7 +209,6 @@ event signature_match(state: signature_state, msg: string, data: string) { NOTICE([$note=Count_Signature, $conn=state$conn, $msg=msg, - $filename=sig_id, $n=count_per_resp[dst,sig_id], $sub=fmt("%d matches of signature %s on host %s", count_per_resp[dst,sig_id], @@ -240,7 +239,7 @@ event signature_match(state: signature_state, msg: string, data: string) if ( notice ) NOTICE([$note=Sensitive_Signature, $conn=state$conn, $src=src_addr, - $dst=dst_addr, $filename=sig_id, $msg=fmt("%s: %s", src_addr, msg), + $dst=dst_addr, $msg=fmt("%s: %s", src_addr, msg), $sub=data]); if ( action == SIG_FILE_BUT_NO_SCAN || action == SIG_SUMMARY ) @@ -274,7 +273,7 @@ event signature_match(state: signature_state, msg: string, data: string) $src_addr=orig, $sig_id=sig_id, $event_msg=msg, $host_count=hcount, $sub_msg=horz_scan_msg]); - NOTICE([$note=Multiple_Sig_Responders, $src=orig, $filename=sig_id, + NOTICE([$note=Multiple_Sig_Responders, $src=orig, $msg=msg, $n=hcount, $sub=horz_scan_msg]); last_hthresh[orig] = hcount; @@ -295,7 +294,6 @@ event signature_match(state: signature_state, msg: string, data: string) $sub_msg=vert_scan_msg]); NOTICE([$note=Multiple_Signatures, $src=orig, $dst=resp, - $filename=sig_id, $msg=fmt("%s different signatures triggered", vcount), $n=vcount, $sub=vert_scan_msg]); diff --git a/scripts/base/protocols/http/file-hash.bro b/scripts/base/protocols/http/file-hash.bro index bc7547e51a..2545cbf817 100644 --- a/scripts/base/protocols/http/file-hash.bro +++ b/scripts/base/protocols/http/file-hash.bro @@ -73,7 +73,7 @@ event http_message_done(c: connection, is_orig: bool, stat: http_message_stat) & delete c$http$md5_handle; NOTICE([$note=MD5, $msg=fmt("%s %s %s", c$id$orig_h, c$http$md5, url), - $sub=c$http$md5, $conn=c, $URL=url]); + $sub=c$http$md5, $conn=c]); } } diff --git a/scripts/base/protocols/http/file-ident.bro b/scripts/base/protocols/http/file-ident.bro index b493f02bf0..706ea58558 100644 --- a/scripts/base/protocols/http/file-ident.bro +++ b/scripts/base/protocols/http/file-ident.bro @@ -68,9 +68,7 @@ event signature_match(state: signature_state, msg: string, data: string) &priori local message = fmt("%s %s %s", msg, c$http$method, url); NOTICE([$note=Incorrect_File_Type, $msg=message, - $conn=c, - $method=c$http$method, - $URL=url]); + $conn=c]); } } diff --git a/scripts/base/protocols/socks/main.bro b/scripts/base/protocols/socks/main.bro index 79ae4baa19..df5ee69f16 100644 --- a/scripts/base/protocols/socks/main.bro +++ b/scripts/base/protocols/socks/main.bro @@ -67,7 +67,7 @@ event socks_request(c: connection, version: count, request_type: count, # proxied connection. We treat this as a singular "tunnel". local cid = copy(c$id); cid$orig_p = 0/tcp; - Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS, $payload_proxy=T]); + Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS]); } event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=5 diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 5abb98d39c..1898022978 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -37,7 +37,7 @@ event log_http(rec: HTTP::Info) local url = HTTP::build_url_http(rec); local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url); NOTICE([$note=Malware_Hash_Registry_Match, - $msg=message, $id=rec$id, $URL=url]); + $msg=message, $id=rec$id]); } } } diff --git a/src/Expr.cc b/src/Expr.cc index 9e71f27897..7b28d6f4df 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3921,8 +3921,11 @@ RecordCoerceExpr::RecordCoerceExpr(Expr* op, RecordType* r) { int t_i = t_r->FieldOffset(sub_r->FieldName(i)); if ( t_i < 0 ) - // Orphane field in rhs, that's ok. - continue; + { + ExprError(fmt("orphaned field \"%s\" in record coercion", + sub_r->FieldName(i))); + break; + } BroType* sub_t_i = sub_r->FieldType(i); BroType* sup_t_i = t_r->FieldType(t_i); diff --git a/testing/btest/Baseline/language.record-ceorce-orphan/out b/testing/btest/Baseline/language.record-ceorce-orphan/out new file mode 100644 index 0000000000..aa42d13892 --- /dev/null +++ b/testing/btest/Baseline/language.record-ceorce-orphan/out @@ -0,0 +1,2 @@ +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 19: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; })) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.record-ceorce-orphan/record-ceorce-orphan.bro, line 21: orphaned field "wtf" in record coercion ((coerce [$a=test, $b=42, $wtf=1.0 sec] to record { a:string; b:count; c:interval; })) diff --git a/testing/btest/language/record-ceorce-orphan.bro b/testing/btest/language/record-ceorce-orphan.bro new file mode 100644 index 0000000000..126b99d5ff --- /dev/null +++ b/testing/btest/language/record-ceorce-orphan.bro @@ -0,0 +1,22 @@ +# @TEST-EXEC-FAIL: bro -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +type myrec: record { + a: string; + b: count; + c: interval &optional; +}; + +function myfunc(rec: myrec) + { + print rec; + } + +event bro_init() + { + # Orhpaned fields in a record coercion reflect a programming error, like a typo, so should + # be reported at parse-time to prevent unexpected run-time behavior. + local rec: myrec = [$a="test", $b=42, $wtf=1sec]; + print rec; + myfunc([$a="test", $b=42, $wtf=1sec]); + } From b72fbaf99fd8d40fe0bf38c81bf5f5921c762141 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 24 Jan 2013 10:26:54 -0600 Subject: [PATCH 23/49] Fix memory leak in some reporter messaging cases. Related to the changes in fdd11428. --- src/Reporter.cc | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/Reporter.cc b/src/Reporter.cc index fc3dde3ca0..6bc2577c72 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -320,6 +320,14 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, else mgr.QueueEvent(event, vl); } + else + { + if ( addl ) + { + loop_over_list(*addl, i) + Unref((*addl)[i]); + } + } if ( out ) { @@ -351,12 +359,6 @@ void Reporter::DoLog(const char* prefix, EventHandlerPtr event, FILE* out, if ( out ) fprintf(out, "%s", s.c_str()); - - if ( addl ) - { - loop_over_list(*addl, i) - Unref((*addl)[i]); - } } if ( alloced ) From 7d3b20d4fbe9f1bbe24aa249c2e4591feae4a381 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 25 Jan 2013 11:57:21 -0600 Subject: [PATCH 24/49] Fix runaway reference counting bug in record coercion. The RecordVal ctor refs the type arg via the MutableVal -> Val ctors, so this line was double incrementing the type's ref count, but could only decrement it once upon the Val's destruction. --- src/Expr.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Expr.cc b/src/Expr.cc index 9e71f27897..96ff79bb59 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -3967,7 +3967,7 @@ RecordCoerceExpr::~RecordCoerceExpr() Val* RecordCoerceExpr::Fold(Val* v) const { - RecordVal* val = new RecordVal(Type()->Ref()->AsRecordType()); + RecordVal* val = new RecordVal(Type()->AsRecordType()); RecordVal* rv = v->AsRecordVal(); for ( int i = 0; i < map_size; ++i ) From 8c807d19c37c6e719cb96394e11215e4894fc35c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 Jan 2013 20:08:36 -0800 Subject: [PATCH 25/49] Fixing exit-after-terminate when used with bare mode. --- src/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cc b/src/main.cc index 5999186240..fffedee3dd 100644 --- a/src/main.cc +++ b/src/main.cc @@ -1053,7 +1053,7 @@ int main(int argc, char** argv) io_sources.Register(thread_mgr, true); - if ( io_sources.Size() > 0 || have_pending_timers ) + if ( io_sources.Size() > 0 || have_pending_timers || BifConst::exit_only_after_terminate ) { if ( profiling_logger ) profiling_logger->Log(); From b08bdbce46bdccbb8df5d4904ca87be4d1f36c9e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 Jan 2013 20:09:19 -0800 Subject: [PATCH 26/49] Updating submodule(s). [nomail] --- aux/btest | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/btest b/aux/btest index 3585dc9a7a..ba0700fe44 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 3585dc9a7afe20d70cb77fc2472cc6bce3850b67 +Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb From 01c2bf4e0e28f430940e10756fa47a1ef9ced9c0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 30 Jan 2013 20:09:27 -0800 Subject: [PATCH 27/49] Updating submodule(s). [nomail] --- CHANGES | 8 ++++++++ VERSION | 2 +- aux/btest | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 64b1eb0aa1..cc026a97eb 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,12 @@ +2.1-310 | 2013-01-30 20:09:27 -0800 + + * Add an error for record coercions that would orphan a field. (Jon + Siwek) + + * Fixing several scripts where a field in an inlined record was + never removed after a code refactor. (Jon Siwek) + 2.1-307 | 2013-01-25 13:50:57 -0800 * Fix runaway reference counting bug in record coercion. (Jon Siwek) diff --git a/VERSION b/VERSION index 3148d7e818..a04d961dd5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-307 +2.1-310 diff --git a/aux/btest b/aux/btest index 6c0f099c52..ba0700fe44 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 6c0f099c52f9e7b5d6f3efafb63581a6cc0861a5 +Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb From d3814594ff4db81b9dc5a45ec4f6419fe90fe972 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 1 Feb 2013 08:03:01 -0800 Subject: [PATCH 28/49] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/broccoli | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index cc026a97eb..8b80bec68b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.1-311 | 2013-02-01 08:03:01 -0800 + + * Updating submodule(s). + 2.1-310 | 2013-01-30 20:09:27 -0800 * Add an error for record coercions that would orphan a field. (Jon diff --git a/VERSION b/VERSION index a04d961dd5..44ee79325e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-310 +2.1-311 diff --git a/aux/broccoli b/aux/broccoli index 073404dd29..c1ba9b44c4 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 073404dd29dc6e90ff0e4eb8bc836f8adbf3931e +Subproject commit c1ba9b44c4815c61c54c968f462ec5b0865e5990 From dd24d7af83abb68ccbf61488f246daf06e011972 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Fri, 1 Feb 2013 15:45:21 -0800 Subject: [PATCH 29/49] update input tests to use exit_only_after_terminate --- testing/btest/scripts/base/frameworks/input/basic.bro | 6 ++---- testing/btest/scripts/base/frameworks/input/bignumber.bro | 7 ++----- testing/btest/scripts/base/frameworks/input/binary.bro | 7 ++----- .../scripts/base/frameworks/input/empty-values-hashing.bro | 6 ++---- testing/btest/scripts/base/frameworks/input/emptyvals.bro | 5 +---- testing/btest/scripts/base/frameworks/input/event.bro | 5 +---- testing/btest/scripts/base/frameworks/input/executeraw.bro | 5 +---- .../btest/scripts/base/frameworks/input/invalidnumbers.bro | 5 +---- .../btest/scripts/base/frameworks/input/invalidtext.bro | 5 +---- .../btest/scripts/base/frameworks/input/missing-file.bro | 5 +---- .../scripts/base/frameworks/input/onecolumn-norecord.bro | 5 +---- .../scripts/base/frameworks/input/onecolumn-record.bro | 5 +---- testing/btest/scripts/base/frameworks/input/optional.bro | 5 +---- testing/btest/scripts/base/frameworks/input/port.bro | 5 +---- .../scripts/base/frameworks/input/predicate-stream.bro | 5 +---- testing/btest/scripts/base/frameworks/input/predicate.bro | 5 +---- .../scripts/base/frameworks/input/predicatemodify.bro | 5 +---- .../base/frameworks/input/predicatemodifyandreread.bro | 7 +++---- .../frameworks/input/predicaterefusesecondsamerecord.bro | 5 +---- testing/btest/scripts/base/frameworks/input/raw.bro | 5 +---- testing/btest/scripts/base/frameworks/input/repeat.bro | 5 +---- testing/btest/scripts/base/frameworks/input/reread.bro | 6 ++---- testing/btest/scripts/base/frameworks/input/rereadraw.bro | 5 +---- testing/btest/scripts/base/frameworks/input/set.bro | 5 +---- .../btest/scripts/base/frameworks/input/setseparator.bro | 5 +---- .../scripts/base/frameworks/input/setspecialcases.bro | 5 +---- testing/btest/scripts/base/frameworks/input/stream.bro | 5 +---- testing/btest/scripts/base/frameworks/input/streamraw.bro | 7 +++---- .../scripts/base/frameworks/input/subrecord-event.bro | 5 +---- testing/btest/scripts/base/frameworks/input/subrecord.bro | 5 +---- testing/btest/scripts/base/frameworks/input/tableevent.bro | 5 +---- testing/btest/scripts/base/frameworks/input/twotables.bro | 5 +---- .../scripts/base/frameworks/input/unsupported_types.bro | 5 +---- 33 files changed, 42 insertions(+), 134 deletions(-) diff --git a/testing/btest/scripts/base/frameworks/input/basic.bro b/testing/btest/scripts/base/frameworks/input/basic.bro index dfac84d062..ea6746c4db 100644 --- a/testing/btest/scripts/base/frameworks/input/basic.bro +++ b/testing/btest/scripts/base/frameworks/input/basic.bro @@ -1,10 +1,9 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + @TEST-START-FILE input.log #separator \x09 #path ssh @@ -14,7 +13,6 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/bignumber.bro b/testing/btest/scripts/base/frameworks/input/bignumber.bro index 5b93472551..19546d138c 100644 --- a/testing/btest/scripts/base/frameworks/input/bignumber.bro +++ b/testing/btest/scripts/base/frameworks/input/bignumber.bro @@ -1,10 +1,9 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + @TEST-START-FILE input.log #separator \x09 #fields i c @@ -13,8 +12,6 @@ -9223372036854775800 18446744073709551612 @TEST-END-FILE -@load frameworks/communication/listen - global outfile: file; module A; diff --git a/testing/btest/scripts/base/frameworks/input/binary.bro b/testing/btest/scripts/base/frameworks/input/binary.bro index 8d75abc5a9..d8345dbfd3 100644 --- a/testing/btest/scripts/base/frameworks/input/binary.bro +++ b/testing/btest/scripts/base/frameworks/input/binary.bro @@ -1,10 +1,9 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + redef InputAscii::separator = "|"; redef InputAscii::set_separator = ","; redef InputAscii::empty_field = "(empty)"; @@ -25,8 +24,6 @@ abc\xff\x7cdef|DATA2 #end|2012-07-20-01-49-19 @TEST-END-FILE -@load frameworks/communication/listen - global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro index c8760b467e..5226586ad1 100644 --- a/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro +++ b/testing/btest/scripts/base/frameworks/input/empty-values-hashing.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 @@ -23,8 +20,9 @@ 2 TEST TEST @TEST-END-FILE -@load frameworks/communication/listen +@load base/frameworks/communication # let network-time run +redef exit_only_after_terminate = T; module A; diff --git a/testing/btest/scripts/base/frameworks/input/emptyvals.bro b/testing/btest/scripts/base/frameworks/input/emptyvals.bro index 94b0f1b620..cef0606646 100644 --- a/testing/btest/scripts/base/frameworks/input/emptyvals.bro +++ b/testing/btest/scripts/base/frameworks/input/emptyvals.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ T 1 - 2 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/event.bro b/testing/btest/scripts/base/frameworks/input/event.bro index ba47d5e3f2..ac956a4a19 100644 --- a/testing/btest/scripts/base/frameworks/input/event.bro +++ b/testing/btest/scripts/base/frameworks/input/event.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/executeraw.bro b/testing/btest/scripts/base/frameworks/input/executeraw.bro index 626b9cdfd2..bcec76444f 100644 --- a/testing/btest/scripts/base/frameworks/input/executeraw.bro +++ b/testing/btest/scripts/base/frameworks/input/executeraw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: cat out.tmp | sed 's/^ *//g' >out @@ -17,7 +14,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro index 1deec605ae..3321b41253 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidnumbers.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -17,7 +14,7 @@ Justtext 1 9223372036854775800 -18446744073709551612 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/invalidtext.bro b/testing/btest/scripts/base/frameworks/input/invalidtext.bro index 75efb1247d..ffca41dbee 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidtext.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidtext.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -15,7 +12,7 @@ 5 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro index aa5acf619e..8fb12ba412 100644 --- a/testing/btest/scripts/base/frameworks/input/missing-file.bro +++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro @@ -1,11 +1,8 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff bro/.stderr -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro index c08b1420fb..989d6352a3 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-norecord.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,7 +10,7 @@ T -42 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro index 9e420e75fe..54c8e9007e 100644 --- a/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro +++ b/testing/btest/scripts/base/frameworks/input/onecolumn-record.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -13,7 +10,7 @@ T -42 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/optional.bro b/testing/btest/scripts/base/frameworks/input/optional.bro index 2fe0e5c86f..1fb9bce68b 100644 --- a/testing/btest/scripts/base/frameworks/input/optional.bro +++ b/testing/btest/scripts/base/frameworks/input/optional.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/port.bro b/testing/btest/scripts/base/frameworks/input/port.bro index 081c59559b..31776c5939 100644 --- a/testing/btest/scripts/base/frameworks/input/port.bro +++ b/testing/btest/scripts/base/frameworks/input/port.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -12,7 +9,7 @@ 1.2.3.6 30 unknown @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro index 8cf927e346..8caea9ad7a 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate-stream.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate-stream.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -23,7 +20,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicate.bro b/testing/btest/scripts/base/frameworks/input/predicate.bro index 8fb33242e8..f9ff5c09ee 100644 --- a/testing/btest/scripts/base/frameworks/input/predicate.bro +++ b/testing/btest/scripts/base/frameworks/input/predicate.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro index 17467bbc27..9d5eb3bd07 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodify.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodify.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ 2 T test2 idx2 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro index 5a9e993651..174d145054 100644 --- a/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro +++ b/testing/btest/scripts/base/frameworks/input/predicatemodifyandreread.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 @@ -58,7 +55,9 @@ 1 T test1 idx1 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; + +@load base/frameworks/communication # let network-time run redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro index ba0b468cdc..247b301bfa 100644 --- a/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro +++ b/testing/btest/scripts/base/frameworks/input/predicaterefusesecondsamerecord.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -17,7 +14,7 @@ need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 95 1342569600 1.228.83. need-to-know 8c864306-d21a-37b1-8705-746a786719bf medium 65 1342656000 1.228.83.33 - - 9318 HANARO-AS Hanaro Telecom Inc. 1.224.0.0/13 apnic KR spam infrastructure spamming;malware domain public http://reputation.alienvault.com/reputation.generic @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/raw.bro b/testing/btest/scripts/base/frameworks/input/raw.bro index d15aec22bb..0c4a438549 100644 --- a/testing/btest/scripts/base/frameworks/input/raw.bro +++ b/testing/btest/scripts/base/frameworks/input/raw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,7 +13,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/repeat.bro b/testing/btest/scripts/base/frameworks/input/repeat.bro index a966ac064e..08c7ab1592 100644 --- a/testing/btest/scripts/base/frameworks/input/repeat.bro +++ b/testing/btest/scripts/base/frameworks/input/repeat.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,7 +10,7 @@ 1 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/reread.bro b/testing/btest/scripts/base/frameworks/input/reread.bro index 11aa873f9d..e7639d3e48 100644 --- a/testing/btest/scripts/base/frameworks/input/reread.bro +++ b/testing/btest/scripts/base/frameworks/input/reread.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 2 @@ -59,8 +56,9 @@ F -48 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +@load base/frameworks/communication # let network-time run +redef exit_only_after_terminate = T; redef InputAscii::empty_field = "EMPTY"; module A; diff --git a/testing/btest/scripts/base/frameworks/input/rereadraw.bro b/testing/btest/scripts/base/frameworks/input/rereadraw.bro index 2fdcdc8f9e..16f13c21e1 100644 --- a/testing/btest/scripts/base/frameworks/input/rereadraw.bro +++ b/testing/btest/scripts/base/frameworks/input/rereadraw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -16,7 +13,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/set.bro b/testing/btest/scripts/base/frameworks/input/set.bro index b2b5cea323..95a7a86a28 100644 --- a/testing/btest/scripts/base/frameworks/input/set.bro +++ b/testing/btest/scripts/base/frameworks/input/set.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -16,7 +13,7 @@ 192.168.17.42 @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/setseparator.bro b/testing/btest/scripts/base/frameworks/input/setseparator.bro index b7148d80bd..efc4b259f6 100644 --- a/testing/btest/scripts/base/frameworks/input/setseparator.bro +++ b/testing/btest/scripts/base/frameworks/input/setseparator.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -13,7 +10,7 @@ redef InputAscii::set_separator = "|"; -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro index 022eac9731..27211a590e 100644 --- a/testing/btest/scripts/base/frameworks/input/setspecialcases.bro +++ b/testing/btest/scripts/base/frameworks/input/setspecialcases.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out @@ -17,7 +14,7 @@ @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/stream.bro b/testing/btest/scripts/base/frameworks/input/stream.bro index 1ecd8a2eb0..1d797f30d3 100644 --- a/testing/btest/scripts/base/frameworks/input/stream.bro +++ b/testing/btest/scripts/base/frameworks/input/stream.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 @@ -25,7 +22,7 @@ F -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/streamraw.bro b/testing/btest/scripts/base/frameworks/input/streamraw.bro index 3bc06f7dea..a7cb718975 100644 --- a/testing/btest/scripts/base/frameworks/input/streamraw.bro +++ b/testing/btest/scripts/base/frameworks/input/streamraw.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 3 @@ -10,6 +7,8 @@ # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out +redef exit_only_after_terminate = T; + @TEST-START-FILE input1.log sdfkh:KH;fdkncv;ISEUp34:Fkdj;YVpIODhfDF @TEST-END-FILE @@ -27,7 +26,7 @@ sdf 3rw43wRRERLlL#RWERERERE. @TEST-END-FILE -@load frameworks/communication/listen +@load base/frameworks/communication # let network-time run module A; diff --git a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro index 4e7dc1690a..96774f9c29 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord-event.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord-event.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/subrecord.bro b/testing/btest/scripts/base/frameworks/input/subrecord.bro index 512b8ec58f..6f6c286828 100644 --- a/testing/btest/scripts/base/frameworks/input/subrecord.bro +++ b/testing/btest/scripts/base/frameworks/input/subrecord.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index 723e519237..a409803440 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -19,7 +16,7 @@ 7 T @TEST-END-FILE -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/input/twotables.bro b/testing/btest/scripts/base/frameworks/input/twotables.bro index 83ae86cd46..79b33f7182 100644 --- a/testing/btest/scripts/base/frameworks/input/twotables.bro +++ b/testing/btest/scripts/base/frameworks/input/twotables.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: cp input1.log input.log # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: sleep 5 @@ -34,7 +31,7 @@ F -44 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; redef InputAscii::empty_field = "EMPTY"; diff --git a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro index e1350f61a9..937c963f27 100644 --- a/testing/btest/scripts/base/frameworks/input/unsupported_types.bro +++ b/testing/btest/scripts/base/frameworks/input/unsupported_types.bro @@ -1,6 +1,3 @@ -# (uses listen.bro just to ensure input sources are more reliably fully-read). -# @TEST-SERIALIZE: comm -# # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait -k 5 # @TEST-EXEC: btest-diff out @@ -14,7 +11,7 @@ whatever T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.00 @TEST-END-FILE @load base/protocols/ssh -@load frameworks/communication/listen +redef exit_only_after_terminate = T; global outfile: file; From 26bf99c5a334c8bd23dd436aa3dcfad9c0e80535 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Feb 2013 14:59:02 -0600 Subject: [PATCH 30/49] Add parsing for GTPv1 extension headers and control messages. Added a generic gtpv1_message event generated for any GTP message type. Added specific events for the create/update/delete PDP context request/response messages. Addresses #934. --- scripts/base/frameworks/tunnels/main.bro | 6 +- scripts/base/init-bare.bro | 140 ++++ src/NetVar.cc | 2 - src/NetVar.h | 1 - src/event.bif | 61 ++ src/gtpv1-analyzer.pac | 694 ++++++++++++++++-- src/gtpv1-protocol.pac | 477 +++++++++++- src/types.bif | 14 + .../Baseline/core.tunnels.gtp.ext_header/out | 2 + .../core.tunnels.gtp.non_recursive/out | 2 +- .../core.tunnels.gtp.pdp_ctx_messages/out | 24 + .../dpd.log | 6 +- .../tunnel.log | 4 +- .../Traces/tunnels/gtp/gtp_control_prime.pcap | Bin 0 -> 1292 bytes .../tunnels/gtp/gtp_create_pdp_ctx.pcap | Bin 0 -> 713 bytes .../Traces/tunnels/gtp/gtp_ext_header.pcap | Bin 0 -> 1648 bytes .../btest/core/tunnels/gtp/ext_header.test | 8 + .../core/tunnels/gtp/pdp_ctx_messages.test | 56 ++ 18 files changed, 1435 insertions(+), 62 deletions(-) create mode 100644 testing/btest/Baseline/core.tunnels.gtp.ext_header/out create mode 100644 testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out create mode 100644 testing/btest/Traces/tunnels/gtp/gtp_control_prime.pcap create mode 100644 testing/btest/Traces/tunnels/gtp/gtp_create_pdp_ctx.pcap create mode 100644 testing/btest/Traces/tunnels/gtp/gtp_ext_header.pcap create mode 100644 testing/btest/core/tunnels/gtp/ext_header.test create mode 100644 testing/btest/core/tunnels/gtp/pdp_ctx_messages.test diff --git a/scripts/base/frameworks/tunnels/main.bro b/scripts/base/frameworks/tunnels/main.bro index a3db7061d3..a8fc6c8236 100644 --- a/scripts/base/frameworks/tunnels/main.bro +++ b/scripts/base/frameworks/tunnels/main.bro @@ -88,10 +88,10 @@ redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] }; const teredo_ports = { 3544/udp }; redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] }; -const gtpv1u_ports = { 2152/udp }; -redef dpd_config += { [ANALYZER_GTPV1] = [$ports = gtpv1u_ports] }; +const gtpv1_ports = { 2152/udp, 2123/udp }; +redef dpd_config += { [ANALYZER_GTPV1] = [$ports = gtpv1_ports] }; -redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1u_ports }; +redef likely_server_ports += { ayiya_ports, teredo_ports, gtpv1_ports }; event bro_init() &priority=5 { diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 23321a35dd..b6187df0d9 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -1488,6 +1488,146 @@ type gtpv1_hdr: record { next_type: count &optional; }; +type gtp_cause: count; +type gtp_imsi: count; +type gtp_teardown_ind: bool; +type gtp_nsapi: count; +type gtp_recovery: count; +type gtp_teid1: count; +type gtp_teid_control_plane: count; +type gtp_charging_id: count; +type gtp_charging_gateway_addr: addr; +type gtp_trace_reference: count; +type gtp_trace_type: count; +type gtp_tft: string; +type gtp_trigger_id: string; +type gtp_omc_id: string; +type gtp_reordering_required: bool; +type gtp_proto_config_options: string; +type gtp_charging_characteristics: count; +type gtp_selection_mode: count; +type gtp_access_point_name: string; +type gtp_msisdn: string; + +type gtp_gsn_addr: record { + ## If the GSN Address information element has length 4 or 16, then this + ## field is set to be the informational element's value interpreted as + ## an IPv4 or IPv6 address, respectively. + ip: addr &optional; + ## This field is set if it's not an IPv4 or IPv6 address. + other: string &optional; +}; + +type gtp_end_user_addr: record { + pdp_type_org: count; + pdp_type_num: count; + ## Set if the End User Address information element is IPv4/IPv6. + pdp_ip: addr &optional; + ## Set if the End User Address information element isn't IPv4/IPv6. + pdp_other_addr: string &optional; +}; + +type gtp_rai: record { + mcc: count; + mnc: count; + lac: count; + rac: count; +}; + +type gtp_qos_profile: record { + priority: count; + data: string; +}; + +type gtp_private_extension: record { + id: count; + value: string; +}; + +type gtp_create_pdp_ctx_request_elements: record { + imsi: gtp_imsi &optional; + rai: gtp_rai &optional; + recovery: gtp_recovery &optional; + select_mode: gtp_selection_mode &optional; + data1: gtp_teid1; + cp: gtp_teid_control_plane &optional; + nsapi: gtp_nsapi; + linked_nsapi: gtp_nsapi &optional; + charge_character: gtp_charging_characteristics &optional; + trace_ref: gtp_trace_reference &optional; + trace_type: gtp_trace_type &optional; + end_user_addr: gtp_end_user_addr &optional; + ap_name: gtp_access_point_name &optional; + opts: gtp_proto_config_options &optional; + signal_addr: gtp_gsn_addr; + user_addr: gtp_gsn_addr; + msisdn: gtp_msisdn &optional; + qos_prof: gtp_qos_profile; + tft: gtp_tft &optional; + trigger_id: gtp_trigger_id &optional; + omc_id: gtp_omc_id &optional; + ext: gtp_private_extension &optional; +}; + +type gtp_create_pdp_ctx_response_elements: record { + cause: gtp_cause; + reorder_req: gtp_reordering_required &optional; + recovery: gtp_recovery &optional; + data1: gtp_teid1 &optional; + cp: gtp_teid_control_plane &optional; + charging_id: gtp_charging_id &optional; + end_user_addr: gtp_end_user_addr &optional; + opts: gtp_proto_config_options &optional; + cp_addr: gtp_gsn_addr &optional; + user_addr: gtp_gsn_addr &optional; + qos_prof: gtp_qos_profile &optional; + charge_gateway: gtp_charging_gateway_addr &optional; + ext: gtp_private_extension &optional; +}; + +type gtp_update_pdp_ctx_request_elements: record { + imsi: gtp_imsi &optional; + rai: gtp_rai &optional; + recovery: gtp_recovery &optional; + data1: gtp_teid1; + cp: gtp_teid_control_plane &optional; + nsapi: gtp_nsapi; + trace_ref: gtp_trace_reference &optional; + trace_type: gtp_trace_type &optional; + cp_addr: gtp_gsn_addr; + user_addr: gtp_gsn_addr; + qos_prof: gtp_qos_profile; + tft: gtp_tft &optional; + trigger_id: gtp_trigger_id &optional; + omc_id: gtp_omc_id &optional; + ext: gtp_private_extension &optional; + end_user_addr: gtp_end_user_addr &optional; +}; + +type gtp_update_pdp_ctx_response_elements: record { + cause: gtp_cause; + recovery: gtp_recovery &optional; + data1: gtp_teid1 &optional; + cp: gtp_teid_control_plane &optional; + charging_id: gtp_charging_id &optional; + cp_addr: gtp_gsn_addr &optional; + user_addr: gtp_gsn_addr &optional; + qos_prof: gtp_qos_profile &optional; + charge_gateway: gtp_charging_gateway_addr &optional; + ext: gtp_private_extension &optional; +}; + +type gtp_delete_pdp_ctx_request_elements: record { + teardown_ind: gtp_teardown_ind &optional; + nsapi: gtp_nsapi; + ext: gtp_private_extension &optional; +}; + +type gtp_delete_pdp_ctx_response_elements: record { + cause: gtp_cause; + ext: gtp_private_extension &optional; +}; + ## Definition of "secondary filters". A secondary filter is a BPF filter given as ## index in this table. For each such filter, the corresponding event is raised for ## all matching packets. diff --git a/src/NetVar.cc b/src/NetVar.cc index 1783130f34..248ae15e1a 100644 --- a/src/NetVar.cc +++ b/src/NetVar.cc @@ -5,7 +5,6 @@ #include "Var.h" #include "NetVar.h" -RecordType* gtpv1_hdr_type; RecordType* conn_id; RecordType* endpoint; RecordType* endpoint_stats; @@ -309,7 +308,6 @@ void init_net_var() #include "input.bif.netvar_init" #include "reporter.bif.netvar_init" - gtpv1_hdr_type = internal_type("gtpv1_hdr")->AsRecordType(); conn_id = internal_type("conn_id")->AsRecordType(); endpoint = internal_type("endpoint")->AsRecordType(); endpoint_stats = internal_type("endpoint_stats")->AsRecordType(); diff --git a/src/NetVar.h b/src/NetVar.h index 4bb2d2a7f9..2561fa0ad9 100644 --- a/src/NetVar.h +++ b/src/NetVar.h @@ -8,7 +8,6 @@ #include "EventRegistry.h" #include "Stats.h" -extern RecordType* gtpv1_hdr_type; extern RecordType* conn_id; extern RecordType* endpoint; extern RecordType* endpoint_stats; diff --git a/src/event.bif b/src/event.bif index 8dd940f38b..393021024a 100644 --- a/src/event.bif +++ b/src/event.bif @@ -577,6 +577,13 @@ event teredo_origin_indication%(outer: connection, inner: teredo_hdr%); ## it may become particularly expensive for real-time analysis. event teredo_bubble%(outer: connection, inner: teredo_hdr%); +## Generated for any GTP message with a GTPv1 header. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +event gtpv1_message%(c: connection, hdr: gtpv1_hdr%); + ## Generated for GTPv1 G-PDU packets. That is, packets with a UDP payload ## that includes a GTP header followed by an IPv4 or IPv6 packet. ## @@ -590,6 +597,60 @@ event teredo_bubble%(outer: connection, inner: teredo_hdr%); ## it may become particularly expensive for real-time analysis. event gtpv1_g_pdu_packet%(outer: connection, inner_gtp: gtpv1_hdr, inner_ip: pkt_hdr%); +## Generated for GTPv1-C Create PDP Context Request messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_create_pdp_ctx_request%(c: connection, hdr: gtpv1_hdr, elements: gtp_create_pdp_ctx_request_elements%); + +## Generated for GTPv1-C Create PDP Context Response messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_create_pdp_ctx_response%(c: connection, hdr: gtpv1_hdr, elements: gtp_create_pdp_ctx_response_elements%); + +## Generated for GTPv1-C Update PDP Context Request messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_update_pdp_ctx_request%(c: connection, hdr: gtpv1_hdr, elements: gtp_update_pdp_ctx_request_elements%); + +## Generated for GTPv1-C Update PDP Context Response messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_update_pdp_ctx_response%(c: connection, hdr: gtpv1_hdr, elements: gtp_update_pdp_ctx_response_elements%); + +## Generated for GTPv1-C Delete PDP Context Request messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_delete_pdp_ctx_request%(c: connection, hdr: gtpv1_hdr, elements: gtp_delete_pdp_ctx_request_elements%); + +## Generated for GTPv1-C Delete PDP Context Response messages. +## +## c: The connection over which the message is sent. +## +## hdr: The GTPv1 header. +## +## elements: The set of Information Elements comprising the message. +event gtpv1_delete_pdp_ctx_response%(c: connection, hdr: gtpv1_hdr, elements: gtp_delete_pdp_ctx_response_elements%); + ## Generated for every packet that has a non-empty transport-layer payload. ## This is a very low-level and expensive event that should be avoided when ## at all possible. It's usually infeasible to handle when processing even diff --git a/src/gtpv1-analyzer.pac b/src/gtpv1-analyzer.pac index 08fbdef74a..51d28123cb 100644 --- a/src/gtpv1-analyzer.pac +++ b/src/gtpv1-analyzer.pac @@ -1,4 +1,602 @@ +%code{ +RecordVal* BuildGTPv1Hdr(const GTPv1_Header* pdu) + { + RecordVal* rv = new RecordVal(BifType::Record::gtpv1_hdr); + + rv->Assign(0, new Val(pdu->version(), TYPE_COUNT)); + rv->Assign(1, new Val(pdu->pt_flag(), TYPE_BOOL)); + rv->Assign(2, new Val(pdu->rsv(), TYPE_BOOL)); + rv->Assign(3, new Val(pdu->e_flag(), TYPE_BOOL)); + rv->Assign(4, new Val(pdu->s_flag(), TYPE_BOOL)); + rv->Assign(5, new Val(pdu->pn_flag(), TYPE_BOOL)); + rv->Assign(6, new Val(pdu->msg_type(), TYPE_COUNT)); + rv->Assign(7, new Val(pdu->length(), TYPE_COUNT)); + rv->Assign(8, new Val(pdu->teid(), TYPE_COUNT)); + + if ( pdu->has_opt() ) + { + rv->Assign(9, new Val(pdu->opt_hdr()->seq(), TYPE_COUNT)); + rv->Assign(10, new Val(pdu->opt_hdr()->n_pdu(), TYPE_COUNT)); + rv->Assign(11, new Val(pdu->opt_hdr()->next_type(), TYPE_COUNT)); + } + + return rv; + } + +Val* BuildIMSI(const InformationElement* ie) + { + return new Val(ie->imsi()->value(), TYPE_COUNT); + } + +Val* BuildRAI(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_rai); + ev->Assign(0, new Val(ie->rai()->mcc(), TYPE_COUNT)); + ev->Assign(1, new Val(ie->rai()->mnc(), TYPE_COUNT)); + ev->Assign(2, new Val(ie->rai()->lac(), TYPE_COUNT)); + ev->Assign(3, new Val(ie->rai()->rac(), TYPE_COUNT)); + return ev; + } + +Val* BuildRecovery(const InformationElement* ie) + { + return new Val(ie->recovery()->restart_counter(), TYPE_COUNT); + } + +Val* BuildSelectionMode(const InformationElement* ie) + { + return new Val(ie->selection_mode()->mode(), TYPE_COUNT); + } + +Val* BuildTEID1(const InformationElement* ie) + { + return new Val(ie->teid1()->value(), TYPE_COUNT); + } + +Val* BuildTEID_ControlPlane(const InformationElement* ie) + { + return new Val(ie->teidcp()->value(), TYPE_COUNT); + } + +Val* BuildNSAPI(const InformationElement* ie) + { + return new Val(ie->nsapi()->nsapi(), TYPE_COUNT); + } + +Val* BuildChargingCharacteristics(const InformationElement* ie) + { + return new Val(ie->charging_characteristics()->value(), TYPE_COUNT); + } + +Val* BuildTraceReference(const InformationElement* ie) + { + return new Val(ie->trace_reference()->value(), TYPE_COUNT); + } + +Val* BuildTraceType(const InformationElement* ie) + { + return new Val(ie->trace_type()->value(), TYPE_COUNT); + } + +Val* BuildEndUserAddr(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_end_user_addr); + ev->Assign(0, new Val(ie->end_user_addr()->pdp_type_org(), TYPE_COUNT)); + ev->Assign(1, new Val(ie->end_user_addr()->pdp_type_num(), TYPE_COUNT)); + + int len = ie->end_user_addr()->pdp_addr().length(); + + if ( len > 0 ) + { + const uint8* d = ie->end_user_addr()->pdp_addr().data(); + + switch ( ie->end_user_addr()->pdp_type_num() ) { + case 0x21: + ev->Assign(2, new AddrVal( + IPAddr(IPv4, (const uint32*) d, IPAddr::Network))); + break; + case 0x57: + ev->Assign(2, new AddrVal( + IPAddr(IPv6, (const uint32*) d, IPAddr::Network))); + break; + default: + ev->Assign(3, new StringVal( + new BroString((const u_char*) d, len, 0))); + break; + } + } + + return ev; + } + +Val* BuildAccessPointName(const InformationElement* ie) + { + BroString* bs = new BroString((const u_char*) ie->ap_name()->value().data(), + ie->ap_name()->value().length(), 0); + return new StringVal(bs); + } + +Val* BuildProtoConfigOptions(const InformationElement* ie) + { + const u_char* d = (const u_char*) ie->proto_config_opts()->value().data(); + int len = ie->proto_config_opts()->value().length(); + return new StringVal(new BroString(d, len, 0)); + } + +Val* BuildGSN_Addr(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_gsn_addr); + + int len = ie->gsn_addr()->value().length(); + const uint8* d = ie->gsn_addr()->value().data(); + + if ( len == 4 ) + ev->Assign(0, new AddrVal( + IPAddr(IPv4, (const uint32*) d, IPAddr::Network))); + else if ( len == 16 ) + ev->Assign(0, new AddrVal( + IPAddr(IPv6, (const uint32*) d, IPAddr::Network))); + else + ev->Assign(1, new StringVal(new BroString((const u_char*) d, len, 0))); + + return ev; + } + +Val* BuildMSISDN(const InformationElement* ie) + { + const u_char* d = (const u_char*) ie->msisdn()->value().data(); + int len = ie->msisdn()->value().length(); + return new StringVal(new BroString(d, len, 0)); + } + +Val* BuildQoS_Profile(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_qos_profile); + + const u_char* d = (const u_char*) ie->qos_profile()->data().data(); + int len = ie->qos_profile()->data().length(); + + ev->Assign(0, new Val(ie->qos_profile()->alloc_retention_priority(), + TYPE_COUNT)); + ev->Assign(1, new StringVal(new BroString(d, len, 0))); + + return ev; + } + +Val* BuildTrafficFlowTemplate(const InformationElement* ie) + { + const uint8* d = ie->traffic_flow_template()->value().data(); + int len = ie->traffic_flow_template()->value().length(); + return new StringVal(new BroString((const u_char*) d, len, 0)); + } + +Val* BuildTriggerID(const InformationElement* ie) + { + const uint8* d = ie->trigger_id()->value().data(); + int len = ie->trigger_id()->value().length(); + return new StringVal(new BroString((const u_char*) d, len, 0)); + } + +Val* BuildOMC_ID(const InformationElement* ie) + { + const uint8* d = ie->omc_id()->value().data(); + int len = ie->omc_id()->value().length(); + return new StringVal(new BroString((const u_char*) d, len, 0)); + } + +Val* BuildPrivateExt(const InformationElement* ie) + { + RecordVal* ev = new RecordVal(BifType::Record::gtp_private_extension); + + const uint8* d = ie->private_ext()->value().data(); + int len = ie->private_ext()->value().length(); + + ev->Assign(0, new Val(ie->private_ext()->id(), TYPE_COUNT)); + ev->Assign(1, new StringVal(new BroString((const u_char*) d, len, 0))); + + return ev; + } + +Val* BuildCause(const InformationElement* ie) + { + return new Val(ie->cause()->value(), TYPE_COUNT); + } + +Val* BuildReorderReq(const InformationElement* ie) + { + return new Val(ie->reorder_req()->req(), TYPE_BOOL); + } + +Val* BuildChargingID(const InformationElement* ie) + { + return new Val(ie->charging_id()->value(), TYPE_COUNT);; + } + +Val* BuildChargingGatewayAddr(const InformationElement* ie) + { + const uint8* d = ie->charging_gateway_addr()->value().data(); + int len = ie->charging_gateway_addr()->value().length(); + if ( len == 4 ) + return new AddrVal(IPAddr(IPv4, (const uint32*) d, IPAddr::Network)); + else if ( len == 16 ) + return new AddrVal(IPAddr(IPv6, (const uint32*) d, IPAddr::Network)); + else + return 0; + } + +Val* BuildTeardownInd(const InformationElement* ie) + { + return new Val(ie->teardown_ind()->ind(), TYPE_BOOL); + } + +void CreatePDP_Request(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_create_pdp_ctx_request ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_create_pdp_ctx_request_elements); + + const vector * v = pdu->create_pdp_ctx_request(); + + bool second_nsapi = false; + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_IMSI: + rv->Assign(0, BuildIMSI(ie)); + break; + case GTPv1::TYPE_RAI: + rv->Assign(1, BuildRAI(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(2, BuildRecovery(ie)); + break; + case GTPv1::TYPE_SELECTION_MODE: + rv->Assign(3, BuildSelectionMode(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(4, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(5, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_NSAPI: + if ( second_nsapi ) + rv->Assign(7, BuildNSAPI(ie)); + else + { + second_nsapi = true; + rv->Assign(6, BuildNSAPI(ie)); + } + break; + case GTPv1::TYPE_CHARGING_CHARACTERISTICS: + rv->Assign(8, BuildChargingCharacteristics(ie)); + break; + case GTPv1::TYPE_TRACE_REFERENCE: + rv->Assign(9, BuildTraceReference(ie)); + break; + case GTPv1::TYPE_TRACE_TYPE: + rv->Assign(10, BuildTraceType(ie)); + break; + case GTPv1::TYPE_END_USER_ADDR: + rv->Assign(11, BuildEndUserAddr(ie)); + break; + case GTPv1::TYPE_ACCESS_POINT_NAME: + rv->Assign(12, BuildAccessPointName(ie)); + break; + case GTPv1::TYPE_PROTO_CONFIG_OPTIONS: + rv->Assign(13, BuildProtoConfigOptions(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(15, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(14, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_MSISDN: + rv->Assign(16, BuildMSISDN(ie)); + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(17, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_TRAFFIC_FLOW_TEMPLATE: + rv->Assign(18, BuildTrafficFlowTemplate(ie)); + break; + case GTPv1::TYPE_TRIGGER_ID: + rv->Assign(19, BuildTriggerID(ie)); + break; + case GTPv1::TYPE_OMC_ID: + rv->Assign(20, BuildOMC_ID(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(21, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_create_pdp_ctx_request(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void CreatePDP_Response(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_create_pdp_ctx_response ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_create_pdp_ctx_response_elements); + + const vector * v = pdu->create_pdp_ctx_response(); + + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_CAUSE: + rv->Assign(0, BuildCause(ie)); + break; + case GTPv1::TYPE_REORDER_REQ: + rv->Assign(1, BuildReorderReq(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(2, BuildRecovery(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(3, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(4, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_CHARGING_ID: + rv->Assign(5, BuildChargingID(ie)); + break; + case GTPv1::TYPE_END_USER_ADDR: + rv->Assign(6, BuildEndUserAddr(ie)); + break; + case GTPv1::TYPE_PROTO_CONFIG_OPTIONS: + rv->Assign(7, BuildProtoConfigOptions(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(9, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(8, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(10, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_CHARGING_GATEWAY_ADDR: + rv->Assign(11, BuildChargingGatewayAddr(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(12, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_create_pdp_ctx_response(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void UpdatePDP_Request(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_update_pdp_ctx_request ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_update_pdp_ctx_request_elements); + + const vector * v = pdu->update_pdp_ctx_request(); + + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_IMSI: + rv->Assign(0, BuildIMSI(ie)); + break; + case GTPv1::TYPE_RAI: + rv->Assign(1, BuildRAI(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(2, BuildRecovery(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(3, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(4, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_NSAPI: + rv->Assign(5, BuildNSAPI(ie)); + break; + case GTPv1::TYPE_TRACE_REFERENCE: + rv->Assign(6, BuildTraceReference(ie)); + break; + case GTPv1::TYPE_TRACE_TYPE: + rv->Assign(7, BuildTraceType(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(9, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(8, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(10, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_TRAFFIC_FLOW_TEMPLATE: + rv->Assign(11, BuildTrafficFlowTemplate(ie)); + break; + case GTPv1::TYPE_TRIGGER_ID: + rv->Assign(12, BuildTriggerID(ie)); + break; + case GTPv1::TYPE_OMC_ID: + rv->Assign(13, BuildOMC_ID(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(14, BuildPrivateExt(ie)); + break; + case GTPv1::TYPE_END_USER_ADDR: + rv->Assign(15, BuildEndUserAddr(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_update_pdp_ctx_request(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void UpdatePDP_Response(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_update_pdp_ctx_response ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_update_pdp_ctx_response_elements); + + const vector * v = pdu->update_pdp_ctx_response(); + + bool second_gsn_addr = false; + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_CAUSE: + rv->Assign(0, BuildCause(ie)); + break; + case GTPv1::TYPE_RECOVERY: + rv->Assign(1, BuildRecovery(ie)); + break; + case GTPv1::TYPE_TEID1: + rv->Assign(2, BuildTEID1(ie)); + break; + case GTPv1::TYPE_TEID_CONTROL_PLANE: + rv->Assign(3, BuildTEID_ControlPlane(ie)); + break; + case GTPv1::TYPE_CHARGING_ID: + rv->Assign(4, BuildChargingID(ie)); + break; + case GTPv1::TYPE_GSN_ADDR: + if ( second_gsn_addr ) + rv->Assign(6, BuildGSN_Addr(ie)); + else + { + second_gsn_addr = true; + rv->Assign(5, BuildGSN_Addr(ie)); + } + break; + case GTPv1::TYPE_QOS_PROFILE: + rv->Assign(7, BuildQoS_Profile(ie)); + break; + case GTPv1::TYPE_CHARGING_GATEWAY_ADDR: + rv->Assign(8, BuildChargingGatewayAddr(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(9, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_update_pdp_ctx_response(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void DeletePDP_Request(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_delete_pdp_ctx_request ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_delete_pdp_ctx_request_elements); + + const vector * v = pdu->delete_pdp_ctx_request(); + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_TEARDOWN_IND: + rv->Assign(0, BuildTeardownInd(ie)); + break; + case GTPv1::TYPE_NSAPI: + rv->Assign(1, BuildNSAPI(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(2, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_delete_pdp_ctx_request(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } + +void DeletePDP_Response(const BroAnalyzer& a, const GTPv1_Header* pdu) + { + if ( ! ::gtpv1_delete_pdp_ctx_response ) return; + + RecordVal* rv = new RecordVal( + BifType::Record::gtp_delete_pdp_ctx_response_elements); + + const vector * v = pdu->delete_pdp_ctx_response(); + + for ( size_t i = 0; i < v->size(); ++i ) + { + const InformationElement* ie = (*v)[i]; + + switch ( ie->type() ) { + case GTPv1::TYPE_CAUSE: + rv->Assign(0, BuildCause(ie)); + break; + case GTPv1::TYPE_PRIVATE_EXT: + rv->Assign(1, BuildPrivateExt(ie)); + break; + default: + a->Weird(fmt("gtp_invalid_info_element_%d", (*v)[i]->type())); + break; + } + } + + BifEvent::generate_gtpv1_delete_pdp_ctx_response(a, a->Conn(), + BuildGTPv1Hdr(pdu), rv); + } +%} + connection GTPv1_Conn(bro_analyzer: BroAnalyzer) { upflow = GTPv1_Flow(true); @@ -27,17 +625,17 @@ connection GTPv1_Conn(bro_analyzer: BroAnalyzer) %} } -%code{ -inline void violate(const char* r, const BroAnalyzer& a, const bytestring& p) - { - a->ProtocolViolation(r, (const char*) p.data(), p.length()); - } -%} - flow GTPv1_Flow(is_orig: bool) { datagram = GTPv1_Header withcontext(connection, this); + function violate(r: string, pdu: GTPv1_Header): void + %{ + BroAnalyzer a = connection()->bro_analyzer(); + const_bytestring b = ${pdu.sourcedata}; + a->ProtocolViolation(r.c_str(), (const char*) b.begin(), b.length()); + %} + function process_gtpv1(pdu: GTPv1_Header): bool %{ BroAnalyzer a = connection()->bro_analyzer(); @@ -55,14 +653,14 @@ flow GTPv1_Flow(is_orig: bool) if ( e && e->LastType() == BifEnum::Tunnel::GTPv1 ) { // GTP is never tunneled in GTP so, this must be a regular packet - violate("GTP-in-GTP", a, ${pdu.packet}); + violate("GTP-in-GTP", pdu); return false; } if ( ${pdu.version} != 1 ) { // Only know of GTPv1 with Version == 1 - violate("GTPv1 bad Version", a, ${pdu.packet}); + violate("GTPv1 bad Version", pdu); return false; } @@ -72,21 +670,46 @@ flow GTPv1_Flow(is_orig: bool) return false; } - if ( ${pdu.e_flag} ) - { - // TODO: can't currently parse past extension headers - return false; - } + if ( ::gtpv1_message ) + BifEvent::generate_gtpv1_message(a, c, BuildGTPv1Hdr(pdu)); - if ( ${pdu.msg_type} != 0xff ) - { - // Only interested in decapsulating user plane data beyond here. + switch ( ${pdu.msg_type} ) { + case 16: + CreatePDP_Request(a, pdu); + return true; + case 17: + CreatePDP_Response(a, pdu); + return true; + case 18: + UpdatePDP_Request(a, pdu); + return true; + case 19: + UpdatePDP_Response(a, pdu); + return true; + case 20: + DeletePDP_Request(a, pdu); + return true; + case 21: + DeletePDP_Response(a, pdu); + return true; + case 255: + return process_g_pdu(pdu); + default: return false; - } + } + + return false; + %} + + function process_g_pdu(pdu: GTPv1_Header): bool + %{ + BroAnalyzer a = connection()->bro_analyzer(); + Connection *c = a->Conn(); + const EncapsulationStack* e = c->GetEncapsulation(); if ( ${pdu.packet}.length() < (int)sizeof(struct ip) ) { - violate("Truncated GTPv1", a, ${pdu.packet}); + violate("Truncated GTPv1", pdu); return false; } @@ -94,7 +717,7 @@ flow GTPv1_Flow(is_orig: bool) if ( ip->ip_v != 4 && ip->ip_v != 6 ) { - violate("non-IP packet in GTPv1", a, ${pdu.packet}); + violate("non-IP packet in GTPv1", pdu); return false; } @@ -113,10 +736,10 @@ flow GTPv1_Flow(is_orig: bool) } else if ( result < 0 ) - violate("Truncated GTPv1", a, ${pdu.packet}); + violate("Truncated GTPv1", pdu); else - violate("GTPv1 payload length", a, ${pdu.packet}); + violate("GTPv1 payload length", pdu); if ( result != 0 ) { @@ -125,37 +748,16 @@ flow GTPv1_Flow(is_orig: bool) } if ( ::gtpv1_g_pdu_packet ) - { - RecordVal* rv = new RecordVal(gtpv1_hdr_type); - - rv->Assign(0, new Val(${pdu.version}, TYPE_COUNT)); - rv->Assign(1, new Val(${pdu.pt_flag}, TYPE_BOOL)); - rv->Assign(2, new Val(${pdu.rsv}, TYPE_BOOL)); - rv->Assign(3, new Val(${pdu.e_flag}, TYPE_BOOL)); - rv->Assign(4, new Val(${pdu.s_flag}, TYPE_BOOL)); - rv->Assign(5, new Val(${pdu.pn_flag}, TYPE_BOOL)); - rv->Assign(6, new Val(${pdu.msg_type}, TYPE_COUNT)); - rv->Assign(7, new Val(ntohs(${pdu.length}), TYPE_COUNT)); - rv->Assign(8, new Val(ntohl(${pdu.teid}), TYPE_COUNT)); - - if ( ${pdu.has_opt} ) - { - rv->Assign(9, new Val(ntohs(${pdu.opt_hdr.seq}), TYPE_COUNT)); - rv->Assign(10, new Val(${pdu.opt_hdr.n_pdu}, TYPE_COUNT)); - rv->Assign(11, new Val(${pdu.opt_hdr.next_type}, TYPE_COUNT)); - } - - BifEvent::generate_gtpv1_g_pdu_packet(a, c, rv, + BifEvent::generate_gtpv1_g_pdu_packet(a, c, BuildGTPv1Hdr(pdu), inner->BuildPktHdrVal()); - } EncapsulatingConn ec(c, BifEnum::Tunnel::GTPv1); sessions->DoNextInnerPacket(network_time(), 0, inner, e, ec); - return (result == 0) ? true : false; + return true; %} - }; refine typeattr GTPv1_Header += &let { proc_gtpv1 = $context.flow.process_gtpv1(this); }; + diff --git a/src/gtpv1-protocol.pac b/src/gtpv1-protocol.pac index 5bf31a48ee..0a87665fd2 100644 --- a/src/gtpv1-protocol.pac +++ b/src/gtpv1-protocol.pac @@ -4,11 +4,27 @@ type GTPv1_Header = record { msg_type: uint8; length: uint16; teid: uint32; + opt: case has_opt of { true -> opt_hdr: GTPv1_Opt_Header; false -> no_opt: empty; - } &requires(has_opt); - packet: bytestring &restofdata; + }; + + ext: case e_flag of { + true -> ext_hdrs: GTPv1_Ext_Header[] &until($element.next_type == 0); + false -> no_ext: empty; + }; + + msg: case msg_type of { + 16 -> create_pdp_ctx_request: InformationElement[]; + 17 -> create_pdp_ctx_response: InformationElement[]; + 18 -> update_pdp_ctx_request: InformationElement[]; + 19 -> update_pdp_ctx_response: InformationElement[]; + 20 -> delete_pdp_ctx_request: InformationElement[]; + 21 -> delete_pdp_ctx_response: InformationElement[]; + 255 -> packet: bytestring &restofdata; + default -> unknown: bytestring &restofdata; + }; } &let { version: uint8 = (flags & 0xE0) >> 5; @@ -18,10 +34,463 @@ type GTPv1_Header = record { s_flag: bool = flags & 0x02; pn_flag: bool = flags & 0x01; has_opt: bool = flags & 0x07; -} &byteorder = littleendian; +} &byteorder = bigendian, &exportsourcedata; type GTPv1_Opt_Header = record { seq: uint16; n_pdu: uint8; next_type: uint8; -} +}; + +type GTPv1_Ext_Header = record { + length: uint8; + contents: bytestring &length=(length * 4 - 2); + next_type: uint8; +}; + +enum InfoElementType { + TYPE_CAUSE = 1, + TYPE_IMSI = 2, + TYPE_RAI = 3, + TYPE_TLLI = 4, + TYPE_P_TMSI = 5, + TYPE_REORDER_REQ = 8, + TYPE_AUTHN_TRIPLET = 9, + TYPE_MAP_CAUSE = 11, + TYPE_P_TMSI_SIG = 12, + TYPE_MS_VALID = 13, + TYPE_RECOVERY = 14, + TYPE_SELECTION_MODE = 15, + TYPE_TEID1 = 16, + TYPE_TEID_CONTROL_PLANE = 17, + TYPE_TEID2 = 18, + TYPE_TEARDOWN_IND = 19, + TYPE_NSAPI = 20, + TYPE_RANAP_CAUSE = 21, + TYPE_RAB_CTX = 22, + TYPE_RADIO_PRIORITY_SMS = 23, + TYPE_RADIO_PRIORITY = 24, + TYPE_PACKET_FLOW_ID = 25, + TYPE_CHARGING_CHARACTERISTICS = 26, + TYPE_TRACE_REFERENCE = 27, + TYPE_TRACE_TYPE = 28, + TYPE_MS_NOT_REACHABLE_REASON = 29, + TYPE_CHARGING_ID = 127, + TYPE_END_USER_ADDR = 128, + TYPE_MM_CTX = 129, + TYPE_PDP_CTX = 130, + TYPE_ACCESS_POINT_NAME = 131, + TYPE_PROTO_CONFIG_OPTIONS = 132, + TYPE_GSN_ADDR = 133, + TYPE_MSISDN = 134, + TYPE_QOS_PROFILE = 135, + TYPE_AUTHN_QUINTUPLET = 136, + TYPE_TRAFFIC_FLOW_TEMPLATE = 137, + TYPE_TARGET_ID = 138, + TYPE_UTRAN_TRANSPARENT_CONTAINER = 139, + TYPE_RAB_SETUP_INFO = 140, + TYPE_EXT_HEADER_TYPE_LIST = 141, + TYPE_TRIGGER_ID = 142, + TYPE_OMC_ID = 143, + TYPE_CHARGING_GATEWAY_ADDR = 251, + TYPE_PRIVATE_EXT = 255, +}; + +type InformationElement = record { + type: uint8; + + len: case is_tlv of { + true -> tlv_len: uint16; + false -> no_len: empty; + }; + + value: case type of { + TYPE_CAUSE -> cause: Cause; + TYPE_IMSI -> imsi: IMSI; + TYPE_RAI -> rai: RAI; + TYPE_TLLI -> tlli: TLLI; + TYPE_P_TMSI -> p_tmsi: P_TMSI; + TYPE_REORDER_REQ -> reorder_req: ReorderReq; + TYPE_AUTHN_TRIPLET -> authn_triplet: AuthN_Triplet; + TYPE_MAP_CAUSE -> map_cause: MAP_Cause; + TYPE_P_TMSI_SIG -> p_tmsi_sig: P_TMSI_Sig; + TYPE_MS_VALID -> ms_valid: MS_Valid; + TYPE_RECOVERY -> recovery: Recovery; + TYPE_SELECTION_MODE -> selection_mode: SelectionMode; + TYPE_TEID1 -> teid1: TEID1; + TYPE_TEID_CONTROL_PLANE -> teidcp: TEID_ControlPlane; + TYPE_TEID2 -> teid2: TEID2; + TYPE_TEARDOWN_IND -> teardown_ind: TeardownInd; + TYPE_NSAPI -> nsapi: NSAPI; + TYPE_RANAP_CAUSE -> ranap_cause: RANAP_Cause; + TYPE_RAB_CTX -> rab_ctx: RAB_Ctx; + TYPE_RADIO_PRIORITY_SMS -> radio_priority_sms: RadioPrioritySMS; + TYPE_RADIO_PRIORITY -> radio_priority: RadioPriority; + TYPE_PACKET_FLOW_ID -> packet_flow_id: PacketFlowID; + TYPE_CHARGING_CHARACTERISTICS -> charging_characteristics: ChargingCharacteristics; + TYPE_TRACE_REFERENCE -> trace_reference: TraceReference; + TYPE_TRACE_TYPE -> trace_type: TraceType; + TYPE_MS_NOT_REACHABLE_REASON -> ms_not_reachable_reason: MS_Not_Reachable_Reason; + TYPE_CHARGING_ID -> charging_id: ChargingID; + TYPE_END_USER_ADDR -> end_user_addr: EndUserAddr(length); + TYPE_MM_CTX -> mm_ctx: MM_Ctx(length); + TYPE_PDP_CTX -> pdp_ctx: PDP_Ctx(length); + TYPE_ACCESS_POINT_NAME -> ap_name: AP_Name(length); + TYPE_PROTO_CONFIG_OPTIONS -> proto_config_opts: ProtoConfigOpts(length); + TYPE_GSN_ADDR -> gsn_addr: GSN_Addr(length); + TYPE_MSISDN -> msisdn: MSISDN(length); + TYPE_QOS_PROFILE -> qos_profile: QoS_Profile(length); + TYPE_AUTHN_QUINTUPLET -> authn_quintuplet: AuthN_Quintuplet(length); + TYPE_TRAFFIC_FLOW_TEMPLATE -> traffic_flow_template: TrafficFlowTemplate(length); + TYPE_TARGET_ID -> target_id: TargetID(length); + TYPE_UTRAN_TRANSPARENT_CONTAINER -> utran_transparent_container: UTRAN_TransparentContainer(length); + TYPE_RAB_SETUP_INFO -> rab_setup_info: RAB_SetupInfo(length); + TYPE_EXT_HEADER_TYPE_LIST -> ext_hdr_type_list: ExtHdrTypeList(length); + TYPE_TRIGGER_ID -> trigger_id: TriggerID(length); + TYPE_OMC_ID -> omc_id: OMC_ID(length); + TYPE_CHARGING_GATEWAY_ADDR -> charging_gateway_addr: ChargingGatewayAddr(length); + TYPE_PRIVATE_EXT -> private_ext: PrivateExt(length); + default -> unknown: bytestring &length=length; + } &requires(length); + +} &let { + is_tlv: bool = (type & 0x80); + length: uint16 = is_tlv ? tlv_len : Get_IE_Len(type); +}; + +type Cause = record { + value: uint8; +}; + +function decode_imsi(v: uint8[8]): uint64 + %{ + uint64 rval = 0; + uint8 digits[16]; + for ( size_t i = 0; i < v->size(); ++i ) + { + digits[2 * i + 1] = ((*v)[i] & 0xf0) >> 4; + digits[2 * i] = (*v)[i] & 0x0f; + } + int power = 0; + for ( int i = 15; i >= 0; --i ) + { + if ( digits[i] == 0x0f ) continue; + rval += digits[i] * pow(10, power); + ++power; + } + return rval; + %} + +type IMSI = record { + tbcd_encoded_value: uint8[8]; +} &let { + value: uint64 = decode_imsi(tbcd_encoded_value); +}; + +type RAI = record { + mcc2_mcc1: uint8; + mnc3_mcc3: uint8; + mnc2_mnc1: uint8; + lac: uint16; + rac: uint8; +} &let { + mcc1: uint8 = (mcc2_mcc1 & 0x0f); + mcc2: uint8 = ((mcc2_mcc1 & 0xf0)>>4); + mcc3: uint8 = (mnc3_mcc3 & 0x0f); + mcc: uint16 = mcc1 * 100 + mcc2 * 10 + mcc3; + mnc1: uint8 = (mnc2_mnc1 & 0x0f); + mnc2: uint8 = ((mnc2_mnc1 & 0xf0)>>4); + mnc3: uint8 = (mnc3_mcc3 & 0xf0)>>4; + mnc: uint16 = (mnc3 & 0x0f) ? mnc1 * 10 + mnc2 : mnc1 * 100 + mnc2 * 10 + mnc3; +}; + +type TLLI = record { + value: uint32; +}; + +type P_TMSI = record { + value: uint32; +}; + +type ReorderReq = record { + value: uint8; +} &let { + req: bool = value & 0x01; +}; + +type AuthN_Triplet = record { + rand: bytestring &length=16; + sres: uint32; + kc: uint64; +}; + +type MAP_Cause = record { + value: uint8; +}; + +type P_TMSI_Sig = record { + value: bytestring &length=3; +}; + +type MS_Valid = record { + value: uint8; +}; + +type Recovery = record { + restart_counter: uint8; +}; + +type SelectionMode = record { + value: uint8; +} &let { + mode: uint8 = value & 0x01; +}; + +type TEID1 = record { + value: uint32; +}; + +type TEID_ControlPlane = record { + value: uint32; +}; + +type TEID2 = record { + spare_nsapi: uint8; + teid2: uint32; +}; + +type TeardownInd = record { + value: uint8; +} &let { + ind: bool = value & 0x01; +}; + +type NSAPI = record { + xxxx_nsapi: uint8; +} &let { + nsapi: uint8 = xxxx_nsapi & 0x0f; +}; + +type RANAP_Cause = record { + value: uint8; +}; + +type RAB_Ctx = record { + spare_nsapi: uint8; + dl_gtpu_seq_num: uint16; + ul_gtpu_seq_num: uint16; + dl_pdcp_seq_num: uint16; + ul_pdcp_seq_num: uint16; +}; + +type RadioPrioritySMS = record { + value: uint8; +}; + +type RadioPriority = record { + nsapi_radio_priority: uint8; +}; + +type PacketFlowID = record { + rsv_nsapi: uint8; + packet_flow_id: uint8; +}; + +type ChargingCharacteristics = record { + value: uint16; +}; + +type TraceReference = record { + value: uint16; +}; + +type TraceType = record { + value: uint16; +}; + +type MS_Not_Reachable_Reason = record { + value: uint8; +}; + +type ChargingID = record { + value: uint32; +}; + +type EndUserAddr(n: uint16) = record { + spare_pdp_type_org: uint8; + pdp_type_num: uint8; + pdp_addr: bytestring &length=(n-2); +} &let { + pdp_type_org: uint8 = spare_pdp_type_org & 0x0f; +}; + +type MM_Ctx(n: uint16) = record { + spare_cksn_ksi: uint8; + security_params: uint8; + + keys: case gsm_keys of { + true -> kc: uint64; + false -> ck_ik: bytestring &length=32; + }; + + vector_len: case have_triplets of { + true -> no_quint_len: empty; + false -> quint_len: uint16; + }; + + vectors: case have_triplets of { + true -> triplets: AuthN_Triplet[num_vectors]; + false -> quintuplets: AuthN_Quintuplet(quint_len)[num_vectors]; + } &requires(num_vectors); + + drx_param: uint16; + ms_net_capability_len: uint8; + ms_net_capability: bytestring &length=ms_net_capability_len; + container_len: uint16; + container: bytestring &length=container_len; + +} &let { + security_mode: uint8 = security_params >> 6; + gsm_keys: bool = security_mode & 0x01; + have_triplets: bool = (security_mode == 1); + num_vectors: uint8 = (security_params & 0x38) >> 3; +}; + +type PDP_Ctx(n: uint16) = record { + rsv_nsapi: uint8; + xxxx_sapi: uint8; + qos_sub_len: uint8; + qos_sub: QoS_Profile(qos_sub_len); + qos_req_len: uint8; + qos_req: QoS_Profile(qos_req_len); + qos_neg_len: uint8; + qos_neg: QoS_Profile(qos_neg_len); + snd: uint16; + snu: uint16; + send_npdu_num: uint8; + recv_npdu_num: uint8; + ul_teid_cp: TEID_ControlPlane; + ul_teid_data1: TEID1; + pdp_ctx_id: uint8; + spare_pdp_type_org: uint8; + pdp_type_num: uint8; + pdp_addr_len: uint8; + pdp_addr: bytestring &length=pdp_addr_len; + ggsn_addr_control_plane_len: uint8; + ggsn_addr_control_plane: bytestring &length=ggsn_addr_control_plane_len; + ggsn_addr_user_traffic_len: uint8; + ggsn_addr_user_traffic: bytestring &length=ggsn_addr_user_traffic_len; + apn_len: uint8; + apn: AP_Name(apn_len); + spare_transaction_id: uint8; + transaction_id: uint8; +}; + +type AP_Name(n: uint16) = record { + value: bytestring &length=n; +}; + +type ProtoConfigOpts(n: uint16) = record { + value: bytestring &length=n; +}; + +type GSN_Addr(n: uint16) = record { + value: bytestring &length=n; +}; + +type MSISDN(n: uint16) = record { + value: bytestring &length=n; +}; + +type QoS_Profile(n: uint16) = record { + alloc_retention_priority: uint8; + data: bytestring &length=n-1; +}; + +type AuthN_Quintuplet(n: uint16) = record { + rand: bytestring &length=16; + xres_len: uint8; + xres: bytestring &length=xres_len; + ck: bytestring &length=16; + ik: bytestring &length=16; + autn_len: uint8; + autn: bytestring &length=autn_len; +}; + +type TrafficFlowTemplate(n: uint16) = record { + value: bytestring &length=n; +}; + +type TargetID(n: uint16) = record { + value: bytestring &length=n; +}; + +type UTRAN_TransparentContainer(n: uint16) = record { + value: bytestring &length=n; +}; + +type RAB_SetupInfo(n: uint16) = record { + xxxx_nsapi: uint8; + + have_teid: case n of { + 1 -> no_teid: empty; + default -> teid: TEID1; + }; + + have_addr: case n of { + 1 -> no_addr: empty; + default -> rnc_addr: bytestring &length=n-5; + }; +}; + +type ExtHdrTypeList(n: uint16) = record { + value: uint8[n]; +}; + +type TriggerID(n: uint16) = record { + value: bytestring &length=n; +}; + +type OMC_ID(n: uint16) = record { + value: bytestring &length=n; +}; + +type ChargingGatewayAddr(n: uint16) = record { + value: bytestring &length=n; +}; + +type PrivateExt(n: uint16) = record { + id: uint16; + value: bytestring &length=n-2; +}; + +function Get_IE_Len(t: uint8): uint16 = + case t of { + TYPE_CAUSE -> 1; + TYPE_IMSI -> 8; + TYPE_RAI -> 6; + TYPE_TLLI -> 4; + TYPE_P_TMSI -> 4; + TYPE_REORDER_REQ -> 1; + TYPE_AUTHN_TRIPLET -> 28; + TYPE_MAP_CAUSE -> 1; + TYPE_P_TMSI_SIG -> 3; + TYPE_MS_VALID -> 1; + TYPE_RECOVERY -> 1; + TYPE_SELECTION_MODE -> 1; + TYPE_TEID1 -> 4; + TYPE_TEID_CONTROL_PLANE -> 4; + TYPE_TEID2 -> 5; + TYPE_TEARDOWN_IND -> 1; + TYPE_NSAPI -> 1; + TYPE_RANAP_CAUSE -> 1; + TYPE_RAB_CTX -> 9; + TYPE_RADIO_PRIORITY_SMS -> 1; + TYPE_RADIO_PRIORITY -> 1; + TYPE_PACKET_FLOW_ID -> 2; + TYPE_CHARGING_CHARACTERISTICS -> 2; + TYPE_TRACE_REFERENCE -> 2; + TYPE_TRACE_TYPE -> 2; + TYPE_MS_NOT_REACHABLE_REASON -> 1; + TYPE_CHARGING_ID -> 4; + }; diff --git a/src/types.bif b/src/types.bif index 888310419c..27009d2e1c 100644 --- a/src/types.bif +++ b/src/types.bif @@ -211,3 +211,17 @@ enum Mode %{ %} module GLOBAL; + +type gtpv1_hdr: record; +type gtp_create_pdp_ctx_request_elements: record; +type gtp_create_pdp_ctx_response_elements: record; +type gtp_update_pdp_ctx_request_elements: record; +type gtp_update_pdp_ctx_response_elements: record; +type gtp_delete_pdp_ctx_request_elements: record; +type gtp_delete_pdp_ctx_response_elements: record; + +type gtp_end_user_addr: record; +type gtp_rai: record; +type gtp_qos_profile: record; +type gtp_private_extension: record; +type gtp_gsn_addr: record; diff --git a/testing/btest/Baseline/core.tunnels.gtp.ext_header/out b/testing/btest/Baseline/core.tunnels.gtp.ext_header/out new file mode 100644 index 0000000000..e76a7f35de --- /dev/null +++ b/testing/btest/Baseline/core.tunnels.gtp.ext_header/out @@ -0,0 +1,2 @@ +gtpv1_message, [orig_h=10.155.148.149, orig_p=9000/udp, resp_h=10.155.148.157, resp_p=2152/udp] +[version=1, pt_flag=T, rsv=F, e_flag=T, s_flag=T, pn_flag=F, msg_type=255, length=1508, teid=1050199, seq=5, n_pdu=0, next_type=192] diff --git a/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out b/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out index a299c4d592..49bb5f7399 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out +++ b/testing/btest/Baseline/core.tunnels.gtp.non_recursive/out @@ -1 +1 @@ -protocol_violation, [orig_h=74.125.216.149, orig_p=2152/udp, resp_h=10.131.138.69, resp_p=2152/udp], GTP-in-GTP [n\xd9'|\x00\x00\x01\xb6[\xf6\xdc0\xb7d\xe5\xe6\xa76\x91\xfbk\x0e\x02\xc8A\x05\xa8\xe6\xf3Gi\x80(]\xcew\x84\xae}\xd2...] +protocol_violation, [orig_h=74.125.216.149, orig_p=2152/udp, resp_h=10.131.138.69, resp_p=2152/udp], GTP-in-GTP [\x80\xe1Bc.\xe20\xebn\xd9'|\x00\x00\x01\xb6[\xf6\xdc0\xb7d\xe5\xe6\xa76\x91\xfbk\x0e\x02\xc8A\x05\xa8\xe6\xf3Gi\x80...] diff --git a/testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out b/testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out new file mode 100644 index 0000000000..fcdfe94824 --- /dev/null +++ b/testing/btest/Baseline/core.tunnels.gtp.pdp_ctx_messages/out @@ -0,0 +1,24 @@ +gtpv1_message, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=137, teid=0, seq=4875, n_pdu=0, next_type=0] +gtp create request, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=137, teid=0, seq=4875, n_pdu=0, next_type=0] +[imsi=460004100000101, rai=[mcc=460, mnc=6, lac=65534, rac=255], recovery=176, select_mode=1, data1=854600697, cp=854600697, nsapi=5, linked_nsapi=, charge_character=, trace_ref=, trace_type=, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=, pdp_other_addr=], ap_name=^Feetest, opts=\x80\x80!^V^A^A\0^V^C^F\0\0\0\0\x81^F\0\0\0\0\x83^F\0\0\0\0, signal_addr=[ip=192.169.100.1, other=], user_addr=[ip=192.169.100.1, other=], msisdn=\x91hQ"^A\0^A\xf1, qos_prof=[priority=2, data=\x1bB\x1fs\x8c@@tK@@], tft=, trigger_id=, omc_id=, ext=[id=10923, value=^B^A^C]] +gtpv1_message, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=101, teid=854600697, seq=4875, n_pdu=0, next_type=0] +gtp create response, [orig_h=192.169.100.1, orig_p=34273/udp, resp_h=10.100.200.33, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=101, teid=854600697, seq=4875, n_pdu=0, next_type=0] +[cause=128, reorder_req=F, recovery=24, data1=268435589, cp=268435584, charging_id=103000009, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=192.168.252.130, pdp_other_addr=], opts=\x80\x80!^P^D^A\0^P\x81^F\0\0\0\0\x83^F\0\0\0\0\x80!^J^C^A\0^J^C^F\xc0\xa8\xfc\x82, cp_addr=[ip=10.100.200.34, other=], user_addr=[ip=10.100.200.49, other=], qos_prof=[priority=2, data=\x1bB\x1fs\x8c@@tK@@], charge_gateway=, ext=] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=1, length=4, teid=0, seq=3072, n_pdu=0, next_type=0] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=2, length=6, teid=0, seq=3072, n_pdu=0, next_type=0] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=104, teid=0, seq=3073, n_pdu=0, next_type=0] +gtp create request, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=16, length=104, teid=0, seq=3073, n_pdu=0, next_type=0] +[imsi=240010123456789, rai=, recovery=3, select_mode=1, data1=1, cp=1, nsapi=0, linked_nsapi=, charge_character=2048, trace_ref=, trace_type=, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=, pdp_other_addr=], ap_name=^Hinternet, opts=\x80\xc0#^Q^A^A\0^Q^Cmig^Hhemmelig, signal_addr=[ip=127.0.0.2, other=], user_addr=[ip=127.0.0.2, other=], msisdn=\x91d^G^R2T\xf6, qos_prof=[priority=0, data=^K\x92\x1f], tft=, trigger_id=, omc_id=, ext=] +gtpv1_message, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=78, teid=1, seq=3073, n_pdu=0, next_type=0] +gtp create response, [orig_h=127.0.0.2, orig_p=2123/udp, resp_h=127.0.0.1, resp_p=2123/udp] +[version=1, pt_flag=T, rsv=F, e_flag=F, s_flag=T, pn_flag=F, msg_type=17, length=78, teid=1, seq=3073, n_pdu=0, next_type=0] +[cause=128, reorder_req=F, recovery=1, data1=1, cp=1, charging_id=1, end_user_addr=[pdp_type_org=1, pdp_type_num=33, pdp_ip=192.168.0.2, pdp_other_addr=], opts=\x80\x80!^P^B\0\0^P\x81^F\0\0\0\0\x83^F\0\0\0\0, cp_addr=[ip=127.0.0.1, other=], user_addr=[ip=127.0.0.1, other=], qos_prof=[priority=0, data=^K\x92\x1f], charge_gateway=, ext=] diff --git a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log index 221fa16f4f..fcd110f8ab 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log +++ b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/dpd.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path dpd -#open 2012-10-19-17-38-54 +#open 2013-01-25-21-49-19 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto analyzer failure_reason #types time string addr port addr port enum string string -1333458853.075889 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 udp GTPV1 Truncated GTPv1 [E\x00\x05\xc8G\xea@\x00\x80\x06\xb6\x83\x0a\x83w&\xd9\x14\x9c\x04\xd9\xc2\x00P\xddh\xb4\x8f41eVP\x10\x10\xe0u\xcf\x00\x00...] -#close 2012-10-19-17-38-54 +1333458853.075889 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 udp GTPV1 Truncated GTPv1 [0\xff\x00\xac\x98\x13\x01LE\x00\x05\xc8G\xea@\x00\x80\x06\xb6\x83\x0a\x83w&\xd9\x14\x9c\x04\xd9\xc2\x00P\xddh\xb4\x8f41eV...] +#close 2013-01-25-21-49-19 diff --git a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log index 659090a581..070a754702 100644 --- a/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log +++ b/testing/btest/Baseline/core.tunnels.gtp.unknown_or_too_short/tunnel.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path tunnel -#open 2012-10-19-17-38-54 +#open 2013-01-25-21-49-19 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action #types time string addr port addr port enum enum 1333458853.034734 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 Tunnel::GTPv1 Tunnel::DISCOVER 1333458853.108391 UWkUyAuUGXf 173.86.159.28 2152 213.72.147.186 2152 Tunnel::GTPv1 Tunnel::CLOSE -#close 2012-10-19-17-38-54 +#close 2013-01-25-21-49-19 diff --git a/testing/btest/Traces/tunnels/gtp/gtp_control_prime.pcap b/testing/btest/Traces/tunnels/gtp/gtp_control_prime.pcap new file mode 100644 index 0000000000000000000000000000000000000000..ebed358a265429d5d7cecb975a3a8c2a8b53e3a7 GIT binary patch literal 1292 zcmb`GO=uHQ5P;|HhV3RzHoG;gTIk8j0qtI?9Hu> z7NH9E*pmw4&p|LET2H}JkAfiHMLhJDQWXRx&b++XT~C4z%)Z@a-uJ$lY~C+b?n>Z; zX4LBd3?FVx)a)-86d1yLN_VJm0-z5d4sqk?>ni{fCR1(IhRCyCi#2dT+(HyPtH2Sw zr*sFW^b1Or2t?wPj{8wd0)?uHqA}tu=M-4LCrWp4&Ya*RJvLLfMl2oDR2{^oq~pM1 zW2xEinjB_2zGMbnBG9RV1JY`22K;I!UrH78sZtr*oobK47#MOcGpVLixm@aEX7U>N z=nS!3hnCytTH36{w^{7Ebv{a_QeHtG@WkbJaMPkNo$_>?EUj1ujAJPyQ|74H^|~U~ z;yX}QW7UTsl_BCdFa(__d+L#3Fnur)u`wGb-Ff0;M~+52EArJ7%dXataNH zh9SH5(bQo=N0&N=7@S|h4mBSQ4@ZYW|2Q*}`-yVXXhrD`E_X@DT_!OS;`>yuxC|tV05; zJ>$avFusdJoGp*yS0%qY{VA=&=M|#TNssX))gNdLhP1G5v~7sAcWm6WxpPa`Rx`RS z*4?vx$IjlqUH!ZF?A^D2;DB}T(BRNv+V^xJ08fxX=?-pqM;M;;bUe41M8g7Bdkw2l ccyteUu<;LQ$jDL-{P;@Lru`^;^WW@$0e@fV*Z=?k literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/tunnels/gtp/gtp_create_pdp_ctx.pcap b/testing/btest/Traces/tunnels/gtp/gtp_create_pdp_ctx.pcap new file mode 100644 index 0000000000000000000000000000000000000000..78283156d57ce5d254f9ad62c400e161c6b67611 GIT binary patch literal 713 zcmaiyOK8+k6o&tso5V}TjtTRy_-fNxR9gymm~~C7F6w}cAn4Aa%z&FINaM;y(@ayR zK5^sFg`gm|B8uoz(1@TGceN8wQ1#&a_k1{WIm&&}iJobNyPtiF13M}dm1 zD~bR{4|m7sipS5X@abL(`#ul(SE1f4yeJ491>BWSz=-wzK)AGG7*56)H8F{@ISlOS zVrxV1N`!dN^ftNeZ*~Z_}$)w>sta@A5o5OPjM?mlbE? zSdU}w+c8PhoL2FSK1o3f?wvdnb6tyx9;O~SIAk512=7;zQF&_z>!OH-*H7~%=H(Bm z1Qo2nA%CpJi7eWTl8<-%GV>dQKP(y2ng^w2*RjPki+?}`P-Uq<%N)dwvnTqi1+2PVpe-<`1)KnSF&Q^N1!5Iv$%RR(S zCHmUS@L${#^W!NUw}?=RPJicKuVV(T%ahBOat_3#sNsTY$QeL91Et=qubfU@ yF1+yvyon>LYcmbQ7rd)6t|JW)eVuY7Uyg>EOJh9I(KSA(v)D)f|7{%Z{_q>RB(gRD literal 0 HcmV?d00001 diff --git a/testing/btest/Traces/tunnels/gtp/gtp_ext_header.pcap b/testing/btest/Traces/tunnels/gtp/gtp_ext_header.pcap new file mode 100644 index 0000000000000000000000000000000000000000..453ebf934a9d06a74140e1686c616ec0e3c09d59 GIT binary patch literal 1648 zcmca|c+)~A1{MYw`2U}Qff2}=J^Py9%2sBESF8+REX2UbWXSMhyNAF-4Gsp^2-Z6a zJqir}1kI0d&7Lw9i03M6aAdH4VPG)(&-#QxfGwPXmEiy*Ckuls1M8hTzZ@9s*)Bcj zn!W85*X&)Ei`2zSrwKh{R$>g9)gZv2x2_9lD@G{G_B#0BO}`VE_OC literal 0 HcmV?d00001 diff --git a/testing/btest/core/tunnels/gtp/ext_header.test b/testing/btest/core/tunnels/gtp/ext_header.test new file mode 100644 index 0000000000..6316acb184 --- /dev/null +++ b/testing/btest/core/tunnels/gtp/ext_header.test @@ -0,0 +1,8 @@ +# @TEST-EXEC: bro -r $TRACES/tunnels/gtp/gtp_ext_header.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +event gtpv1_message(c: connection, hdr: gtpv1_hdr) + { + print "gtpv1_message", c$id; + print hdr; + } diff --git a/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test b/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test new file mode 100644 index 0000000000..7405c8d019 --- /dev/null +++ b/testing/btest/core/tunnels/gtp/pdp_ctx_messages.test @@ -0,0 +1,56 @@ +# @TEST-EXEC: bro -r $TRACES/tunnels/gtp/gtp_control_prime.pcap -r $TRACES/tunnels/gtp/gtp_create_pdp_ctx.pcap %INPUT >out +# @TEST-EXEC: btest-diff out + +event gtpv1_message(c: connection, hdr: gtpv1_hdr) + { + print "gtpv1_message", c$id; + print hdr; + } + +event gtpv1_create_pdp_ctx_request(c: connection, hdr: gtpv1_hdr, + elements: gtp_create_pdp_ctx_request_elements) + { + print "gtp create request", c$id; + print hdr; + print elements; + } + +event gtpv1_create_pdp_ctx_response(c: connection, hdr: gtpv1_hdr, + elements: gtp_create_pdp_ctx_response_elements) + { + print "gtp create response", c$id; + print hdr; + print elements; + } + +event gtpv1_update_pdp_ctx_request(c: connection, hdr: gtpv1_hdr, + elements: gtp_update_pdp_ctx_request_elements) + { + print "gtp update request", c$id; + print hdr; + print elements; + } + +event gtpv1_update_pdp_ctx_response(c: connection, hdr: gtpv1_hdr, + elements: gtp_update_pdp_ctx_response_elements) + { + print "gtp update response", c$id; + print hdr; + print elements; + } + +event gtpv1_delete_pdp_ctx_request(c: connection, hdr: gtpv1_hdr, + elements: gtp_delete_pdp_ctx_request_elements) + { + print "gtp delete request", c$id; + print hdr; + print elements; + } + +event gtpv1_delete_pdp_ctx_response(c: connection, hdr: gtpv1_hdr, + elements: gtp_delete_pdp_ctx_response_elements) + { + print "gtp delete response", c$id; + print hdr; + print elements; + } From 9f8ba408ba0112d8d053fdade3a21f0bf1cfa106 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 11 Feb 2013 14:36:14 -0500 Subject: [PATCH 31/49] Updates for the notices framework. - Moved the Notice::notice event and Notice::policy table to both be hooks. - Renamed the old Notice::policy to Notice::policy_table and documented it as deprecated. --- doc/scripts/DocSourcesList.cmake | 1 + scripts/base/frameworks/notice/__load__.bro | 4 +- .../frameworks/notice/actions/add-geodata.bro | 15 ++- .../base/frameworks/notice/actions/drop.bro | 21 ++-- .../frameworks/notice/actions/email_admin.bro | 2 +- .../base/frameworks/notice/actions/page.bro | 2 +- .../frameworks/notice/actions/pp-alarms.bro | 2 +- scripts/base/frameworks/notice/cluster.bro | 28 +++-- .../notice/extend-email/hostnames.bro | 2 +- scripts/base/frameworks/notice/main.bro | 112 +++++++----------- .../base/frameworks/notice/non-cluster.bro | 14 +++ .../canonified_loaded_scripts.log | 5 +- .../manager-1.notice.log | 10 +- .../manager-1.notice.log | 10 +- .../manager-1.notice.log | 10 +- .../notice.log | 10 +- .../notice.log | 10 +- .../base/frameworks/metrics/notice.bro | 24 ---- .../base/frameworks/notice/mail-alarms.bro | 6 +- 19 files changed, 129 insertions(+), 159 deletions(-) create mode 100644 scripts/base/frameworks/notice/non-cluster.bro delete mode 100644 testing/btest/scripts/base/frameworks/metrics/notice.bro diff --git a/doc/scripts/DocSourcesList.cmake b/doc/scripts/DocSourcesList.cmake index 6ed14ca943..117430223e 100644 --- a/doc/scripts/DocSourcesList.cmake +++ b/doc/scripts/DocSourcesList.cmake @@ -57,6 +57,7 @@ rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro) rest_target(${psd} base/frameworks/notice/cluster.bro) rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro) rest_target(${psd} base/frameworks/notice/main.bro) +rest_target(${psd} base/frameworks/notice/non-cluster.bro) rest_target(${psd} base/frameworks/notice/weird.bro) rest_target(${psd} base/frameworks/packet-filter/main.bro) rest_target(${psd} base/frameworks/packet-filter/netstats.bro) diff --git a/scripts/base/frameworks/notice/__load__.bro b/scripts/base/frameworks/notice/__load__.bro index 4548e98dc2..bcf3cd6898 100644 --- a/scripts/base/frameworks/notice/__load__.bro +++ b/scripts/base/frameworks/notice/__load__.bro @@ -17,7 +17,9 @@ @if ( Cluster::is_enabled() ) @load ./cluster +@else +@load ./non-cluster @endif # Load here so that it can check whether clustering is enabled. -@load ./actions/pp-alarms +@load ./actions/pp-alarms \ No newline at end of file diff --git a/scripts/base/frameworks/notice/actions/add-geodata.bro b/scripts/base/frameworks/notice/actions/add-geodata.bro index 9f6909595c..c762be3b86 100644 --- a/scripts/base/frameworks/notice/actions/add-geodata.bro +++ b/scripts/base/frameworks/notice/actions/add-geodata.bro @@ -27,18 +27,17 @@ export { ## Notice types which should have the "remote" location looked up. ## If GeoIP support is not built in, this does nothing. const lookup_location_types: set[Notice::Type] = {} &redef; - - ## Add a helper to the notice policy for looking up GeoIP data. - redef Notice::policy += { - [$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); }, - $action = ACTION_ADD_GEODATA, - $priority = 10], - }; } +hook policy(n: Notice::Info) &priority=10 + { + if ( n$note in Notice::lookup_location_types ) + add n$actions[ACTION_ADD_GEODATA]; + } + # This is handled at a high priority in case other notice handlers # want to use the data. -event notice(n: Notice::Info) &priority=10 +hook notice(n: Notice::Info) &priority=10 { if ( ACTION_ADD_GEODATA in n$actions && |Site::local_nets| > 0 && diff --git a/scripts/base/frameworks/notice/actions/drop.bro b/scripts/base/frameworks/notice/actions/drop.bro index 0116dd4ed4..1befd8644f 100644 --- a/scripts/base/frameworks/notice/actions/drop.bro +++ b/scripts/base/frameworks/notice/actions/drop.bro @@ -17,20 +17,13 @@ export { }; } -# This is a little awkward because we want to inject drop along with the -# synchronous functions. -event bro_init() +hook notice(n: Notice::Info) { - local drop_func = function(n: Notice::Info) + if ( ACTION_DROP in n$actions ) { - if ( ACTION_DROP in n$actions ) - { - #local drop = React::drop_address(n$src, ""); - #local addl = drop?$sub ? fmt(" %s", drop$sub) : ""; - #n$dropped = drop$note != Drop::AddressDropIgnored; - #n$msg += fmt(" [%s%s]", drop$note, addl); - } - }; - - add Notice::sync_functions[drop_func]; + #local drop = React::drop_address(n$src, ""); + #local addl = drop?$sub ? fmt(" %s", drop$sub) : ""; + #n$dropped = drop$note != Drop::AddressDropIgnored; + #n$msg += fmt(" [%s%s]", drop$note, addl); + } } diff --git a/scripts/base/frameworks/notice/actions/email_admin.bro b/scripts/base/frameworks/notice/actions/email_admin.bro index 7484a1c606..fb82f2b960 100644 --- a/scripts/base/frameworks/notice/actions/email_admin.bro +++ b/scripts/base/frameworks/notice/actions/email_admin.bro @@ -18,7 +18,7 @@ export { }; } -event notice(n: Notice::Info) &priority=-5 +hook notice(n: Notice::Info) &priority=-5 { if ( |Site::local_admins| > 0 && ACTION_EMAIL_ADMIN in n$actions ) diff --git a/scripts/base/frameworks/notice/actions/page.bro b/scripts/base/frameworks/notice/actions/page.bro index 16a3463126..e29b2bf0ee 100644 --- a/scripts/base/frameworks/notice/actions/page.bro +++ b/scripts/base/frameworks/notice/actions/page.bro @@ -15,7 +15,7 @@ export { const mail_page_dest = "" &redef; } -event notice(n: Notice::Info) &priority=-5 +hook notice(n: Notice::Info) &priority=-5 { if ( ACTION_PAGE in n$actions ) email_notice_to(n, mail_page_dest, F); diff --git a/scripts/base/frameworks/notice/actions/pp-alarms.bro b/scripts/base/frameworks/notice/actions/pp-alarms.bro index 0f19d3da8f..52312c2624 100644 --- a/scripts/base/frameworks/notice/actions/pp-alarms.bro +++ b/scripts/base/frameworks/notice/actions/pp-alarms.bro @@ -105,7 +105,7 @@ event bro_init() $postprocessor=pp_postprocessor]); } -event notice(n: Notice::Info) &priority=-5 +hook notice(n: Notice::Info) &priority=-5 { if ( ! want_pp() ) return; diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro index e812c3fdca..02f11533fa 100644 --- a/scripts/base/frameworks/notice/cluster.bro +++ b/scripts/base/frameworks/notice/cluster.bro @@ -21,22 +21,11 @@ redef Cluster::manager2worker_events += /Notice::begin_suppression/; redef Cluster::worker2manager_events += /Notice::cluster_notice/; @if ( Cluster::local_node_type() != Cluster::MANAGER ) - -# The notice policy is completely handled by the manager and shouldn't be -# done by workers or proxies to save time for packet processing. -redef Notice::policy = table(); - event Notice::begin_suppression(n: Notice::Info) { suppressing[n$note, n$identifier] = n; } -event Notice::notice(n: Notice::Info) - { - # Send the locally generated notice on to the manager. - event Notice::cluster_notice(n); - } - event bro_init() &priority=-3 { # Workers and proxies need to disable the notice streams because notice @@ -54,3 +43,20 @@ event Notice::cluster_notice(n: Notice::Info) NOTICE(n); } @endif + +module GLOBAL; + +## This is the entry point in the global namespace for notice framework. +function NOTICE(n: Notice::Info) + { + # Suppress this notice if necessary. + if ( Notice::is_being_suppressed(n) ) + return; + +@if ( Cluster::local_node_type() == Cluster::MANAGER ) + Notice::internal_NOTICE(n); +@else + # For non-managers, send the notice on to the manager. + event Notice::cluster_notice(n); +@endif + } diff --git a/scripts/base/frameworks/notice/extend-email/hostnames.bro b/scripts/base/frameworks/notice/extend-email/hostnames.bro index 2ec6dbb23f..18442f453d 100644 --- a/scripts/base/frameworks/notice/extend-email/hostnames.bro +++ b/scripts/base/frameworks/notice/extend-email/hostnames.bro @@ -13,7 +13,7 @@ module Notice; # reference to the original notice) global tmp_notice_storage: table[string] of Notice::Info &create_expire=max_email_delay+10secs; -event Notice::notice(n: Notice::Info) &priority=10 +hook notice(n: Notice::Info) &priority=10 { if ( ! n?$src && ! n?$dst ) return; diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index e9b29e7392..5d3cc48091 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -102,10 +102,6 @@ export { ## The actions which have been applied to this notice. actions: set[Notice::Action] &log &optional; - ## These are policy items that returned T and applied their action - ## to the notice. - policy_items: set[count] &log &optional; - ## By adding chunks of text into this element, other scripts can ## expand on notices that are being emailed. The normal way to add text ## is to extend the vector by handling the :bro:id:`Notice::notice` @@ -185,32 +181,15 @@ export { }; ## Defines a notice policy that is extensible on a per-site basis. - ## All notice processing is done through this variable. - const policy: set[PolicyItem] = { - [$pred(n: Notice::Info) = { return (n$note in Notice::ignored_types); }, - $halt=T, $priority = 9], - [$pred(n: Notice::Info) = { return (n$note in Notice::not_suppressed_types); }, - $action = ACTION_NO_SUPPRESS, - $priority = 9], - [$pred(n: Notice::Info) = { return (n$note in Notice::alarmed_types); }, - $action = ACTION_ALARM, - $priority = 8], - [$pred(n: Notice::Info) = { return (n$note in Notice::emailed_types); }, - $action = ACTION_EMAIL, - $priority = 8], - [$pred(n: Notice::Info) = { - if (n$note in Notice::type_suppression_intervals) - { - n$suppress_for=Notice::type_suppression_intervals[n$note]; - return T; - } - return F; - }, - $action = ACTION_NONE, - $priority = 8], - [$action = ACTION_LOG, - $priority = 0], - } &redef; + ## All notice processing is done through this variable. This variable + ## is the former 'policy' variable, and + ## this variable is deprecated and will be removed in a future version. + ## All notice policy decisions are going to be done through the + ## 'policy' hook now. + const policy_table: set[PolicyItem] = {} &redef; + + ## The hook to modify notice handling. + global policy: hook(n: Notice::Info); ## Local system sendmail program. const sendmail = "/usr/sbin/sendmail" &redef; @@ -240,25 +219,11 @@ export { ## This is the event that is called as the entry point to the ## notice framework by the global :bro:id:`NOTICE` function. By the time ## this event is generated, default values have already been filled out in - ## the :bro:type:`Notice::Info` record and synchronous functions in the - ## :bro:id:`Notice::sync_functions` have already been called. The notice + ## the :bro:type:`Notice::Info` record and the notice ## policy has also been applied. ## ## n: The record containing notice data. - global notice: event(n: Info); - - ## This is a set of functions that provide a synchronous way for scripts - ## extending the notice framework to run before the normal event based - ## notice pathway that most of the notice framework takes. This is helpful - ## in cases where an action against a notice needs to happen immediately - ## and can't wait the short time for the event to bubble up to the top of - ## the event queue. An example is the IP address dropping script that - ## can block IP addresses that have notices generated because it - ## needs to operate closer to real time than the event queue allows it to. - ## Normally the event based extension model using the - ## :bro:id:`Notice::notice` event will work fine if there aren't harder - ## real time constraints. - const sync_functions: set[function(n: Notice::Info)] = set() &redef; + global notice: hook(n: Info); ## This event is generated when a notice begins to be suppressed. ## @@ -266,6 +231,11 @@ export { ## about to be suppressed. global begin_suppression: event(n: Notice::Info); + ## A function to determine if an event is supposed to be suppressed. + ## + ## n: The record containing the notice in question. + global is_being_suppressed: function(n: Notice::Info): bool; + ## This event is generated on each occurence of an event being suppressed. ## ## n: The record containing notice data regarding the notice type @@ -424,9 +394,7 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) } else { - event reporter_info(network_time(), - fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens), - ""); + Reporter::info(fmt("Notice email delay tokens weren't released in time (%s).", n$email_delay_tokens)); } } } @@ -468,7 +436,26 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool) piped_exec(fmt("%s -t -oi", sendmail), email_text); } -event notice(n: Notice::Info) &priority=-5 +hook Notice::policy(n: Notice::Info) &priority=10 + { + if ( n$note in Notice::ignored_types ) + break; + + if ( n$note in Notice::not_suppressed_types ) + add n$actions[ACTION_NO_SUPPRESS]; + if ( n$note in Notice::alarmed_types ) + add n$actions[ACTION_ALARM]; + if ( n$note in Notice::emailed_types ) + add n$actions[ACTION_EMAIL]; + + if ( n$note in Notice::type_suppression_intervals ) + n$suppress_for=Notice::type_suppression_intervals[n$note]; + + # Logging is a default action. It can be removed in a later hook if desired. + add n$actions[ACTION_LOG]; + } + +hook Notice::notice(n: Notice::Info) &priority=-5 { if ( ACTION_EMAIL in n$actions ) email_notice_to(n, mail_dest, T); @@ -565,16 +552,12 @@ function apply_policy(n: Notice::Info) if ( ! n?$email_delay_tokens ) n$email_delay_tokens = set(); - if ( ! n?$policy_items ) - n$policy_items = set(); - for ( i in ordered_policy ) { # If there's no predicate or the predicate returns F. if ( ! ordered_policy[i]?$pred || ordered_policy[i]$pred(n) ) { add n$actions[ordered_policy[i]$action]; - add n$policy_items[int_to_count(i)]; # If the predicate matched and there was a suppression interval, # apply it to the notice now. @@ -587,6 +570,9 @@ function apply_policy(n: Notice::Info) } } + # Apply the hook based policy. + hook Notice::policy(n); + # Apply the suppression time after applying the policy so that policy # items can give custom suppression intervals. If there is no # suppression interval given yet, the default is applied. @@ -610,7 +596,7 @@ event bro_init() &priority=10 Log::create_stream(Notice::POLICY_LOG, [$columns=PolicyItem]); local tmp: table[count] of set[PolicyItem] = table(); - for ( pi in policy ) + for ( pi in policy_table ) { if ( pi$priority < 0 || pi$priority > 10 ) Reporter::fatal("All Notice::PolicyItem priorities must be within 0 and 10"); @@ -638,25 +624,13 @@ event bro_init() &priority=10 function internal_NOTICE(n: Notice::Info) { - # Suppress this notice if necessary. - if ( is_being_suppressed(n) ) - return; - # Fill out fields that might be empty and do the policy processing. apply_policy(n); - # Run the synchronous functions with the notice. - for ( func in sync_functions ) - func(n); - # Generate the notice event with the notice. - event Notice::notice(n); + hook Notice::notice(n); } module GLOBAL; -## This is the entry point in the global namespace for notice framework. -function NOTICE(n: Notice::Info) - { - Notice::internal_NOTICE(n); - } +global NOTICE: function(n: Notice::Info); \ No newline at end of file diff --git a/scripts/base/frameworks/notice/non-cluster.bro b/scripts/base/frameworks/notice/non-cluster.bro new file mode 100644 index 0000000000..002ec0da34 --- /dev/null +++ b/scripts/base/frameworks/notice/non-cluster.bro @@ -0,0 +1,14 @@ + +@load ./main + +module GLOBAL; + +## This is the entry point in the global namespace for notice framework. +function NOTICE(n: Notice::Info) + { + # Suppress this notice if necessary. + if ( Notice::is_being_suppressed(n) ) + return; + + Notice::internal_NOTICE(n); + } diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 7fd3a1bdc8..d9e8ee0703 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2012-11-05-23-29-45 +#open 2013-02-11-18-44-43 #fields name #types string scripts/base/init-bare.bro @@ -53,6 +53,7 @@ scripts/base/init-default.bro scripts/base/frameworks/cluster/./main.bro scripts/base/frameworks/control/__load__.bro scripts/base/frameworks/control/./main.bro + scripts/base/frameworks/notice/./non-cluster.bro scripts/base/frameworks/notice/./actions/pp-alarms.bro scripts/base/frameworks/dpd/__load__.bro scripts/base/frameworks/dpd/./main.bro @@ -118,4 +119,4 @@ scripts/base/init-default.bro scripts/base/protocols/syslog/./main.bro scripts/base/misc/find-checksum-offloading.bro scripts/policy/misc/loaded-scripts.bro -#close 2012-11-05-23-29-45 +#close 2013-02-11-18-44-43 diff --git a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log index 217b3ed49b..c87853e2b4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.metrics.cluster-intermediate-update/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-50-59 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749059.978651 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - 1.2.3.4 - - -#close 2012-07-20-01-51-08 +#open 2013-02-11-18-41-03 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1360608063.517719 - - - - - - Test_Notice Threshold crossed by metric_index(host=1.2.3.4) 100/100 - 1.2.3.4 - - 100 manager-1 Notice::ACTION_LOG 3600.000000 F - - - - - 1.2.3.4 - - +#close 2013-02-11-18-41-03 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log index 6c93cb875e..ddbb59c565 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-51-18 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749078.270791 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-07-20-01-51-27 +#open 2013-02-11-18-45-43 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1360608343.088948 - - - - - - Test_Notice test notice! - - - - - worker-1 Notice::ACTION_LOG 3600.000000 F - - - - - - - - +#close 2013-02-11-18-45-43 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log index 88f25b066f..2f163a5491 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression-cluster/manager-1.notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-51-36 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1342749096.545663 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-07-20-01-51-45 +#open 2013-02-11-18-45-14 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1360608314.794257 - - - - - - Test_Notice test notice! - - - - - worker-2 Notice::ACTION_LOG 3600.000000 F - - - - - - - - +#close 2013-02-11-18-45-17 diff --git a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log index 7c7254f87e..f7f4148548 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log +++ b/testing/btest/Baseline/scripts.base.frameworks.notice.suppression/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-07-20-01-49-23 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double -1342748963.685754 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - -#close 2012-07-20-01-49-23 +#open 2013-02-11-18-32-39 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double +1360607559.193954 - - - - - - Test_Notice test - - - - - bro Notice::ACTION_LOG 3600.000000 F - - - - - +#close 2013-02-11-18-32-39 diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log index f9292344a8..da5489e0b7 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/notice.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path notice -#open 2012-10-05-21-45-15 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions policy_items suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network -#types time string addr port addr port enum enum string string addr addr port count string table[enum] table[count] interval bool string string string double double addr string subnet -1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 6 3600.000000 F - - - - - - - - -#close 2012-10-05-21-45-15 +#open 2013-02-11-18-33-41 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto note msg sub src dst p n peer_descr actions suppress_for dropped remote_location.country_code remote_location.region remote_location.city remote_location.latitude remote_location.longitude metric_index.host metric_index.str metric_index.network +#types time string addr port addr port enum enum string string addr addr port count string table[enum] interval bool string string string double double addr string subnet +1348168976.558309 arKYeMETxOg 192.168.57.103 35391 192.168.57.101 55968 tcp GridFTP::Data_Channel GridFTP data channel over threshold 2 bytes - 192.168.57.103 192.168.57.101 55968 - bro Notice::ACTION_LOG 3600.000000 F - - - - - - - - +#close 2013-02-11-18-33-41 diff --git a/testing/btest/scripts/base/frameworks/metrics/notice.bro b/testing/btest/scripts/base/frameworks/metrics/notice.bro deleted file mode 100644 index 0ac9faa956..0000000000 --- a/testing/btest/scripts/base/frameworks/metrics/notice.bro +++ /dev/null @@ -1,24 +0,0 @@ -# @TEST-EXEC: bro %INPUT -# @TEST-EXEC: btest-diff notice.log - - -redef enum Notice::Type += { - Test_Notice, -}; - -redef enum Metrics::ID += { - TEST_METRIC, -}; - -event bro_init() &priority=5 - { - Metrics::add_filter(TEST_METRIC, - [$name="foo-bar", - $break_interval=3secs, - $note=Test_Notice, - $notice_threshold=2, - $log=F]); - Metrics::add_data(TEST_METRIC, [$host=1.2.3.4], 3); - Metrics::add_data(TEST_METRIC, [$host=6.5.4.3], 2); - Metrics::add_data(TEST_METRIC, [$host=7.2.1.5], 1); - } diff --git a/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro index 3116b1025a..0970ec0c76 100644 --- a/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro +++ b/testing/btest/scripts/base/frameworks/notice/mail-alarms.bro @@ -1,7 +1,11 @@ # @TEST-EXEC: bro -C -r $TRACES/web.trace %INPUT # @TEST-EXEC: btest-diff alarm-mail.txt -redef Notice::policy += { [$action = Notice::ACTION_ALARM, $priority = 1 ] }; +hook Notice::policy(n: Notice::Info) &priority=1 + { + add n$actions[Notice::ACTION_ALARM]; + } + redef Notice::force_email_summaries = T; redef enum Notice::Type += { From 7e5115460c2da630825b7ef994591f95aef4f4dc Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 19 Feb 2013 11:38:17 -0600 Subject: [PATCH 32/49] Fix three bugs with 'when' and 'return when' statements. Addresses #946 - 'when' statements were problematic when used in a function/event/hook that had local variables with an assigned function value. This was because 'when' blocks operate on a clone of the frame and the cloning process serializes locals and the serialization of functions had an infinite cycle in it (ID -> BroFunc -> ID -> BroFunc ...). The ID was only used for the function name and type information, so refactoring Func and subclasses to depend on those two things instead fixes the issue. - 'return when' blocks, specifically, didn't work whenever execution of the containing function's body does another function call before reaching the 'return when' block, because of an assertion. This was was due to logic in CallExpr::Eval always clearing the CallExpr associated with the Frame after doing the call, instead of restoring any previous CallExpr, which the code in Trigger::Eval expected to have available. - An assert could be reached when the condition of a 'when' statement depended on checking the value of global state variables. The assert in Trigger::QueueTrigger that checks that the Trigger isn't disabled would get hit because Trigger::Eval/Timeout disable themselves after running, but don't unregister themselves from the NotifierRegistry, which keeps calling QueueTrigger for every state access of the global. --- src/Debug.cc | 2 +- src/Expr.cc | 3 +- src/Func.cc | 83 +++++++++---------- src/Func.h | 17 ++-- src/Stats.cc | 2 +- src/Trigger.cc | 2 + src/Trigger.h | 1 - src/input/Manager.cc | 4 +- src/logging/Manager.cc | 2 +- .../Baseline/language.returnwhen/bro..stdout | 12 +++ testing/btest/language/returnwhen.bro | 79 ++++++++++++++++++ 11 files changed, 144 insertions(+), 63 deletions(-) create mode 100644 testing/btest/Baseline/language.returnwhen/bro..stdout create mode 100644 testing/btest/language/returnwhen.bro diff --git a/src/Debug.cc b/src/Debug.cc index 535e193685..8cf2e38596 100644 --- a/src/Debug.cc +++ b/src/Debug.cc @@ -763,7 +763,7 @@ int dbg_handle_debug_input() Frame* curr_frame = g_frame_stack.back(); const BroFunc* func = curr_frame->GetFunction(); if ( func ) - current_module = func->GetID()->ModuleName(); + current_module = extract_module_name(func->Name()); else current_module = GLOBAL_MODULE_NAME; diff --git a/src/Expr.cc b/src/Expr.cc index 3df4d781a0..1a647e8c01 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -4639,12 +4639,13 @@ Val* CallExpr::Eval(Frame* f) const { const ::Func* func = func_val->AsFunc(); calling_expr = this; + const CallExpr* current_call = f ? f->GetCall() : 0; if ( f ) f->SetCall(this); ret = func->Call(v, f); // No try/catch here; we pass exceptions upstream. if ( f ) - f->ClearCall(); + f->SetCall(current_call); // Don't Unref() the arguments, as Func::Call already did that. delete v; diff --git a/src/Func.cc b/src/Func.cc index 9b94b15d97..69af3fb93c 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -54,13 +54,13 @@ bool did_builtin_init = false; vector Func::unique_ids; -Func::Func() : scope(0), id(0), return_value(0) +Func::Func() : scope(0), type(0) { unique_id = unique_ids.size(); unique_ids.push_back(this); } -Func::Func(Kind arg_kind) : scope(0), kind(arg_kind), id(0), return_value(0) +Func::Func(Kind arg_kind) : scope(0), kind(arg_kind), type(0) { unique_id = unique_ids.size(); unique_ids.push_back(this); @@ -68,6 +68,7 @@ Func::Func(Kind arg_kind) : scope(0), kind(arg_kind), id(0), return_value(0) Func::~Func() { + Unref(type); } void Func::AddBody(Stmt* /* new_body */, id_list* /* new_inits */, @@ -129,6 +130,12 @@ bool Func::DoSerialize(SerialInfo* info) const if ( ! SERIALIZE(char(kind) ) ) return false; + if ( ! type->Serialize(info) ) + return false; + + if ( ! SERIALIZE(Name()) ) + return false; + // We don't serialize scope as only global functions are considered here // anyway. return true; @@ -160,12 +167,24 @@ bool Func::DoUnserialize(UnserialInfo* info) return false; kind = (Kind) c; + + type = BroType::Unserialize(info); + if ( ! type ) + return false; + + const char* n; + if ( ! UNSERIALIZE_STR(&n, 0) ) + return false; + + name = n; + delete [] n; + return true; } void Func::DescribeDebug(ODesc* d, const val_list* args) const { - id->Describe(d); + d->Add(Name()); RecordType* func_args = FType()->Args(); if ( args ) @@ -196,21 +215,6 @@ void Func::DescribeDebug(ODesc* d, const val_list* args) const } } -void Func::SetID(ID *arg_id) - { - id = arg_id; - - return_value = - new ID(string(string(id->Name()) + "_returnvalue").c_str(), - SCOPE_FUNCTION, false); - return_value->SetType(FType()->YieldType()->Ref()); - } - -ID* Func::GetReturnValueID() const - { - return return_value; - } - TraversalCode Func::Traverse(TraversalCallback* cb) const { // FIXME: Make a fake scope for builtins? @@ -226,12 +230,6 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const tc = scope->Traverse(cb); HANDLE_TC_STMT_PRE(tc); - if ( GetReturnValueID() ) - { - tc = GetReturnValueID()->Traverse(cb); - HANDLE_TC_STMT_PRE(tc); - } - for ( unsigned int i = 0; i < bodies.size(); ++i ) { tc = bodies[i].stmts->Traverse(cb); @@ -249,7 +247,8 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits, int arg_frame_size, int priority) : Func(BRO_FUNC) { - id = arg_id; + name = arg_id->Name(); + type = arg_id->Type()->Ref(); frame_size = arg_frame_size; if ( arg_body ) @@ -263,7 +262,6 @@ BroFunc::BroFunc(ID* arg_id, Stmt* arg_body, id_list* aggr_inits, BroFunc::~BroFunc() { - Unref(id); for ( unsigned int i = 0; i < bodies.size(); ++i ) Unref(bodies[i].stmts); } @@ -378,7 +376,8 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const (flow != FLOW_RETURN /* we fell off the end */ || ! result /* explicit return with no result */) && ! f->HasDelayed() ) - reporter->Warning("non-void function returns without a value: %s", id->Name()); + reporter->Warning("non-void function returns without a value: %s", + Name()); if ( result && g_trace_state.DoTrace() ) { @@ -421,8 +420,7 @@ void BroFunc::AddBody(Stmt* new_body, id_list* new_inits, int new_frame_size, void BroFunc::Describe(ODesc* d) const { - if ( id ) - id->Describe(d); + d->Add(Name()); d->NL(); d->AddCount(frame_size); @@ -450,14 +448,14 @@ IMPLEMENT_SERIAL(BroFunc, SER_BRO_FUNC); bool BroFunc::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_BRO_FUNC, Func); - return id->Serialize(info) && SERIALIZE(frame_size); + return SERIALIZE(frame_size); } bool BroFunc::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(Func); - id = ID::Unserialize(info); - return id && UNSERIALIZE(&frame_size); + + return UNSERIALIZE(&frame_size); } BuiltinFunc::BuiltinFunc(built_in_func arg_func, const char* arg_name, @@ -465,15 +463,16 @@ BuiltinFunc::BuiltinFunc(built_in_func arg_func, const char* arg_name, : Func(BUILTIN_FUNC) { func = arg_func; - name = copy_string(make_full_var_name(GLOBAL_MODULE_NAME, arg_name).c_str()); + name = make_full_var_name(GLOBAL_MODULE_NAME, arg_name); is_pure = arg_is_pure; - id = lookup_ID(name, GLOBAL_MODULE_NAME, false); + ID* id = lookup_ID(Name(), GLOBAL_MODULE_NAME, false); if ( ! id ) - reporter->InternalError("built-in function %s missing", name); + reporter->InternalError("built-in function %s missing", Name()); if ( id->HasVal() ) - reporter->InternalError("built-in function %s multiply defined", name); + reporter->InternalError("built-in function %s multiply defined", Name()); + type = id->Type()->Ref(); id->SetVal(new Val(this)); } @@ -491,7 +490,7 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const #ifdef PROFILE_BRO_FUNCTIONS DEBUG_MSG("Function: %s\n", Name()); #endif - SegmentProfiler(segment_logger, name); + SegmentProfiler(segment_logger, Name()); if ( sample_logger ) sample_logger->FunctionSeen(this); @@ -522,8 +521,7 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const void BuiltinFunc::Describe(ODesc* d) const { - if ( id ) - id->Describe(d); + d->Add(Name()); d->AddCount(is_pure); } @@ -532,16 +530,13 @@ IMPLEMENT_SERIAL(BuiltinFunc, SER_BUILTIN_FUNC); bool BuiltinFunc::DoSerialize(SerialInfo* info) const { DO_SERIALIZE(SER_BUILTIN_FUNC, Func); - - // We ignore the ID. Func::Serialize() will rebind us anyway. - return SERIALIZE(name); + return true; } bool BuiltinFunc::DoUnserialize(UnserialInfo* info) { DO_UNSERIALIZE(Func); - id = 0; - return UNSERIALIZE_STR(&name, 0); + return true; } void builtin_error(const char* msg, BroObj* arg) diff --git a/src/Func.h b/src/Func.h index 7f9627b66d..3413e0bef1 100644 --- a/src/Func.h +++ b/src/Func.h @@ -47,15 +47,11 @@ public: virtual void SetScope(Scope* newscope) { scope = newscope; } virtual Scope* GetScope() const { return scope; } - virtual FuncType* FType() const - { - return (FuncType*) id->Type()->AsFuncType(); - } + virtual FuncType* FType() const { return type->AsFuncType(); } Kind GetKind() const { return kind; } - const ID* GetID() const { return id; } - void SetID(ID *arg_id); + const char* Name() const { return name.c_str(); } virtual void Describe(ODesc* d) const = 0; virtual void DescribeDebug(ODesc* d, const val_list* args) const; @@ -64,7 +60,6 @@ public: bool Serialize(SerialInfo* info) const; static Func* Unserialize(UnserialInfo* info); - ID* GetReturnValueID() const; virtual TraversalCode Traverse(TraversalCallback* cb) const; uint32 GetUniqueFuncID() const { return unique_id; } @@ -79,8 +74,8 @@ protected: vector bodies; Scope* scope; Kind kind; - ID* id; - ID* return_value; + BroType* type; + string name; uint32 unique_id; static vector unique_ids; }; @@ -119,18 +114,16 @@ public: int IsPure() const; Val* Call(val_list* args, Frame* parent) const; - const char* Name() const { return name; } built_in_func TheFunc() const { return func; } void Describe(ODesc* d) const; protected: - BuiltinFunc() { func = 0; name = 0; is_pure = 0; } + BuiltinFunc() { func = 0; is_pure = 0; } DECLARE_SERIAL(BuiltinFunc); built_in_func func; - const char* name; int is_pure; }; diff --git a/src/Stats.cc b/src/Stats.cc index 8d48c47a25..1bccb8f9be 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -338,7 +338,7 @@ SampleLogger::~SampleLogger() void SampleLogger::FunctionSeen(const Func* func) { - load_samples->Assign(new StringVal(func->GetID()->Name()), 0); + load_samples->Assign(new StringVal(func->Name()), 0); } void SampleLogger::LocationSeen(const Location* loc) diff --git a/src/Trigger.cc b/src/Trigger.cc index b7e08b557e..e7e887e055 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -251,6 +251,7 @@ bool Trigger::Eval() timer_mgr->Cancel(timer); Disable(); + UnregisterAll(); Unref(this); return true; @@ -337,6 +338,7 @@ void Trigger::Timeout() } Disable(); + UnregisterAll(); Unref(this); } diff --git a/src/Trigger.h b/src/Trigger.h index 8e04fb9189..dd2e171ef6 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -79,7 +79,6 @@ private: friend class TriggerTimer; void Init(); - void DeleteTrigger(); void Register(ID* id); void Register(Val* val); void UnregisterAll(); diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d9006d66a2..e8e12b64c4 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -483,7 +483,7 @@ bool Manager::CreateEventStream(RecordVal* fval) Unref(fields); // ref'd by lookupwithdefault stream->num_fields = fieldsV.size(); stream->fields = fields->Ref()->AsRecordType(); - stream->event = event_registry->Lookup(event->GetID()->Name()); + stream->event = event_registry->Lookup(event->Name()); stream->want_record = ( want_record->InternalInt() == 1 ); Unref(want_record); // ref'd by lookupwithdefault @@ -644,7 +644,7 @@ bool Manager::CreateTableStream(RecordVal* fval) stream->tab = dst->AsTableVal(); stream->rtype = val ? val->AsRecordType() : 0; stream->itype = idx->AsRecordType(); - stream->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + stream->event = event ? event_registry->Lookup(event->Name()) : 0; stream->currDict = new PDict(InputHash); stream->currDict->SetDeleteFunc(input_hash_delete_func); stream->lastDict = new PDict(InputHash); diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index f62cd1685d..1ab83d84ba 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -365,7 +365,7 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval) streams[idx]->id = id->Ref()->AsEnumVal(); streams[idx]->enabled = true; streams[idx]->name = id->Type()->AsEnumType()->Lookup(idx); - streams[idx]->event = event ? event_registry->Lookup(event->GetID()->Name()) : 0; + streams[idx]->event = event ? event_registry->Lookup(event->Name()) : 0; streams[idx]->columns = columns->Ref()->AsRecordType(); DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", diff --git a/testing/btest/Baseline/language.returnwhen/bro..stdout b/testing/btest/Baseline/language.returnwhen/bro..stdout new file mode 100644 index 0000000000..d213d7bd02 --- /dev/null +++ b/testing/btest/Baseline/language.returnwhen/bro..stdout @@ -0,0 +1,12 @@ +dummy from async_func() from bro_init() +async_func() return result in bro_init(), flag in my_set +dummy from bro_init() when block +hi! +dummy from async_func() from do_another() +async_func() return result in do_another(), flag in my_set +dummy from do_another() when block +hi! +dummy from async_func() from do_another() +async_func() return result in do_another(), timeout +dummy from do_another() when block +hi! diff --git a/testing/btest/language/returnwhen.bro b/testing/btest/language/returnwhen.bro new file mode 100644 index 0000000000..593841eb7e --- /dev/null +++ b/testing/btest/language/returnwhen.bro @@ -0,0 +1,79 @@ +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 +# @TEST-EXEC: btest-diff bro/.stdout + +redef exit_only_after_terminate = T; + +global my_set: set[string] = set(); +global flag: string = "flag"; +global done: bool = F; + +function dummyfunc(s: string): string + { + return "dummy " + s; + } + +function async_func(s: string): string + { + print dummyfunc("from async_func() " + s); + + return when ( flag in my_set ) + { + return flag + " in my_set"; + } + timeout 3sec + { + return "timeout"; + } + } + +event set_flag() + { + add my_set[flag]; + } + +event do_another() + { + delete my_set[flag]; + + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + if ( ! done ) + schedule 1sec { set_flag() }; + + when ( local result = async_func("from do_another()") ) + { + print "async_func() return result in do_another()", result; + print local_dummy("from do_another() when block"); + print anon("hi"); + if ( result == "timeout" ) + terminate(); + else + { + done = T; + schedule 10msec { do_another() }; + } + } + } + +event bro_init() + { + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + schedule 1sec { set_flag() }; + + when ( local result = async_func("from bro_init()") ) + { + print "async_func() return result in bro_init()", result; + print local_dummy("from bro_init() when block"); + print anon("hi"); + if ( result == "timeout" ) terminate(); + schedule 10msec { do_another() }; + } + } + + From d158c7ffdfed8ac58e107efaba660a1d541cb7d0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 19 Feb 2013 16:19:16 -0600 Subject: [PATCH 33/49] Fix memory leaks resulting from 'when' and 'return when' statements. Addresses #946. --- src/Frame.cc | 1 + src/StateAccess.cc | 13 +++- src/Trigger.cc | 10 ++- src/Trigger.h | 2 +- src/Type.cc | 2 + testing/btest/core/leaks/returnwhen.bro | 84 +++++++++++++++++++++++++ 6 files changed, 106 insertions(+), 6 deletions(-) create mode 100644 testing/btest/core/leaks/returnwhen.bro diff --git a/src/Frame.cc b/src/Frame.cc index f86fa32805..8a21b02b9e 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -87,6 +87,7 @@ Frame* Frame::Clone() void Frame::SetTrigger(Trigger* arg_trigger) { + ClearTrigger(); if ( arg_trigger ) Ref(arg_trigger); trigger = arg_trigger; diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 2d0a8dfc5a..0deaef5bb5 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -926,17 +926,22 @@ void NotifierRegistry::Register(ID* id, NotifierRegistry::Notifier* notifier) DBG_LOG(DBG_NOTIFIERS, "registering ID %s for notifier %s", id->Name(), notifier->Name()); + Attr* attr = new Attr(ATTR_TRACKED); + if ( id->Attrs() ) - id->Attrs()->AddAttr(new Attr(ATTR_TRACKED)); + { + if ( ! id->Attrs()->FindAttr(ATTR_TRACKED) ) + id->Attrs()->AddAttr(attr); + } else { attr_list* a = new attr_list; - Attr* attr = new Attr(ATTR_TRACKED); a->append(attr); id->SetAttrs(new Attributes(a, id->Type(), false)); - Unref(attr); } + Unref(attr); + NotifierMap::iterator i = ids.find(id->Name()); if ( i != ids.end() ) @@ -967,7 +972,9 @@ void NotifierRegistry::Unregister(ID* id, NotifierRegistry::Notifier* notifier) if ( i == ids.end() ) return; + Attr* attr = id->Attrs()->FindAttr(ATTR_TRACKED); id->Attrs()->RemoveAttr(ATTR_TRACKED); + Unref(attr); NotifierSet* s = i->second; s->erase(notifier); diff --git a/src/Trigger.cc b/src/Trigger.cc index e7e887e055..a567528232 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -242,6 +242,7 @@ bool Trigger::Eval() trigger->Cache(frame->GetCall(), v); trigger->Release(); + frame->ClearTrigger(); } Unref(v); @@ -251,7 +252,6 @@ bool Trigger::Eval() timer_mgr->Cancel(timer); Disable(); - UnregisterAll(); Unref(this); return true; @@ -331,6 +331,7 @@ void Trigger::Timeout() #endif trigger->Cache(frame->GetCall(), v); trigger->Release(); + frame->ClearTrigger(); } Unref(v); @@ -338,7 +339,6 @@ void Trigger::Timeout() } Disable(); - UnregisterAll(); Unref(this); } @@ -426,6 +426,12 @@ Val* Trigger::Lookup(const CallExpr* expr) return (i != cache.end()) ? i->second : 0; } +void Trigger::Disable() + { + UnregisterAll(); + disabled = true; + } + const char* Trigger::Name() const { assert(location); diff --git a/src/Trigger.h b/src/Trigger.h index dd2e171ef6..b752ea8ada 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -49,7 +49,7 @@ public: // Disable this trigger completely. Needed because Unref'ing the trigger // may not immediately delete it as other references may still exist. - void Disable() { disabled = true; } + void Disable(); virtual void Describe(ODesc* d) const { d->Add(""); } diff --git a/src/Type.cc b/src/Type.cc index 1fb813efa1..6ac6070660 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -696,7 +696,9 @@ string FuncType::FlavorString() const FuncType::~FuncType() { + Unref(args); Unref(arg_types); + Unref(yield); } BroType* FuncType::YieldType() diff --git a/testing/btest/core/leaks/returnwhen.bro b/testing/btest/core/leaks/returnwhen.bro new file mode 100644 index 0000000000..9fd9a794cd --- /dev/null +++ b/testing/btest/core/leaks/returnwhen.bro @@ -0,0 +1,84 @@ +# Needs perftools support. +# +# @TEST-GROUP: leaks +# +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# +# @TEST-EXEC: btest-bg-run bro HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local bro -m -b %INPUT +# @TEST-EXEC: btest-bg-wait 15 + +redef exit_only_after_terminate = T; + +global my_set: set[string] = set(); +global flag: string = "flag"; +global done: bool = F; + +function dummyfunc(s: string): string + { + return "dummy " + s; + } + +function async_func(s: string): string + { + print dummyfunc("from async_func() " + s); + + return when ( flag in my_set ) + { + return flag + " in my_set"; + } + timeout 3sec + { + return "timeout"; + } + } + +event set_flag() + { + add my_set[flag]; + } + +event do_another() + { + delete my_set[flag]; + + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + if ( ! done ) + schedule 1sec { set_flag() }; + + when ( local result = async_func("from do_another()") ) + { + print "async_func() return result in do_another()", result; + print local_dummy("from do_another() when block"); + print anon("hi"); + if ( result == "timeout" ) + terminate(); + else + { + done = T; + schedule 10msec { do_another() }; + } + } + } + +event bro_init() + { + local local_dummy = dummyfunc; + + local anon = function(s: string): string { return s + "!"; }; + + schedule 1sec { set_flag() }; + + when ( local result = async_func("from bro_init()") ) + { + print "async_func() return result in bro_init()", result; + print local_dummy("from bro_init() when block"); + print anon("hi"); + if ( result == "timeout" ) terminate(); + schedule 10msec { do_another() }; + } + } + + From dd9f361bc739f5aa4bc11f70569499e9115d0d50 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 22 Feb 2013 10:45:22 -0600 Subject: [PATCH 34/49] Fix build on OpenBSD 5.2. --- src/bro_inet_ntop.h | 1 + src/threading/SerialTypes.h | 1 + 2 files changed, 2 insertions(+) diff --git a/src/bro_inet_ntop.h b/src/bro_inet_ntop.h index 00326b092e..c018403893 100644 --- a/src/bro_inet_ntop.h +++ b/src/bro_inet_ntop.h @@ -5,6 +5,7 @@ extern "C" { #endif +#include #include const char * diff --git a/src/threading/SerialTypes.h b/src/threading/SerialTypes.h index 60aee2411e..f4f0bc0957 100644 --- a/src/threading/SerialTypes.h +++ b/src/threading/SerialTypes.h @@ -2,6 +2,7 @@ #ifndef THREADING_SERIALIZATIONTYPES_H #define THREADING_SERIALIZATIONTYPES_H +#include #include #include #include From 2481f9f83772a4e934f72c1bf9ac35fd0ea7c096 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 27 Feb 2013 16:04:36 -0600 Subject: [PATCH 35/49] Fix possible null pointer dereference in identify_data BIF. There was no check/handling for if magic_buffer() returns null. Also centralized libmagic calls for consistent error handling/output. --- src/FileAnalyzer.cc | 27 +++++---------------------- src/FileAnalyzer.h | 2 -- src/bro.bif | 28 ++++++---------------------- src/util.cc | 32 ++++++++++++++++++++++++++++++++ src/util.h | 4 ++++ 5 files changed, 47 insertions(+), 46 deletions(-) diff --git a/src/FileAnalyzer.cc b/src/FileAnalyzer.cc index d4064e8144..172f1aaa1d 100644 --- a/src/FileAnalyzer.cc +++ b/src/FileAnalyzer.cc @@ -2,6 +2,7 @@ #include "FileAnalyzer.h" #include "Reporter.h" +#include "util.h" magic_t File_Analyzer::magic = 0; magic_t File_Analyzer::magic_mime = 0; @@ -11,11 +12,8 @@ File_Analyzer::File_Analyzer(Connection* conn) { buffer_len = 0; - if ( ! magic ) - { - InitMagic(&magic, MAGIC_NONE); - InitMagic(&magic_mime, MAGIC_MIME); - } + bro_init_magic(&magic, MAGIC_NONE); + bro_init_magic(&magic_mime, MAGIC_MIME); } void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig) @@ -49,10 +47,10 @@ void File_Analyzer::Identify() const char* mime = 0; if ( magic ) - descr = magic_buffer(magic, buffer, buffer_len); + descr = bro_magic_buffer(magic, buffer, buffer_len); if ( magic_mime ) - mime = magic_buffer(magic_mime, buffer, buffer_len); + mime = bro_magic_buffer(magic_mime, buffer, buffer_len); val_list* vl = new val_list; vl->append(BuildConnVal()); @@ -61,18 +59,3 @@ void File_Analyzer::Identify() vl->append(new StringVal(mime ? mime : "")); ConnectionEvent(file_transferred, vl); } - -void File_Analyzer::InitMagic(magic_t* magic, int flags) - { - *magic = magic_open(flags); - - if ( ! *magic ) - reporter->Error("can't init libmagic: %s", magic_error(*magic)); - - else if ( magic_load(*magic, 0) < 0 ) - { - reporter->Error("can't load magic file: %s", magic_error(*magic)); - magic_close(*magic); - *magic = 0; - } - } diff --git a/src/FileAnalyzer.h b/src/FileAnalyzer.h index dcf9d22e8e..ac5c783e6b 100644 --- a/src/FileAnalyzer.h +++ b/src/FileAnalyzer.h @@ -29,8 +29,6 @@ protected: char buffer[BUFFER_SIZE]; int buffer_len; - static void InitMagic(magic_t* magic, int flags); - static magic_t magic; static magic_t magic_mime; }; diff --git a/src/bro.bif b/src/bro.bif index 8cea9d9123..29aa178316 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -16,6 +16,7 @@ #include "digest.h" #include "Reporter.h" #include "IPAddr.h" +#include "util.h" using namespace std; @@ -844,38 +845,21 @@ extern "C" { ## return_mime: If true, the function returns a short MIME type string (e.g., ## ``text/plain`` instead of a more elaborate textual description). ## -## Returns: The MIME type of *data*. +## Returns: The MIME type of *data*, or "" if there was an error. function identify_data%(data: string, return_mime: bool%): string %{ - const char* descr = ""; - static magic_t magic_mime = 0; static magic_t magic_descr = 0; magic_t* magic = return_mime ? &magic_mime : &magic_descr; + bro_init_magic(magic, return_mime ? MAGIC_MIME : MAGIC_NONE); if( ! *magic ) - { - *magic = magic_open(return_mime ? MAGIC_MIME : MAGIC_NONE); + return new StringVal(""); - if ( ! *magic ) - { - reporter->Error("can't init libmagic: %s", magic_error(*magic)); - return new StringVal(""); - } + const char* desc = bro_magic_buffer(*magic, data->Bytes(), data->Len()); - if ( magic_load(*magic, 0) < 0 ) - { - reporter->Error("can't load magic file: %s", magic_error(*magic)); - magic_close(*magic); - *magic = 0; - return new StringVal(""); - } - } - - descr = magic_buffer(*magic, data->Bytes(), data->Len()); - - return new StringVal(descr); + return new StringVal(desc ? desc : ""); %} ## Performs an entropy test on the given data. diff --git a/src/util.cc b/src/util.cc index c36ff6a31c..1c3de862a5 100644 --- a/src/util.cc +++ b/src/util.cc @@ -1527,3 +1527,35 @@ void operator delete[](void* v) } #endif + +void bro_init_magic(magic_t* cookie_ptr, int flags) + { + if ( ! cookie_ptr || *cookie_ptr ) return; + + *cookie_ptr = magic_open(flags); + + if ( ! *cookie_ptr ) + { + const char* err = magic_error(*cookie_ptr); + reporter->Error("can't init libmagic: %s", err ? err : "unknown"); + } + + else if ( magic_load(*cookie_ptr, 0) < 0 ) + { + const char* err = magic_error(*cookie_ptr); + reporter->Error("can't load magic file: %s", err ? err : "unknown"); + magic_close(*cookie_ptr); + *cookie_ptr = 0; + } + } + +const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length) + { + const char* rval = magic_buffer(cookie, buffer, length); + if ( ! rval ) + { + const char* err = magic_error(cookie); + reporter->Error("magic_buffer error: %s", err ? err : "unknown"); + } + return rval; + } diff --git a/src/util.h b/src/util.h index 7d65f42fa8..5d4115773d 100644 --- a/src/util.h +++ b/src/util.h @@ -15,6 +15,7 @@ #include #include #include +#include #include "config.h" #if __STDC__ @@ -364,4 +365,7 @@ struct CompareString } }; +void bro_init_magic(magic_t* cookie_ptr, int flags); +const char* bro_magic_buffer(magic_t cookie, const void* buffer, size_t length); + #endif From c88babf6ef169cb2007188ffecfeccea5aed6882 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 1 Mar 2013 16:42:16 -0600 Subject: [PATCH 36/49] Fix init of local sets/vectors via curly brace initializer lists. --- src/Expr.cc | 21 ++++++--- .../language.container-ctor-scope/out | 44 +++++++++++++++++++ .../btest/language/container-ctor-scope.bro | 38 ++++++++++++++++ 3 files changed, 98 insertions(+), 5 deletions(-) create mode 100644 testing/btest/Baseline/language.container-ctor-scope/out create mode 100644 testing/btest/language/container-ctor-scope.bro diff --git a/src/Expr.cc b/src/Expr.cc index 3df4d781a0..db82f95ef1 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2507,15 +2507,26 @@ bool AssignExpr::TypeCheck(attr_list* attrs) attr_copy->append((*attrs)[i]); } - op2 = new TableConstructorExpr(op2->AsListExpr(), attr_copy); + if ( op1->Type()->IsSet() ) + op2 = new SetConstructorExpr(op2->AsListExpr(), attr_copy); + else + op2 = new TableConstructorExpr(op2->AsListExpr(), attr_copy); return true; } - if ( bt1 == TYPE_VECTOR && bt2 == bt1 && - op2->Type()->AsVectorType()->IsUnspecifiedVector() ) + if ( bt1 == TYPE_VECTOR ) { - op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType()); - return true; + if ( bt2 == bt1 && op2->Type()->AsVectorType()->IsUnspecifiedVector() ) + { + op2 = new VectorCoerceExpr(op2, op1->Type()->AsVectorType()); + return true; + } + + if ( op2->Tag() == EXPR_LIST ) + { + op2 = new VectorConstructorExpr(op2->AsListExpr()); + return true; + } } if ( op1->Type()->Tag() == TYPE_RECORD && diff --git a/testing/btest/Baseline/language.container-ctor-scope/out b/testing/btest/Baseline/language.container-ctor-scope/out new file mode 100644 index 0000000000..4f9acfce86 --- /dev/null +++ b/testing/btest/Baseline/language.container-ctor-scope/out @@ -0,0 +1,44 @@ +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +2/tcp, +1/tcp, +3/tcp +} +{ +2/tcp, +1/tcp, +3/tcp +} +[1/tcp, 2/tcp, 3/tcp, 1/tcp] +[1/tcp, 2/tcp, 3/tcp, 1/tcp] +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +[2/tcp] = 2, +[1/tcp] = 1, +[3/tcp] = 3 +} +{ +2/tcp, +1/tcp, +3/tcp +} +{ +2/tcp, +1/tcp, +3/tcp +} +[1/tcp, 2/tcp, 3/tcp, 1/tcp] +[1/tcp, 2/tcp, 3/tcp, 1/tcp] diff --git a/testing/btest/language/container-ctor-scope.bro b/testing/btest/language/container-ctor-scope.bro new file mode 100644 index 0000000000..fd1939a459 --- /dev/null +++ b/testing/btest/language/container-ctor-scope.bro @@ -0,0 +1,38 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +# All various container contructors should work at both global and local scope. + +global gt1: table[port] of count = table( [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 ); +global gs1: set[port] = set( 1/tcp, 2/tcp, 3/tcp ); +global gv1: vector of port = vector( 1/tcp, 2/tcp, 3/tcp, 1/tcp ); + +global gt2: table[port] of count = { [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 }; +global gs2: set[port] = { 1/tcp, 2/tcp, 3/tcp }; +global gv2: vector of port = { 1/tcp, 2/tcp, 3/tcp, 1/tcp }; + +local t1: table[port] of count = table( [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 ); +local s1: set[port] = set( 1/tcp, 2/tcp, 3/tcp ); +local v1: vector of port = vector( 1/tcp, 2/tcp, 3/tcp, 1/tcp ); + +local t2: table[port] of count = { [1/tcp] = 1, [2/tcp] = 2, [3/tcp] = 3 }; +local s2: set[port] = { 1/tcp, 2/tcp, 3/tcp }; +local v2: vector of port = { 1/tcp, 2/tcp, 3/tcp, 1/tcp }; + +print gt1; +print gt2; + +print gs1; +print gs2; + +print gv1; +print gv2; + +print t1; +print t2; + +print s1; +print s2; + +print v1; +print v2; From 00eeadf2f5b9b4627c822ce9fc57dd181ab056c8 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 6 Mar 2013 09:44:25 -0500 Subject: [PATCH 37/49] Complete removal of the old table based notice policy mechanism. --- scripts/base/frameworks/notice/cluster.bro | 22 ++--- scripts/base/frameworks/notice/main.bro | 103 +-------------------- 2 files changed, 9 insertions(+), 116 deletions(-) diff --git a/scripts/base/frameworks/notice/cluster.bro b/scripts/base/frameworks/notice/cluster.bro index 02f11533fa..c722f37bab 100644 --- a/scripts/base/frameworks/notice/cluster.bro +++ b/scripts/base/frameworks/notice/cluster.bro @@ -25,15 +25,6 @@ event Notice::begin_suppression(n: Notice::Info) { suppressing[n$note, n$identifier] = n; } - -event bro_init() &priority=-3 - { - # Workers and proxies need to disable the notice streams because notice - # events are forwarded directly instead of being logged remotely. - Log::disable_stream(Notice::LOG); - Log::disable_stream(Notice::POLICY_LOG); - Log::disable_stream(Notice::ALARM_LOG); - } @endif @if ( Cluster::local_node_type() == Cluster::MANAGER ) @@ -46,17 +37,16 @@ event Notice::cluster_notice(n: Notice::Info) module GLOBAL; -## This is the entry point in the global namespace for notice framework. +## This is the entry point in the global namespace for the notice framework. function NOTICE(n: Notice::Info) { # Suppress this notice if necessary. if ( Notice::is_being_suppressed(n) ) return; -@if ( Cluster::local_node_type() == Cluster::MANAGER ) - Notice::internal_NOTICE(n); -@else - # For non-managers, send the notice on to the manager. - event Notice::cluster_notice(n); -@endif + if ( Cluster::local_node_type() == Cluster::MANAGER ) + Notice::internal_NOTICE(n); + else + # For non-managers, send the notice on to the manager. + event Notice::cluster_notice(n); } diff --git a/scripts/base/frameworks/notice/main.bro b/scripts/base/frameworks/notice/main.bro index 5d3cc48091..60415b8907 100644 --- a/scripts/base/frameworks/notice/main.bro +++ b/scripts/base/frameworks/notice/main.bro @@ -10,9 +10,6 @@ export { redef enum Log::ID += { ## This is the primary logging stream for notices. LOG, - ## This is the notice policy auditing log. It records what the current - ## notice policy is at Bro init time. - POLICY_LOG, ## This is the alarm stream. ALARM_LOG, }; @@ -42,9 +39,6 @@ export { ## version of the alarm log is emailed in bulk to the address(es) ## configured in :bro:id:`Notice::mail_dest`. ACTION_ALARM, - ## Indicates that the notice should not be supressed by the normal - ## duplicate notice suppression that the notice framework does. - ACTION_NO_SUPPRESS, }; ## The notice framework is able to do automatic notice supression by @@ -138,9 +132,8 @@ export { identifier: string &optional; ## This field indicates the length of time that this - ## unique notice should be suppressed. This field is automatically - ## filled out and should not be written to by any other script. - suppress_for: interval &log &optional; + ## unique notice should be suppressed. + suppress_for: interval &log &default=default_suppression_interval; }; ## Ignored notice types. @@ -155,39 +148,6 @@ export { ## intervals for entire notice types. const type_suppression_intervals: table[Notice::Type] of interval = {} &redef; - ## This is the record that defines the items that make up the notice policy. - type PolicyItem: record { - ## This is the exact positional order in which the - ## :bro:type:`Notice::PolicyItem` records are checked. - ## This is set internally by the notice framework. - position: count &log &optional; - ## Define the priority for this check. Items are checked in ordered - ## from highest value (10) to lowest value (0). - priority: count &log &default=5; - ## An action given to the notice if the predicate return true. - action: Notice::Action &log &default=ACTION_NONE; - ## The pred (predicate) field is a function that returns a boolean T - ## or F value. If the predicate function return true, the action in - ## this record is applied to the notice that is given as an argument - ## to the predicate function. If no predicate is supplied, it's - ## assumed that the PolicyItem always applies. - pred: function(n: Notice::Info): bool &log &optional; - ## Indicates this item should terminate policy processing if the - ## predicate returns T. - halt: bool &log &default=F; - ## This defines the length of time that this particular notice should - ## be supressed. - suppress_for: interval &log &optional; - }; - - ## Defines a notice policy that is extensible on a per-site basis. - ## All notice processing is done through this variable. This variable - ## is the former 'policy' variable, and - ## this variable is deprecated and will be removed in a future version. - ## All notice policy decisions are going to be done through the - ## 'policy' hook now. - const policy_table: set[PolicyItem] = {} &redef; - ## The hook to modify notice handling. global policy: hook(n: Notice::Info); @@ -308,10 +268,6 @@ global suppressing: table[Type, string] of Notice::Info = {} &create_expire=0secs &expire_func=per_notice_suppression_interval; -# This is an internal variable used to store the notice policy ordered by -# priority. -global ordered_policy: vector of PolicyItem = vector(); - function log_mailing_postprocessor(info: Log::RotationInfo): bool { if ( ! reading_traces() && mail_dest != "" ) @@ -442,7 +398,7 @@ hook Notice::policy(n: Notice::Info) &priority=10 break; if ( n$note in Notice::not_suppressed_types ) - add n$actions[ACTION_NO_SUPPRESS]; + n$suppress_for=0secs; if ( n$note in Notice::alarmed_types ) add n$actions[ACTION_ALARM]; if ( n$note in Notice::emailed_types ) @@ -467,7 +423,6 @@ hook Notice::notice(n: Notice::Info) &priority=-5 # Normally suppress further notices like this one unless directed not to. # n$identifier *must* be specified for suppression to function at all. if ( n?$identifier && - ACTION_NO_SUPPRESS !in n$actions && [n$note, n$identifier] !in suppressing && n$suppress_for != 0secs ) { @@ -552,24 +507,6 @@ function apply_policy(n: Notice::Info) if ( ! n?$email_delay_tokens ) n$email_delay_tokens = set(); - for ( i in ordered_policy ) - { - # If there's no predicate or the predicate returns F. - if ( ! ordered_policy[i]?$pred || ordered_policy[i]$pred(n) ) - { - add n$actions[ordered_policy[i]$action]; - - # If the predicate matched and there was a suppression interval, - # apply it to the notice now. - if ( ordered_policy[i]?$suppress_for ) - n$suppress_for = ordered_policy[i]$suppress_for; - - # If the policy item wants to halt policy processing, do it now! - if ( ordered_policy[i]$halt ) - break; - } - } - # Apply the hook based policy. hook Notice::policy(n); @@ -588,40 +525,6 @@ function apply_policy(n: Notice::Info) delete n$iconn; } -# Create the ordered notice policy automatically which will be used at runtime -# for prioritized matching of the notice policy. -event bro_init() &priority=10 - { - # Create the policy log here because it's only written to in this handler. - Log::create_stream(Notice::POLICY_LOG, [$columns=PolicyItem]); - - local tmp: table[count] of set[PolicyItem] = table(); - for ( pi in policy_table ) - { - if ( pi$priority < 0 || pi$priority > 10 ) - Reporter::fatal("All Notice::PolicyItem priorities must be within 0 and 10"); - - if ( pi$priority !in tmp ) - tmp[pi$priority] = set(); - add tmp[pi$priority][pi]; - } - - local rev_count = vector(10,9,8,7,6,5,4,3,2,1,0); - for ( i in rev_count ) - { - local j = rev_count[i]; - if ( j in tmp ) - { - for ( pi in tmp[j] ) - { - pi$position = |ordered_policy|; - ordered_policy[|ordered_policy|] = pi; - Log::write(Notice::POLICY_LOG, pi); - } - } - } - } - function internal_NOTICE(n: Notice::Info) { # Fill out fields that might be empty and do the policy processing. From e56a33b6c5801ef0dedbd1c4bc4ee388a149f74e Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Wed, 6 Mar 2013 10:04:50 -0500 Subject: [PATCH 38/49] Update notice framework documentation to represent the new reality. --- doc/notice.rst | 155 ++++++++++++++++++------------------------------- 1 file changed, 58 insertions(+), 97 deletions(-) diff --git a/doc/notice.rst b/doc/notice.rst index 5849e605a9..3aba73885a 100644 --- a/doc/notice.rst +++ b/doc/notice.rst @@ -6,7 +6,7 @@ Notice Framework One of the easiest ways to customize Bro is writing a local notice policy. Bro can detect a large number of potentially interesting - situations, and the notice policy tells which of them the user wants to be + situations, and the notice policy hook which of them the user wants to be acted upon in some manner. In particular, the notice policy can specify actions to be taken, such as sending an email or compiling regular alarm emails. This page gives an introduction into writing such a notice @@ -24,8 +24,8 @@ of interest for the user. However, none of these scripts determines the importance of what it finds itself. Instead, the scripts only flag situations as *potentially* interesting, leaving it to the local configuration to define which of them are in fact actionable. This decoupling of detection and -reporting allows Bro to address the different needs that sites have: -definitions of what constitutes an attack or even a compromise differ quite a +reporting allows Bro to address the different needs that sites have. +Definitions of what constitutes an attack or even a compromise differ quite a bit between environments, and activity deemed malicious at one site might be fully acceptable at another. @@ -40,7 +40,7 @@ More information about raising notices can be found in the `Raising Notices`_ section. Once a notice is raised, it can have any number of actions applied to it by -the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_ +writing :bro:see:`Notice::policy` hooks which is described in the `Notice Policy`_ section below. Such actions can be to send a mail to the configured address(es) or to simply ignore the notice. Currently, the following actions are defined: @@ -68,12 +68,6 @@ are defined: - Send an email to the email address or addresses given in the :bro:see:`Notice::mail_page_dest` variable. - * - Notice::ACTION_NO_SUPPRESS - - This action will disable the built in notice suppression for the - notice. Keep in mind that this action will need to be applied to - every notice that shouldn't be suppressed including each of the future - notices that would have normally been suppressed. - How these notice actions are applied to notices is discussed in the `Notice Policy`_ and `Notice Policy Shortcuts`_ sections. @@ -83,26 +77,24 @@ Processing Notices Notice Policy ************* -The predefined set :bro:see:`Notice::policy` provides the mechanism for -applying actions and other behavior modifications to notices. Each entry -of :bro:see:`Notice::policy` is a record of the type -:bro:see:`Notice::PolicyItem` which defines a condition to be matched -against all raised notices and one or more of a variety of behavior -modifiers. The notice policy is defined by adding any number of -:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy` -set. +The hook :bro:see:`Notice::policy` provides the mechanism for applying +actions and generally modifying the notice before it's sent onward to +the action plugins. Hooks can be thought of as multi-bodied functions +and using them looks very similar to handling events. The difference +is that they don't go through the event queue like events. Users should +directly make modifications to the :bro:see:`Notice::Info` record +given as the argument to the hook. Here's a simple example which tells Bro to send an email for all notices of type :bro:see:`SSH::Login` if the server is 10.0.0.1: .. code:: bro - redef Notice::policy += { - [$pred(n: Notice::Info) = { - return n$note == SSH::Login && n$id$resp_h == 10.0.0.1; - }, - $action = Notice::ACTION_EMAIL] - }; + hook Notice::policy(n: Notice::Info) + { + if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 ) + add n$actions[Notice::ACTION_EMAIL]; + } .. note:: @@ -110,78 +102,21 @@ type :bro:see:`SSH::Login` if the server is 10.0.0.1: such that it is only raised when Bro heuristically detects a successful login. No apparently failed logins will raise this notice. -While the syntax might look a bit convoluted at first, it provides a lot of -flexibility due to having access to Bro's full programming language. - -Predicate Field -^^^^^^^^^^^^^^^ - -The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred`` -which defines the entry's condition in the form of a predicate written -as a Bro function. The function is passed the notice as a -:bro:see:`Notice::Info` record and it returns a boolean value indicating -if the entry is applicable to that particular notice. - -.. note:: - - The lack of a predicate in a ``Notice::PolicyItem`` is implicitly true - (``T``) since an implicit false (``F``) value would never be used. - -Bro evaluates the predicates of each entry in the order defined by the -``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid -values are 0-10 with 10 being earliest evaluated. If ``$priority`` is -omitted, the default priority is 5. - -Behavior Modification Fields -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that -indicate ways that either the notice or notice processing should be modified -if the predicate field (``$pred``) evaluated to true (``T``). Those fields are -explained in more detail in the following table. - -.. list-table:: - :widths: 20 30 20 - :header-rows: 1 - - * - Field - - Description - - Example - - * - ``$action=`` - - Each :bro:see:`Notice::PolicyItem` can have a single action - applied to the notice with this field. - - ``$action = Notice::ACTION_EMAIL`` - - * - ``$suppress_for=`` - - This field makes it possible for a user to modify the behavior of the - notice framework's automated suppression of intrinsically similar - notices. More information about the notice framework's automated - suppression can be found in the `Automated Suppression`_ section of - this document. - - ``$suppress_for = 10mins`` - - * - ``$halt=`` - - This field can be used for modification of the notice policy - evaluation. To stop processing of notice policy items before - evaluating all of them, set this field to ``T`` and make the ``$pred`` - field return ``T``. :bro:see:`Notice::PolicyItem` records defined at - a higher priority as defined by the ``$priority`` field will still be - evaluated but those at a lower priority won't. - - ``$halt = T`` - - +Hooks can also have priorities applied to order their execution like events +with a default priority of 0. Greater values are executed first. Setting +a hook body to run before default hook bodies might look like this: .. code:: bro - redef Notice::policy += { - [$pred(n: Notice::Info) = { - return n$note == SSH::Login && n$id$resp_h == 10.0.0.1; - }, - $action = Notice::ACTION_EMAIL, - $priority=5] - }; + hook Notice::policy(n: Notice::Info) &priority=5 + { + if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 ) + add n$actions[Notice::ACTION_EMAIL]; + } +Hooks can also abort later hook bodies with the ``break`` keyword. This +is primarily useful if one wants to completely preempt processing by +lower priority :bro:see:`Notice::policy` hooks. Notice Policy Shortcuts *********************** @@ -189,7 +124,7 @@ Notice Policy Shortcuts Although the notice framework provides a great deal of flexibility and configurability there are many times that the full expressiveness isn't needed and actually becomes a hindrance to achieving results. The framework provides -a default :bro:see:`Notice::policy` suite as a way of giving users the +a default :bro:see:`Notice::policy` hook body as a way of giving users the shortcuts to easily apply many common actions to notices. These are implemented as sets and tables indexed with a @@ -377,19 +312,45 @@ Setting the ``$identifier`` field is left to those raising notices because it's assumed that the script author who is raising the notice understands the full problem set and edge cases of the notice which may not be readily apparent to users. If users don't want the suppression to take place or simply -want a different interval, they can always modify it with the -:bro:see:`Notice::policy`. +want a different interval, they can set a notice's suppression +interval to ``0secs`` or delete the value from the ``$identifier`` field in +a :bro:see:`Notice::policy` hook. Extending Notice Framework -------------------------- -Adding Custom Notice Actions -**************************** +There are a couple of mechanism currently for extending the notice framework +and adding new capability. Extending Notice Emails *********************** +If there is extra information that you would like to add to emails, that is +possible to add by writing :bro:see:`Notice::policy` hooks. + +There is a field in the :bro:see:`Notice::Info` record named +``$email_body_sections`` which will be included verbatim when email is being +sent. An example of including some information from an HTTP request is +included below. + +.. code:: bro + + hook Notice::policy(n: Notice::Info) + { + if ( n?$conn && n$conn?$http && n$conn$http?$host ) + n$email_body_sections[|email_body_sections|] = fmt("HTTP host header: %s", n$conn$http$host); + } + + Cluster Considerations ---------------------- +As a user/developer of Bro, the main cluster concern with the notice framework +is understanding what runs where. When a notice is generated on a worker, the +worker checks to see if the notice shoudl be suppressed based on information +locally maintained in the worker process. If it's not being +suppressed, the worker forwards the notice directly to the manager and does no more +local processing. The manager then runs the :bro:see:`Notice::policy` hook and +executes all of the actions determined to be run. + From 986b346e3f3d58a4052815bc1ee9c9dc7a19638f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 6 Mar 2013 13:45:42 -0800 Subject: [PATCH 39/49] remove the byte_len and length bifs --- scripts/base/frameworks/signatures/main.bro | 6 ++--- scripts/base/utils/paths.bro | 2 +- scripts/base/utils/strings.bro | 2 +- scripts/policy/protocols/http/detect-MHR.bro | 2 +- src/bro.bif | 23 -------------------- src/strings.bif | 13 ----------- testing/btest/bifs/byte_len.bro | 10 --------- testing/btest/bifs/length.bro | 22 ------------------- 8 files changed, 6 insertions(+), 74 deletions(-) delete mode 100644 testing/btest/bifs/byte_len.bro delete mode 100644 testing/btest/bifs/length.bro diff --git a/scripts/base/frameworks/signatures/main.bro b/scripts/base/frameworks/signatures/main.bro index 4102214075..7e2f71ea8c 100644 --- a/scripts/base/frameworks/signatures/main.bro +++ b/scripts/base/frameworks/signatures/main.bro @@ -161,7 +161,7 @@ event signature_match(state: signature_state, msg: string, data: string) return; # Trim the matched data down to something reasonable - if ( byte_len(data) > 140 ) + if ( |data| > 140 ) data = fmt("%s...", sub_bytes(data, 0, 140)); local src_addr: addr; @@ -259,8 +259,8 @@ event signature_match(state: signature_state, msg: string, data: string) add vert_table[orig, resp][sig_id]; - local hcount = length(horiz_table[orig, sig_id]); - local vcount = length(vert_table[orig, resp]); + local hcount = |horiz_table[orig, sig_id]|; + local vcount = |vert_table[orig, resp]|; if ( hcount in horiz_scan_thresholds && hcount != last_hthresh[orig] ) { diff --git a/scripts/base/utils/paths.bro b/scripts/base/utils/paths.bro index 92f0745be4..aa083ddf5b 100644 --- a/scripts/base/utils/paths.bro +++ b/scripts/base/utils/paths.bro @@ -27,7 +27,7 @@ function compress_path(dir: string): string const cdup_sep = /((\/)*([^\/]|\\\/)+)?((\/)+\.\.(\/)*)/; local parts = split_n(dir, cdup_sep, T, 1); - if ( length(parts) > 1 ) + if ( |parts| > 1 ) { # reaching a point with two parent dir references back-to-back means # we don't know about anything higher in the tree to pop off diff --git a/scripts/base/utils/strings.bro b/scripts/base/utils/strings.bro index 2836f368b4..560ba6d160 100644 --- a/scripts/base/utils/strings.bro +++ b/scripts/base/utils/strings.bro @@ -6,7 +6,7 @@ ## characters. function is_string_binary(s: string): bool { - return byte_len(gsub(s, /[\x00-\x7f]/, "")) * 100 / |s| >= 25; + return |gsub(s, /[\x00-\x7f]/, "")| * 100 / |s| >= 25; } ## Joins a set of string together, with elements delimited by a constant string. diff --git a/scripts/policy/protocols/http/detect-MHR.bro b/scripts/policy/protocols/http/detect-MHR.bro index 1898022978..0594276c93 100644 --- a/scripts/policy/protocols/http/detect-MHR.bro +++ b/scripts/policy/protocols/http/detect-MHR.bro @@ -32,7 +32,7 @@ event log_http(rec: HTTP::Info) { # Data is returned as " " local MHR_answer = split1(MHR_result, / /); - if ( length(MHR_answer) == 2 && to_count(MHR_answer[2]) >= MHR_threshold ) + if ( |MHR_answer| == 2 && to_count(MHR_answer[2]) >= MHR_threshold ) { local url = HTTP::build_url_http(rec); local message = fmt("%s %s %s", rec$id$orig_h, rec$md5, url); diff --git a/src/bro.bif b/src/bro.bif index 8cea9d9123..15c2e8da3f 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1033,29 +1033,6 @@ function clear_table%(v: any%): any return 0; %} -## Returns the number of elements in a container. This function works with all -## container types, i.e., sets, tables, and vectors. -## -## v: The container whose elements are counted. -## -## Returns: The number of elements in *v*. -function length%(v: any%): count - %{ - TableVal* tv = v->Type()->Tag() == TYPE_TABLE ? v->AsTableVal() : 0; - - if ( tv ) - return new Val(tv->Size(), TYPE_COUNT); - - else if ( v->Type()->Tag() == TYPE_VECTOR ) - return new Val(v->AsVectorVal()->Size(), TYPE_COUNT); - - else - { - builtin_error("length() requires a table/set/vector argument"); - return new Val(0, TYPE_COUNT); - } - %} - ## Checks whether two objects reference the same internal object. This function ## uses equality comparison of C++ raw pointer values to determine if the two ## objects are the same. diff --git a/src/strings.bif b/src/strings.bif index dc5e064dc6..22f5429a8b 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -265,19 +265,6 @@ function edit%(arg_s: string, arg_edit_char: string%): string return new StringVal(new BroString(1, byte_vec(new_s), ind)); %} -## Returns the number of characters (bytes) in the given string. The -## length computation includes any embedded NULs, and also a trailing NUL, -## if any (which is why the function isn't called ``strlen``; to remind -## the user that Bro strings can include NULs). -## -## s: The string to compute the length for. -## -## Returns: The number of characters in *s*. -function byte_len%(s: string%): count - %{ - return new Val(s->Len(), TYPE_COUNT); - %} - ## Get a substring from a string, given a starting position and length. ## ## s: The string to obtain a substring from. diff --git a/testing/btest/bifs/byte_len.bro b/testing/btest/bifs/byte_len.bro deleted file mode 100644 index bd15b0c390..0000000000 --- a/testing/btest/bifs/byte_len.bro +++ /dev/null @@ -1,10 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT >out -# @TEST-EXEC: btest-diff out - -event bro_init() - { - local a = "hello\0there"; - - print byte_len(a); - } diff --git a/testing/btest/bifs/length.bro b/testing/btest/bifs/length.bro deleted file mode 100644 index ca82d7eab7..0000000000 --- a/testing/btest/bifs/length.bro +++ /dev/null @@ -1,22 +0,0 @@ -# -# @TEST-EXEC: bro -b %INPUT > out -# @TEST-EXEC: btest-diff out - -event bro_init() - { - local mytable: table[string] of string = { ["key1"] = "val1" }; - local myset: set[count] = set( 3, 6, 2, 7 ); - local myvec: vector of string = vector( "value1", "value2" ); - - print length(mytable); - print length(myset); - print length(myvec); - - mytable = table(); - myset = set(); - myvec = vector(); - - print length(mytable); - print length(myset); - print length(myvec); - } From 8f259f866d7ca05cf4aaeb507ed3891cb742f776 Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 6 Mar 2013 14:08:06 -0800 Subject: [PATCH 40/49] change vector assignment operator and remove unnecessary argument (expr) --- src/BroString.cc | 2 +- src/CompHash.cc | 2 +- src/Expr.cc | 43 ++++++++++++++++++--------------------- src/ICMP.cc | 2 +- src/IP.cc | 4 ++-- src/NFS.cc | 2 +- src/SmithWaterman.cc | 4 ++-- src/StateAccess.cc | 4 ++-- src/TunnelEncapsulation.h | 2 +- src/Val.cc | 9 ++++---- src/Val.h | 10 ++++----- src/bro.bif | 8 ++++---- src/input/Manager.cc | 2 +- src/modbus-analyzer.pac | 36 ++++++++++++++++---------------- src/strings.bif | 2 +- 15 files changed, 63 insertions(+), 69 deletions(-) diff --git a/src/BroString.cc b/src/BroString.cc index e05995b156..19f7a16566 100644 --- a/src/BroString.cc +++ b/src/BroString.cc @@ -369,7 +369,7 @@ VectorVal* BroString:: VecToPolicy(Vec* vec) BroString* string = (*vec)[i]; StringVal* val = new StringVal(string->Len(), (const char*) string->Bytes()); - result->Assign(i+1, val, 0); + result->Assign(i+1, val); } return result; diff --git a/src/CompHash.cc b/src/CompHash.cc index 306b3ea83e..05d3e515d2 100644 --- a/src/CompHash.cc +++ b/src/CompHash.cc @@ -856,7 +856,7 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0, if ( have_val ) kp1 = RecoverOneVal(k, kp1, k_end, vt->YieldType(), value, false); - vv->Assign(index, value, 0); + vv->Assign(index, value); } pval = vv; diff --git a/src/Expr.cc b/src/Expr.cc index 3df4d781a0..19295e92d5 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -485,7 +485,7 @@ Val* UnaryExpr::Eval(Frame* f) const for ( unsigned int i = 0; i < v_op->Size(); ++i ) { Val* v_i = v_op->Lookup(i); - result->Assign(i, v_i ? Fold(v_i) : 0, this); + result->Assign(i, v_i ? Fold(v_i) : 0); } Unref(v); @@ -625,10 +625,9 @@ Val* BinaryExpr::Eval(Frame* f) const if ( v_op1->Lookup(i) && v_op2->Lookup(i) ) v_result->Assign(i, Fold(v_op1->Lookup(i), - v_op2->Lookup(i)), - this); + v_op2->Lookup(i))); else - v_result->Assign(i, 0, this); + v_result->Assign(i, 0); // SetError("undefined element in vector operation"); } @@ -648,10 +647,9 @@ Val* BinaryExpr::Eval(Frame* f) const if ( vv_i ) v_result->Assign(i, is_vec1 ? - Fold(vv_i, v2) : Fold(v1, vv_i), - this); + Fold(vv_i, v2) : Fold(v1, vv_i)); else - v_result->Assign(i, 0, this); + v_result->Assign(i, 0); // SetError("Undefined element in vector operation"); } @@ -1049,10 +1047,10 @@ Val* IncrExpr::Eval(Frame* f) const if ( elt ) { Val* new_elt = DoSingleEval(f, elt); - v_vec->Assign(i, new_elt, this, OP_INCR); + v_vec->Assign(i, new_elt, OP_INCR); } else - v_vec->Assign(i, 0, this, OP_INCR); + v_vec->Assign(i, 0, OP_INCR); } op->Assign(f, v_vec, OP_INCR); } @@ -1919,7 +1917,7 @@ Val* BoolExpr::Eval(Frame* f) const result = new VectorVal(Type()->AsVectorType()); result->Resize(vector_v->Size()); result->AssignRepeat(0, result->Size(), - scalar_v, this); + scalar_v); } else result = vector_v->Ref()->AsVectorVal(); @@ -1957,10 +1955,10 @@ Val* BoolExpr::Eval(Frame* f) const (! op1->IsZero() && ! op2->IsZero()) : (! op1->IsZero() || ! op2->IsZero()); - result->Assign(i, new Val(local_result, TYPE_BOOL), this); + result->Assign(i, new Val(local_result, TYPE_BOOL)); } else - result->Assign(i, 0, this); + result->Assign(i, 0); } Unref(v1); @@ -2334,10 +2332,9 @@ Val* CondExpr::Eval(Frame* f) const if ( local_cond ) result->Assign(i, local_cond->IsZero() ? - b->Lookup(i) : a->Lookup(i), - this); + b->Lookup(i) : a->Lookup(i)); else - result->Assign(i, 0, this); + result->Assign(i, 0); } return result; @@ -2961,7 +2958,7 @@ Val* IndexExpr::Eval(Frame* f) const for ( unsigned int i = 0; i < v_v2->Size(); ++i ) { if ( v_v2->Lookup(i)->AsBool() ) - v_result->Assign(v_result->Size() + 1, v_v1->Lookup(i), this); + v_result->Assign(v_result->Size() + 1, v_v1->Lookup(i)); } } else @@ -2971,7 +2968,7 @@ Val* IndexExpr::Eval(Frame* f) const // Probably only do this if *all* are negative. v_result->Resize(v_v2->Size()); for ( unsigned int i = 0; i < v_v2->Size(); ++i ) - v_result->Assign(i, v_v1->Lookup(v_v2->Lookup(i)->CoerceToInt()), this); + v_result->Assign(i, v_v1->Lookup(v_v2->Lookup(i)->CoerceToInt())); } } else @@ -3048,7 +3045,7 @@ void IndexExpr::Assign(Frame* f, Val* v, Opcode op) switch ( v1->Type()->Tag() ) { case TYPE_VECTOR: - if ( ! v1->AsVectorVal()->Assign(v2, v, this, op) ) + if ( ! v1->AsVectorVal()->Assign(v2, v, op) ) Internal("assignment failed"); break; @@ -3620,7 +3617,7 @@ Val* VectorConstructorExpr::Eval(Frame* f) const { Expr* e = exprs[i]; Val* v = e->Eval(f); - if ( ! vec->Assign(i, v, e) ) + if ( ! vec->Assign(i, v) ) { Error(fmt("type mismatch at index %d", i), e); return 0; @@ -3644,7 +3641,7 @@ Val* VectorConstructorExpr::InitVal(const BroType* t, Val* aggr) const Expr* e = exprs[i]; Val* v = check_and_promote(e->Eval(0), t->YieldType(), 1); - if ( ! v || ! vec->Assign(i, v, e) ) + if ( ! v || ! vec->Assign(i, v) ) { Error(fmt("initialization type mismatch at index %d", i), e); return 0; @@ -3865,9 +3862,9 @@ Val* ArithCoerceExpr::Fold(Val* v) const { Val* elt = vv->Lookup(i); if ( elt ) - result->Assign(i, FoldSingleVal(elt, t), this); + result->Assign(i, FoldSingleVal(elt, t)); else - result->Assign(i, 0, this); + result->Assign(i, 0); } return result; @@ -5042,7 +5039,7 @@ Val* ListExpr::InitVal(const BroType* t, Val* aggr) const Expr* e = exprs[i]; check_and_promote_expr(e, vec->Type()->AsVectorType()->YieldType()); Val* v = e->Eval(0); - if ( ! vec->Assign(i, v, e) ) + if ( ! vec->Assign(i, v) ) { e->Error(fmt("type mismatch at index %d", i)); return 0; diff --git a/src/ICMP.cc b/src/ICMP.cc index b9b4e89404..5634e3885f 100644 --- a/src/ICMP.cc +++ b/src/ICMP.cc @@ -829,7 +829,7 @@ VectorVal* ICMP_Analyzer::BuildNDOptionsVal(int caplen, const u_char* data) data += length; caplen -= length; - vv->Assign(vv->Size(), rv, 0); + vv->Assign(vv->Size(), rv); } return vv; diff --git a/src/IP.cc b/src/IP.cc index 16424e26f2..78311cc2d2 100644 --- a/src/IP.cc +++ b/src/IP.cc @@ -63,7 +63,7 @@ static VectorVal* BuildOptionsVal(const u_char* data, int len) len -= opt->ip6o_len + off; } - vv->Assign(vv->Size(), rv, 0); + vv->Assign(vv->Size(), rv); } return vv; @@ -626,7 +626,7 @@ VectorVal* IPv6_Hdr_Chain::BuildVal() const reporter->InternalError("IPv6_Hdr_Chain bad header %d", type); break; } - rval->Assign(rval->Size(), ext_hdr, 0); + rval->Assign(rval->Size(), ext_hdr); } return rval; diff --git a/src/NFS.cc b/src/NFS.cc index 2911ee7f59..d64f7755a3 100644 --- a/src/NFS.cc +++ b/src/NFS.cc @@ -599,7 +599,7 @@ RecordVal* NFS_Interp::nfs3_readdir_reply(bool isplus, const u_char*& buf, entry->Assign(4, nfs3_post_op_fh(buf,n)); } - entries->Assign(pos, entry, 0); + entries->Assign(pos, entry); pos++; } diff --git a/src/SmithWaterman.cc b/src/SmithWaterman.cc index ef329e49a5..e051032756 100644 --- a/src/SmithWaterman.cc +++ b/src/SmithWaterman.cc @@ -96,12 +96,12 @@ VectorVal* BroSubstring::VecToPolicy(Vec* vec) align_val->Assign(0, new StringVal(new BroString(*align.string))); align_val->Assign(1, new Val(align.index, TYPE_COUNT)); - aligns->Assign(j+1, align_val, 0); + aligns->Assign(j+1, align_val); } st_val->Assign(1, aligns); st_val->Assign(2, new Val(bst->IsNewAlignment(), TYPE_BOOL)); - result->Assign(i+1, st_val, 0); + result->Assign(i+1, st_val); } } diff --git a/src/StateAccess.cc b/src/StateAccess.cc index 2d0a8dfc5a..c027624526 100644 --- a/src/StateAccess.cc +++ b/src/StateAccess.cc @@ -371,7 +371,7 @@ void StateAccess::Replay() CheckOld("index assign", target.id, op1.val, op3, v->AsVectorVal()->Lookup(index)); - v->AsVectorVal()->Assign(index, op2 ? op2->Ref() : 0, 0); + v->AsVectorVal()->Assign(index, op2 ? op2->Ref() : 0); } else @@ -421,7 +421,7 @@ void StateAccess::Replay() Val* lookup_op1 = v->AsVectorVal()->Lookup(index); int delta = lookup_op1->CoerceToInt() + amount; Val* new_val = new Val(delta, t); - v->AsVectorVal()->Assign(index, new_val, 0); + v->AsVectorVal()->Assign(index, new_val); } else diff --git a/src/TunnelEncapsulation.h b/src/TunnelEncapsulation.h index e8ca7a48b6..a43f7c0b57 100644 --- a/src/TunnelEncapsulation.h +++ b/src/TunnelEncapsulation.h @@ -186,7 +186,7 @@ public: if ( conns ) { for ( size_t i = 0; i < conns->size(); ++i ) - vv->Assign(i, (*conns)[i].GetRecordVal(), 0); + vv->Assign(i, (*conns)[i].GetRecordVal()); } return vv; diff --git a/src/Val.cc b/src/Val.cc index 20051aff5f..5133550236 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -2921,8 +2921,7 @@ VectorVal::~VectorVal() delete val.vector_val; } -bool VectorVal::Assign(unsigned int index, Val* element, const Expr* assigner, - Opcode op) +bool VectorVal::Assign(unsigned int index, Val* element, Opcode op) { if ( element && ! same_type(element->Type(), vector_type->YieldType(), 0) ) @@ -2983,12 +2982,12 @@ bool VectorVal::Assign(unsigned int index, Val* element, const Expr* assigner, } bool VectorVal::AssignRepeat(unsigned int index, unsigned int how_many, - Val* element, const Expr* assigner) + Val* element) { ResizeAtLeast(index + how_many); for ( unsigned int i = index; i < index + how_many; ++i ) - if ( ! Assign(i, element, assigner) ) + if ( ! Assign(i, element ) ) return false; return true; @@ -3089,7 +3088,7 @@ bool VectorVal::DoUnserialize(UnserialInfo* info) { Val* v; UNSERIALIZE_OPTIONAL(v, Val::Unserialize(info, TYPE_ANY)); - Assign(i, v, 0); + Assign(i, v); } return true; diff --git a/src/Val.h b/src/Val.h index 8ebfb2de0e..4b2705c5b4 100644 --- a/src/Val.h +++ b/src/Val.h @@ -968,18 +968,16 @@ public: // Note: does NOT Ref() the element! Remember to do so unless // the element was just created and thus has refcount 1. // - bool Assign(unsigned int index, Val* element, const Expr* assigner, - Opcode op = OP_ASSIGN); - bool Assign(Val* index, Val* element, const Expr* assigner, - Opcode op = OP_ASSIGN) + bool Assign(unsigned int index, Val* element, Opcode op = OP_ASSIGN); + bool Assign(Val* index, Val* element, Opcode op = OP_ASSIGN) { return Assign(index->AsListVal()->Index(0)->CoerceToUnsigned(), - element, assigner, op); + element, op); } // Assigns the value to how_many locations starting at index. bool AssignRepeat(unsigned int index, unsigned int how_many, - Val* element, const Expr* assigner); + Val* element); // Returns nil if no element was at that value. // Lookup does NOT grow the vector to this size. diff --git a/src/bro.bif b/src/bro.bif index 8cea9d9123..79161ac47a 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -1349,7 +1349,7 @@ function order%(v: any, ...%) : index_vec for ( i = 0; i < n; ++i ) { int ind = ind_vv[i]; - result_v->Assign(i, new Val(ind, TYPE_COUNT), 0); + result_v->Assign(i, new Val(ind, TYPE_COUNT)); } return result_v; @@ -1633,7 +1633,7 @@ function record_type_to_vector%(rt: string%): string_vec for ( int i = 0; i < type->NumFields(); ++i ) { StringVal* val = new StringVal(type->FieldName(i)); - result->Assign(i+1, val, 0); + result->Assign(i+1, val); } } @@ -2139,7 +2139,7 @@ function routing0_data_to_addrs%(s: string%): addr_vec while ( len > 0 ) { IPAddr a(IPv6, (const uint32*) bytes, IPAddr::Network); - rval->Assign(rval->Size(), new AddrVal(a), 0); + rval->Assign(rval->Size(), new AddrVal(a)); bytes += 16; len -= 16; } @@ -2162,7 +2162,7 @@ function addr_to_counts%(a: addr%): index_vec int len = a->AsAddr().GetBytes(&bytes); for ( int i = 0; i < len; ++i ) - rval->Assign(i, new Val(ntohl(bytes[i]), TYPE_COUNT), 0); + rval->Assign(i, new Val(ntohl(bytes[i]), TYPE_COUNT)); return rval; %} diff --git a/src/input/Manager.cc b/src/input/Manager.cc index d9006d66a2..a6df9533e7 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -2107,7 +2107,7 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type) VectorType* vt = new VectorType(type->Ref()); VectorVal* v = new VectorVal(vt); for ( int i = 0; i < val->val.vector_val.size; i++ ) - v->Assign(i, ValueToVal( val->val.set_val.vals[i], type ), 0); + v->Assign(i, ValueToVal( val->val.set_val.vals[i], type )); Unref(vt); return v; diff --git a/src/modbus-analyzer.pac b/src/modbus-analyzer.pac index b03df9d188..a9c773b9e9 100644 --- a/src/modbus-analyzer.pac +++ b/src/modbus-analyzer.pac @@ -149,7 +149,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i=0; i < ${message.registers}->size(); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_holding_registers_response(connection()->bro_analyzer(), @@ -192,7 +192,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i=0; i < (${message.registers})->size(); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_input_registers_response(connection()->bro_analyzer(), @@ -335,7 +335,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < (${message.registers}->size()); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_write_multiple_registers_request(connection()->bro_analyzer(), @@ -371,13 +371,13 @@ refine flow ModbusTCP_Flow += { //for ( unsigned int i = 0; i < (${message.references}->size()); ++i ) // { // Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // // Val* k = new Val((${message.references[i].file_num}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // // Val* l = new Val((${message.references[i].record_num}), TYPE_COUNT); - // t->Assign(i, l, 0, OP_ASSIGN); + // t->Assign(i, l); // } BifEvent::generate_modbus_read_file_record_request(connection()->bro_analyzer(), @@ -398,7 +398,7 @@ refine flow ModbusTCP_Flow += { // { // //TODO: work the reference type in here somewhere // Val* r = new Val(${message.references[i].record_data}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // } BifEvent::generate_modbus_read_file_record_response(connection()->bro_analyzer(), @@ -418,18 +418,18 @@ refine flow ModbusTCP_Flow += { //for ( unsigned int i = 0; i < (${message.references}->size()); ++i ) // { // Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // // Val* k = new Val((${message.references[i].file_num}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // // Val* n = new Val((${message.references[i].record_num}), TYPE_COUNT); - // t->Assign(i, n, 0, OP_ASSIGN); + // t->Assign(i, n); // // for ( unsigned int j = 0; j < (${message.references[i].register_value}->size()); ++j ) // { // k = new Val((${message.references[i].register_value[j]}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // } // } @@ -451,18 +451,18 @@ refine flow ModbusTCP_Flow += { //for ( unsigned int i = 0; i < (${messages.references}->size()); ++i ) // { // Val* r = new Val((${message.references[i].ref_type}), TYPE_COUNT); - // t->Assign(i, r, 0, OP_ASSIGN); + // t->Assign(i, r); // // Val* f = new Val((${message.references[i].file_num}), TYPE_COUNT); - // t->Assign(i, f, 0, OP_ASSIGN); + // t->Assign(i, f); // // Val* rn = new Val((${message.references[i].record_num}), TYPE_COUNT); - // t->Assign(i, rn, 0, OP_ASSIGN); + // t->Assign(i, rn); // // for ( unsigned int j = 0; j<(${message.references[i].register_value}->size()); ++j ) // { // Val* k = new Val((${message.references[i].register_value[j]}), TYPE_COUNT); - // t->Assign(i, k, 0, OP_ASSIGN); + // t->Assign(i, k); // } BifEvent::generate_modbus_write_file_record_response(connection()->bro_analyzer(), @@ -519,7 +519,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < ${message.write_register_values}->size(); ++i ) { Val* r = new Val(${message.write_register_values[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_write_multiple_registers_request(connection()->bro_analyzer(), @@ -550,7 +550,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < ${message.registers}->size(); ++i ) { Val* r = new Val(${message.registers[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_write_multiple_registers_response(connection()->bro_analyzer(), @@ -593,7 +593,7 @@ refine flow ModbusTCP_Flow += { for ( unsigned int i = 0; i < (${message.register_data})->size(); ++i ) { Val* r = new Val(${message.register_data[i]}, TYPE_COUNT); - t->Assign(i, r, 0, OP_ASSIGN); + t->Assign(i, r); } BifEvent::generate_modbus_read_fifo_queue_response(connection()->bro_analyzer(), diff --git a/src/strings.bif b/src/strings.bif index dc5e064dc6..55e2e79c7f 100644 --- a/src/strings.bif +++ b/src/strings.bif @@ -866,7 +866,7 @@ function str_split%(s: string, idx: index_vec%): string_vec for ( BroString::VecIt it = result->begin(); it != result->end(); ++it, ++i ) - result_v->Assign(i, new StringVal(*it), 0); + result_v->Assign(i, new StringVal(*it)); // StringVal now possesses string. delete result; From a005d773691a3df49e8a481097c1a06fac8fec9c Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Wed, 6 Mar 2013 14:14:55 -0800 Subject: [PATCH 41/49] forgot to remove the baselines for the now unnecessary bifs --- testing/btest/Baseline/bifs.byte_len/out | 1 - testing/btest/Baseline/bifs.length/out | 6 ------ 2 files changed, 7 deletions(-) delete mode 100644 testing/btest/Baseline/bifs.byte_len/out delete mode 100644 testing/btest/Baseline/bifs.length/out diff --git a/testing/btest/Baseline/bifs.byte_len/out b/testing/btest/Baseline/bifs.byte_len/out deleted file mode 100644 index b4de394767..0000000000 --- a/testing/btest/Baseline/bifs.byte_len/out +++ /dev/null @@ -1 +0,0 @@ -11 diff --git a/testing/btest/Baseline/bifs.length/out b/testing/btest/Baseline/bifs.length/out deleted file mode 100644 index ad43182650..0000000000 --- a/testing/btest/Baseline/bifs.length/out +++ /dev/null @@ -1,6 +0,0 @@ -1 -4 -2 -0 -0 -0 From 8ee4382721f32660bf038df16dc75f2eaa6a05b1 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 6 Mar 2013 18:32:43 -0800 Subject: [PATCH 42/49] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 2fd9086c9d..09ff521f98 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 2fd9086c9dc0e76f6ff1ae04a60cbbce60507aab +Subproject commit 09ff521f9804a711a59e64192eb110286b56ae23 diff --git a/aux/bro-aux b/aux/bro-aux index bea556198b..a9900d03a0 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit bea556198b69d30d64c0cf1b594e6de71176df6f +Subproject commit a9900d03a0bcc7f5e94999e63068cd36118f9fac diff --git a/aux/broccoli b/aux/broccoli index c1ba9b44c4..6a7ce301ba 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit c1ba9b44c4815c61c54c968f462ec5b0865e5990 +Subproject commit 6a7ce301bab11bfeabf7c03e0a236c8c1f85181c diff --git a/aux/broctl b/aux/broctl index 2bf6b37177..90a400410c 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2bf6b37177b895329173acac2bb98f38a8783bc1 +Subproject commit 90a400410c3c22e579f9ee0d23881c6f25bd259d diff --git a/cmake b/cmake index 14537f56d6..599c3fa916 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 14537f56d66b18ab9d5024f798caf4d1f356fc67 +Subproject commit 599c3fa916a8d746a535666955ab4439bea20471 From 2293443ea0ceec8fca018556f41646fdb8da45d0 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Mar 2013 13:02:33 -0600 Subject: [PATCH 43/49] Fix function type-equivalence requiring same param names, addresses #957 --- src/Type.cc | 6 +-- src/Type.h | 6 +-- .../Baseline/language.func-assignment/out | 4 ++ testing/btest/language/func-assignment.bro | 39 +++++++++++++++++++ 4 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 testing/btest/Baseline/language.func-assignment/out create mode 100644 testing/btest/language/func-assignment.bro diff --git a/src/Type.cc b/src/Type.cc index 6ac6070660..db6e940e87 100644 --- a/src/Type.cc +++ b/src/Type.cc @@ -712,7 +712,7 @@ int FuncType::MatchesIndex(ListExpr*& index) const MATCHES_INDEX_SCALAR : DOES_NOT_MATCH_INDEX; } -int FuncType::CheckArgs(const type_list* args) const +int FuncType::CheckArgs(const type_list* args, bool is_init) const { const type_list* my_args = arg_types->Types(); @@ -720,7 +720,7 @@ int FuncType::CheckArgs(const type_list* args) const return 0; for ( int i = 0; i < my_args->length(); ++i ) - if ( ! same_type((*args)[i], (*my_args)[i]) ) + if ( ! same_type((*args)[i], (*my_args)[i], is_init) ) return 0; return 1; @@ -1722,7 +1722,7 @@ int same_type(const BroType* t1, const BroType* t2, int is_init) return 0; } - return same_type(ft1->Args(), ft2->Args(), is_init); + return ft1->CheckArgs(ft2->ArgTypes()->Types(), is_init); } case TYPE_RECORD: diff --git a/src/Type.h b/src/Type.h index 249d8709c5..0b7620cd68 100644 --- a/src/Type.h +++ b/src/Type.h @@ -370,11 +370,9 @@ public: { Unref(yield); yield = 0; flavor = arg_flav; } int MatchesIndex(ListExpr*& index) const; - int CheckArgs(const type_list* args) const; + int CheckArgs(const type_list* args, bool is_init = false) const; - TypeList* ArgTypes() { return arg_types; } - - ID* GetReturnValueID() const; + TypeList* ArgTypes() const { return arg_types; } void Describe(ODesc* d) const; void DescribeReST(ODesc* d) const; diff --git a/testing/btest/Baseline/language.func-assignment/out b/testing/btest/Baseline/language.func-assignment/out new file mode 100644 index 0000000000..b569f7ee55 --- /dev/null +++ b/testing/btest/Baseline/language.func-assignment/out @@ -0,0 +1,4 @@ +Brogrammers, like bowties, are cool. Brogrammers, like bowties, are cool. Brogrammers, like bowties, are cool. +Brogrammers, like bowties, are cool. Brogrammers, like bowties, are cool. +BROGRAMMERS, LIKE BOWTIES, ARE COOL. +BROGRAMMERS, LIKE BOWTIES, ARE COOL. diff --git a/testing/btest/language/func-assignment.bro b/testing/btest/language/func-assignment.bro new file mode 100644 index 0000000000..576d7f3270 --- /dev/null +++ b/testing/btest/language/func-assignment.bro @@ -0,0 +1,39 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +function double_string(s: string): string + { + return string_cat(s, " ", s); + } + +function triple_string(str: string): string + { + return string_cat(str, " ", str, " ", str); + } + +type sample_function: record { + s: string; + f: function(str: string): string; +}; + +event bro_init() + { + local test_sf: sample_function; + test_sf$s = "Brogrammers, like bowties, are cool."; + + test_sf$f = triple_string; + print test_sf$f(test_sf$s); + + test_sf$f = double_string; + print test_sf$f(test_sf$s); + + # Works as expected + test_sf$f = function(str: string): string + { return to_upper(str); }; + print test_sf$f(test_sf$s); + + # Func arg names shouldn't factor in to the type check. + test_sf$f = function(s: string): string + { return to_upper(s); }; + print test_sf$f(test_sf$s); + } From f4d59f8137865dfa4357b8a7a91c2cf6c816557a Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Mar 2013 14:41:18 -0600 Subject: [PATCH 44/49] Fix compiler warnings. --- src/SerialObj.cc | 2 +- src/patricia.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/SerialObj.cc b/src/SerialObj.cc index 73cab275c2..ab7f63e823 100644 --- a/src/SerialObj.cc +++ b/src/SerialObj.cc @@ -155,7 +155,7 @@ SerialObj* SerialObj::Unserialize(UnserialInfo* info, SerialType type) else { // Broccoli compatibility mode with 32bit pids. - uint32 tmp; + uint32 tmp = 0; result = UNSERIALIZE(&full_obj) && UNSERIALIZE(&tmp); pid = tmp; } diff --git a/src/patricia.c b/src/patricia.c index 6998576fdb..c4815b40ec 100644 --- a/src/patricia.c +++ b/src/patricia.c @@ -336,7 +336,7 @@ ascii2prefix (int family, char *string) memcpy (save, string, cp - string); save[cp - string] = '\0'; string = save; - if (bitlen < 0 || bitlen > maxbitlen) + if (bitlen > maxbitlen) bitlen = maxbitlen; } else { From 7e4963b22ca3b6ebd8c23e36f5f15d9cacbd974f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 7 Mar 2013 14:41:51 -0600 Subject: [PATCH 45/49] Fix new[]/delete mismatch in RE.cc reported by jbaines, addresses #958. --- src/RE.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/RE.cc b/src/RE.cc index b6f1a1361f..87117c1c3a 100644 --- a/src/RE.cc +++ b/src/RE.cc @@ -496,7 +496,7 @@ static RE_Matcher* matcher_merge(const RE_Matcher* re1, const RE_Matcher* re2, safe_snprintf(merge_text, n, "(%s)%s(%s)", text1, merge_op, text2); RE_Matcher* merge = new RE_Matcher(merge_text); - delete merge_text; + delete [] merge_text; merge->Compile(); From f830ed3edf123ae6e07e18a5d33c356a8694b30a Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Mar 2013 19:16:58 -0800 Subject: [PATCH 46/49] s/bro-ids.org/bro.org/g --- INSTALL | 12 +++++------ NEWS | 4 ++-- README | 2 +- doc/_templates/layout.html | 4 ++-- doc/index.rst | 2 +- doc/quickstart.rst | 2 +- doc/reporting-problems.rst | 2 +- doc/upgrade.rst | 4 ++-- scripts/base/frameworks/cluster/main.bro | 2 +- src/bro.bif | 4 ++-- src/logging/writers/DataSeries.cc | 2 +- src/netflow-analyzer.pac | 2 +- src/util.h | 2 +- .../ssh.ds.xml | 2 +- .../out | 20 +++++++++---------- .../ssh.ds.txt | 2 +- .../conn.ds.txt | 2 +- .../conn.ds.txt | 2 +- .../http.ds.txt | 2 +- testing/btest/README | 4 ++-- 20 files changed, 39 insertions(+), 39 deletions(-) diff --git a/INSTALL b/INSTALL index 084ca9dc2e..4604a26bf5 100644 --- a/INSTALL +++ b/INSTALL @@ -4,7 +4,7 @@ .. _MacPorts: http://www.macports.org .. _Fink: http://www.finkproject.org .. _Homebrew: http://mxcl.github.com/homebrew -.. _bro downloads page: http://bro-ids.org/download/index.html +.. _bro downloads page: http://bro.org/download/index.html ============== Installing Bro @@ -189,15 +189,15 @@ Bro releases are bundled into source packages for convenience and available from the `bro downloads page`_. Alternatively, the latest Bro development version can be obtained through git -repositories hosted at `git.bro-ids.org `_. See +repositories hosted at `git.bro.org `_. See our `git development documentation -`_ for comprehensive +`_ for comprehensive information on Bro's use of git revision control, but the short story for downloading the full source code experience for Bro via git is: .. console:: - git clone --recursive git://git.bro-ids.org/bro + git clone --recursive git://git.bro.org/bro .. note:: If you choose to clone the ``bro`` repository non-recursively for a "minimal Bro experience", be aware that compiling it depends on @@ -230,7 +230,7 @@ automatically. Finally, use ``make install-aux`` to install some of the other programs that are in the ``aux/bro-aux`` directory. OpenBSD users, please see our FAQ at -http://www.bro-ids.org/documentation/faq.html if you are having +http://www.bro.org/documentation/faq.html if you are having problems installing Bro. @@ -298,7 +298,7 @@ Running Bro Bro is a complex program and it takes a bit of time to get familiar with it. A good place for newcomers to start is the Quick Start Guide -at http://www.bro-ids.org/documentation/quickstart.html. +at http://www.bro.org/documentation/quickstart.html. For developers that wish to run Bro directly from the ``build/`` directory (i.e., without performing ``make install``), they will have diff --git a/NEWS b/NEWS index 59ab51bc9c..8605dcdbd4 100644 --- a/NEWS +++ b/NEWS @@ -292,7 +292,7 @@ Bro 2.0 As the version number jump suggests, Bro 2.0 is a major upgrade and lots of things have changed. We have assembled a separate upgrade guide with the most important changes compared to Bro 1.5 at -http://www.bro-ids.org/documentation/upgrade.html. You can find +http://www.bro.org/documentation/upgrade.html. You can find the offline version of that document in ``doc/upgrade.rst.``. Compared to the earlier 2.0 Beta version, the major changes in the @@ -300,7 +300,7 @@ final release are: * The default scripts now come with complete reference documentation. See - http://www.bro-ids.org/documentation/index.html. + http://www.bro.org/documentation/index.html. * libz and libmagic are now required dependencies. diff --git a/README b/README index c837afaf92..734246c914 100644 --- a/README +++ b/README @@ -11,7 +11,7 @@ Please see COPYING for licensing information. For more documentation, research publications, and community contact information, please see Bro's home page: - http://www.bro-ids.org + http://www.bro.org On behalf of the Bro Development Team, diff --git a/doc/_templates/layout.html b/doc/_templates/layout.html index 77d9d1de1c..2f8ea02aff 100644 --- a/doc/_templates/layout.html +++ b/doc/_templates/layout.html @@ -10,7 +10,7 @@ {% endblock %} {% block header %} - {% endblock %} @@ -108,6 +108,6 @@ {% endblock %} {% block footer %} - {% endblock %} diff --git a/doc/index.rst b/doc/index.rst index cea72d1759..29b29541b4 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -53,7 +53,7 @@ Other Bro Components The following are snapshots of documentation for components that come with this version of Bro (|version|). Since they can also be used independently, see the `download page -`_ for documentation of any +`_ for documentation of any current, independent component releases. .. toctree:: diff --git a/doc/quickstart.rst b/doc/quickstart.rst index 89c9be5a87..9f64e36d89 100644 --- a/doc/quickstart.rst +++ b/doc/quickstart.rst @@ -111,7 +111,7 @@ protocol-dependent activity that's occurring. E.g. ``http.log``'s next few columns (shortened for brevity) show a request to the root of Bro website:: # method host uri referrer user_agent - GET bro-ids.org / - <...>Chrome/12.0.742.122<...> + GET bro.org / - <...>Chrome/12.0.742.122<...> Some logs are worth explicit mention: diff --git a/doc/reporting-problems.rst b/doc/reporting-problems.rst index 5e55b2ac90..903df76257 100644 --- a/doc/reporting-problems.rst +++ b/doc/reporting-problems.rst @@ -19,7 +19,7 @@ Reporting Problems Generally, when you encounter a problem with Bro, the best thing to do is opening a new ticket in `Bro's issue tracker -`__ and include information on how to +`__ and include information on how to reproduce the issue. Ideally, your ticket should come with the following: diff --git a/doc/upgrade.rst b/doc/upgrade.rst index 9ea006f488..539757537d 100644 --- a/doc/upgrade.rst +++ b/doc/upgrade.rst @@ -254,7 +254,7 @@ Variable Naming - Identifiers may have been renamed to conform to new `scripting conventions - `_ + `_ BroControl @@ -296,7 +296,7 @@ Development Infrastructure Bro development has moved from using SVN to Git for revision control. Users that want to use the latest Bro development snapshot by checking it out from the source repositories should see the `development process -`_. Note that all the various +`_. Note that all the various sub-components now reside in their own repositories. However, the top-level Bro repository includes them as git submodules so it's easy to check them all out simultaneously. diff --git a/scripts/base/frameworks/cluster/main.bro b/scripts/base/frameworks/cluster/main.bro index 766dea912f..4184ad6ded 100644 --- a/scripts/base/frameworks/cluster/main.bro +++ b/scripts/base/frameworks/cluster/main.bro @@ -39,7 +39,7 @@ export { ## The node type doing all the actual traffic analysis. WORKER, ## A node acting as a traffic recorder using the - ## `Time Machine `_ software. + ## `Time Machine `_ software. TIME_MACHINE, }; diff --git a/src/bro.bif b/src/bro.bif index d275dedc30..014c582ec9 100644 --- a/src/bro.bif +++ b/src/bro.bif @@ -3535,7 +3535,7 @@ function lookup_addr%(host: addr%) : string ## Issues an asynchronous TEXT DNS lookup and delays the function result. ## This function can therefore only be called inside a ``when`` condition, -## e.g., ``when ( local h = lookup_hostname_txt("www.bro-ids.org") ) { f(h); }``. +## e.g., ``when ( local h = lookup_hostname_txt("www.bro.org") ) { f(h); }``. ## ## host: The hostname to lookup. ## @@ -3564,7 +3564,7 @@ function lookup_hostname_txt%(host: string%) : string ## Issues an asynchronous DNS lookup and delays the function result. ## This function can therefore only be called inside a ``when`` condition, -## e.g., ``when ( local h = lookup_hostname("www.bro-ids.org") ) { f(h); }``. +## e.g., ``when ( local h = lookup_hostname("www.bro.org") ) { f(h); }``. ## ## host: The hostname to lookup. ## diff --git a/src/logging/writers/DataSeries.cc b/src/logging/writers/DataSeries.cc index 27e021ba2c..bc15c6f5b9 100644 --- a/src/logging/writers/DataSeries.cc +++ b/src/logging/writers/DataSeries.cc @@ -167,7 +167,7 @@ string DataSeries::BuildDSSchemaFromFieldTypes(const vector& vals, string xmlschema = "\n"; + + "\" version=\"1.0\" namespace=\"bro.org\">\n"; for( size_t i = 0; i < vals.size(); ++i ) { diff --git a/src/netflow-analyzer.pac b/src/netflow-analyzer.pac index e89a0181a2..666de076c8 100644 --- a/src/netflow-analyzer.pac +++ b/src/netflow-analyzer.pac @@ -38,7 +38,7 @@ flow NetFlow_Flow { # %cleanup does not only put the cleanup code into the destructor, # but also at the end of the catch clause in NewData(). This is # different from the documentation at - # http://www.bro-ids.org/wiki/index.php/BinPAC_Userguide#.25cleanup.7B....25.7D + # http://www.bro.org/wiki/index.php/BinPAC_Userguide#.25cleanup.7B....25.7D # # Unfortunately this means that we cannot clean up the identifier # string. Note that IOSource destructors seemingly are never diff --git a/src/util.h b/src/util.h index 5d4115773d..f717ecd333 100644 --- a/src/util.h +++ b/src/util.h @@ -76,7 +76,7 @@ typedef int32 ptr_compat_int; #define PRI_PTR_COMPAT_INT PRId32 #define PRI_PTR_COMPAT_UINT PRIu32 #else -# error "Unusual pointer size. Please report to bro@bro-ids.org." +# error "Unusual pointer size. Please report to bro@bro.org." #endif extern "C" diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml index cacc3b0ea4..a1e65c254e 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.options/ssh.ds.xml @@ -1,4 +1,4 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out index 1e5e1b05c6..94f25c37f4 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.rotate/out @@ -19,7 +19,7 @@ test.2011-03-07-12-00-05.ds test 11-03-07_12.00.05 11-03-07_12.59.55 1 dataserie - + @@ -47,7 +47,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -75,7 +75,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -103,7 +103,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -131,7 +131,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -159,7 +159,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -187,7 +187,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -215,7 +215,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -243,7 +243,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + @@ -271,7 +271,7 @@ t id.orig_h id.orig_p id.resp_h id.resp_p - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt index e6abc3f1f6..225217faea 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.test-logging/ssh.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt index 5d95fdc0c0..bd3d367632 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.time-as-int/conn.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt index 5af6f702b8..a85ae29346 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/conn.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt index 756a49a427..bea50e6aba 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.dataseries.wikipedia/http.ds.txt @@ -8,7 +8,7 @@ - + diff --git a/testing/btest/README b/testing/btest/README index 6e8abd24ed..a635251939 100644 --- a/testing/btest/README +++ b/testing/btest/README @@ -1,7 +1,7 @@ This a test suite of small "unit tests" that verify individual pieces of Bro functionality. They all utilize BTest, a simple framework/driver for writing unit tests. More information about BTest can be found at -http://www.bro-ids.org/development/btest.html +http://www.bro.org/development/btest.html The test suite's BTest configuration is handled through the ``btest.cfg`` file. Of particular interest is the "TestDirs" settings, @@ -61,7 +61,7 @@ Adding Tests ============= See either the `BTest documentation -`_ or the existing unit +`_ or the existing unit tests for examples of what they actually look like. The essential components of a new test include: From 74a529d9378075c29626e66285097d3a1c4b12e8 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 7 Mar 2013 19:33:19 -0800 Subject: [PATCH 47/49] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- aux/btest | 2 +- cmake | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/aux/binpac b/aux/binpac index 09ff521f98..72d121ade5 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 09ff521f9804a711a59e64192eb110286b56ae23 +Subproject commit 72d121ade5a37df83d3252646de51cb77ce69a89 diff --git a/aux/bro-aux b/aux/bro-aux index a9900d03a0..ae14da422b 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit a9900d03a0bcc7f5e94999e63068cd36118f9fac +Subproject commit ae14da422bfb252c8a53bd00d3e5fd7da8bc112e diff --git a/aux/broccoli b/aux/broccoli index 6a7ce301ba..e64204fec5 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 6a7ce301bab11bfeabf7c03e0a236c8c1f85181c +Subproject commit e64204fec55759c614a276c1933bbff2069a63db diff --git a/aux/broctl b/aux/broctl index 90a400410c..5eb828fb8e 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 90a400410c3c22e579f9ee0d23881c6f25bd259d +Subproject commit 5eb828fb8e2bc5cb250901597beaedf2cf524f03 diff --git a/aux/btest b/aux/btest index ba0700fe44..d5b8df42cb 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit ba0700fe448895b654b90d50f389f6f1341234cb +Subproject commit d5b8df42cb9c398142e02d4bf8ede835fd0227f4 diff --git a/cmake b/cmake index 599c3fa916..94e72a3075 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 599c3fa916a8d746a535666955ab4439bea20471 +Subproject commit 94e72a3075bb0b9550ad05758963afda394bfb2c From 1fb05da9cd7df317fb758ba24c3b4edb47e5651f Mon Sep 17 00:00:00 2001 From: Bernhard Amann Date: Thu, 7 Mar 2013 20:28:18 -0800 Subject: [PATCH 48/49] Fix race-condition in table-event test. Event depended on the input manager receiving all lines from the reader before the first input event was processed by the scripting layer. --- .../out | 168 ------------------ .../base/frameworks/input/tableevent.bro | 1 - 2 files changed, 169 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out index d76c63ef31..54048a86b8 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out +++ b/testing/btest/Baseline/scripts.base.frameworks.input.tableevent/out @@ -1,189 +1,21 @@ -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=1] T -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=2] T -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=3] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=4] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=5] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=6] F -[source=../input.log, reader=Input::READER_ASCII, mode=Input::MANUAL, name=input, destination={ -[2] = T, -[4] = F, -[6] = F, -[7] = T, -[1] = T, -[5] = F, -[3] = F -}, idx=, val=, want_record=F, ev=line -{ -print outfile, description; -print outfile, tpe; -print outfile, left; -print outfile, right; -try = try + 1; -if (7 == try) -{ -close(outfile); -terminate(); -} - -}, pred=, config={ - -}] Input::EVENT_NEW [i=7] T diff --git a/testing/btest/scripts/base/frameworks/input/tableevent.bro b/testing/btest/scripts/base/frameworks/input/tableevent.bro index a409803440..c83b1361c1 100644 --- a/testing/btest/scripts/base/frameworks/input/tableevent.bro +++ b/testing/btest/scripts/base/frameworks/input/tableevent.bro @@ -35,7 +35,6 @@ global destination: table[int] of Val = table(); event line(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: bool) { - print outfile, description; print outfile, tpe; print outfile, left; print outfile, right; From 0075973249906ce1374948b567d261395f99220e Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 8 Mar 2013 09:41:35 -0800 Subject: [PATCH 49/49] Updating submodule(s). [nomail] --- aux/broctl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/broctl b/aux/broctl index 5eb828fb8e..3d2172a60a 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 5eb828fb8e2bc5cb250901597beaedf2cf524f03 +Subproject commit 3d2172a60aa503745c92cef8ab3020d1dfc13f0d