From 22aa821506b094134cb814a0c898bea8d7632649 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 31 Jul 2014 10:49:33 -0500 Subject: [PATCH 01/71] Split the types and attributes reference doc into two docs Also moved them up in the index so that the more fundamental material comes before the more advanced material in the table of contents. --- doc/script-reference/attributes.rst | 139 +++++++++++++++++ doc/script-reference/index.rst | 3 +- .../{builtins.rst => types.rst} | 144 +----------------- 3 files changed, 142 insertions(+), 144 deletions(-) create mode 100644 doc/script-reference/attributes.rst rename doc/script-reference/{builtins.rst => types.rst} (84%) diff --git a/doc/script-reference/attributes.rst b/doc/script-reference/attributes.rst new file mode 100644 index 0000000000..ca66ab2112 --- /dev/null +++ b/doc/script-reference/attributes.rst @@ -0,0 +1,139 @@ +Attributes +========== + +Attributes occur at the end of type or event declarations and change their +behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T: +set[count] &read_expire=5min`` or ``event foo() &priority=-3``. The Bro +scripting language supports the following attributes. + +.. bro:attr:: &optional + + Allows a record field to be missing. For example the type ``record { + a: addr; b: port &optional; }`` could be instantiated both as + singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``. + +.. bro:attr:: &default + + Uses a default value for a record field, a function/hook/event + parameter, or container elements. For example, ``table[int] of + string &default="foo"`` would create a table that returns the + :bro:type:`string` ``"foo"`` for any non-existing index. + +.. bro:attr:: &redef + + Allows for redefinition of initial object values. This is typically + used with constants, for example, ``const clever = T &redef;`` would + allow the constant to be redefined at some later point during script + execution. + +.. bro:attr:: &rotate_interval + + Rotates a file after a specified interval. + +.. bro:attr:: &rotate_size + + Rotates a file after it has reached a given size in bytes. + +.. bro:attr:: &add_func + + Can be applied to an identifier with &redef to specify a function to + be called any time a "redef += ..." declaration is parsed. The + function takes two arguments of the same type as the identifier, the first + being the old value of the variable and the second being the new + value given after the "+=" operator in the "redef" declaration. The + return value of the function will be the actual new value of the + variable after the "redef" declaration is parsed. + +.. bro:attr:: &delete_func + + Same as &add_func, except for "redef" declarations that use the "-=" + operator. + +.. bro:attr:: &expire_func + + Called right before a container element expires. The function's + first parameter is of the same type of the container and the second + parameter the same type of the container's index. The return + value is an :bro:type:`interval` indicating the amount of additional + time to wait before expiring the container element at the given + index (which will trigger another execution of this function). + +.. bro:attr:: &read_expire + + Specifies a read expiration timeout for container elements. That is, + the element expires after the given amount of time since the last + time it has been read. Note that a write also counts as a read. + +.. bro:attr:: &write_expire + + Specifies a write expiration timeout for container elements. That + is, the element expires after the given amount of time since the + last time it has been written. + +.. bro:attr:: &create_expire + + Specifies a creation expiration timeout for container elements. That + is, the element expires after the given amount of time since it has + been inserted into the container, regardless of any reads or writes. + +.. bro:attr:: &persistent + + Makes a variable persistent, i.e., its value is written to disk (per + default at shutdown time). + +.. bro:attr:: &synchronized + + Synchronizes variable accesses across nodes. The value of a + ``&synchronized`` variable is automatically propagated to all peers + when it changes. + +.. bro:attr:: &encrypt + + Encrypts files right before writing them to disk. + +.. TODO: needs to be documented in more detail. + +.. bro:attr:: &raw_output + + Opens a file in raw mode, i.e., non-ASCII characters are not + escaped. + +.. bro:attr:: &mergeable + + Prefers set union to assignment for synchronized state. This + attribute is used in conjunction with :bro:attr:`&synchronized` + container types: when the same container is updated at two peers + with different value, the propagation of the state causes a race + condition, where the last update succeeds. This can cause + inconsistencies and can be avoided by unifying the two sets, rather + than merely overwriting the old value. + +.. bro:attr:: &priority + + Specifies the execution priority (as a signed integer) of a hook or + event handler. Higher values are executed before lower ones. The + default value is 0. + +.. bro:attr:: &group + + Groups event handlers such that those in the same group can be + jointly activated or deactivated. + +.. bro:attr:: &log + + Writes a record field to the associated log stream. + +.. bro:attr:: &error_handler + + Internally set on the events that are associated with the reporter + framework: :bro:id:`reporter_info`, :bro:id:`reporter_warning`, and + :bro:id:`reporter_error`. It prevents any handlers of those events + from being able to generate reporter messages that go through any of + those events (i.e., it prevents an infinite event recursion). Instead, + such nested reporter messages are output to stderr. + +.. bro:attr:: &type_column + + Used by the input framework. It can be used on columns of type + :bro:type:`port` and specifies the name of an additional column in + the input file which specifies the protocol of the port (tcp/udp/icmp). diff --git a/doc/script-reference/index.rst b/doc/script-reference/index.rst index bd600e4a97..a2c6f0a24f 100644 --- a/doc/script-reference/index.rst +++ b/doc/script-reference/index.rst @@ -5,10 +5,11 @@ Script Reference .. toctree:: :maxdepth: 1 + types + attributes notices proto-analyzers file-analyzers - builtins packages scripts Broxygen Example Script diff --git a/doc/script-reference/builtins.rst b/doc/script-reference/types.rst similarity index 84% rename from doc/script-reference/builtins.rst rename to doc/script-reference/types.rst index 85e9cd14c8..049b43c04a 100644 --- a/doc/script-reference/builtins.rst +++ b/doc/script-reference/types.rst @@ -1,8 +1,5 @@ -Types and Attributes -==================== - Types ------ +===== Every value in a Bro script has a type (see below for a list of all built-in types). Although Bro variables have static types (meaning that their type @@ -859,142 +856,3 @@ The Bro scripting language supports the following built-in types. executed due to one handler body exiting as a result of a ``break`` statement. -Attributes ----------- - -Attributes occur at the end of type/event declarations and change their -behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T: -set[count] &read_expire=5min`` or ``event foo() &priority=-3``. The Bro -scripting language supports the following built-in attributes. - -.. bro:attr:: &optional - - Allows a record field to be missing. For example the type ``record { - a: addr; b: port &optional; }`` could be instantiated both as - singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``. - -.. bro:attr:: &default - - Uses a default value for a record field, a function/hook/event - parameter, or container elements. For example, ``table[int] of - string &default="foo"`` would create a table that returns the - :bro:type:`string` ``"foo"`` for any non-existing index. - -.. bro:attr:: &redef - - Allows for redefinition of initial object values. This is typically - used with constants, for example, ``const clever = T &redef;`` would - allow the constant to be redefined at some later point during script - execution. - -.. bro:attr:: &rotate_interval - - Rotates a file after a specified interval. - -.. bro:attr:: &rotate_size - - Rotates a file after it has reached a given size in bytes. - -.. bro:attr:: &add_func - - Can be applied to an identifier with &redef to specify a function to - be called any time a "redef += ..." declaration is parsed. The - function takes two arguments of the same type as the identifier, the first - being the old value of the variable and the second being the new - value given after the "+=" operator in the "redef" declaration. The - return value of the function will be the actual new value of the - variable after the "redef" declaration is parsed. - -.. bro:attr:: &delete_func - - Same as &add_func, except for "redef" declarations that use the "-=" - operator. - -.. bro:attr:: &expire_func - - Called right before a container element expires. The function's - first parameter is of the same type of the container and the second - parameter the same type of the container's index. The return - value is an :bro:type:`interval` indicating the amount of additional - time to wait before expiring the container element at the given - index (which will trigger another execution of this function). - -.. bro:attr:: &read_expire - - Specifies a read expiration timeout for container elements. That is, - the element expires after the given amount of time since the last - time it has been read. Note that a write also counts as a read. - -.. bro:attr:: &write_expire - - Specifies a write expiration timeout for container elements. That - is, the element expires after the given amount of time since the - last time it has been written. - -.. bro:attr:: &create_expire - - Specifies a creation expiration timeout for container elements. That - is, the element expires after the given amount of time since it has - been inserted into the container, regardless of any reads or writes. - -.. bro:attr:: &persistent - - Makes a variable persistent, i.e., its value is written to disk (per - default at shutdown time). - -.. bro:attr:: &synchronized - - Synchronizes variable accesses across nodes. The value of a - ``&synchronized`` variable is automatically propagated to all peers - when it changes. - -.. bro:attr:: &encrypt - - Encrypts files right before writing them to disk. - -.. TODO: needs to be documented in more detail. - -.. bro:attr:: &raw_output - - Opens a file in raw mode, i.e., non-ASCII characters are not - escaped. - -.. bro:attr:: &mergeable - - Prefers set union to assignment for synchronized state. This - attribute is used in conjunction with :bro:attr:`&synchronized` - container types: when the same container is updated at two peers - with different value, the propagation of the state causes a race - condition, where the last update succeeds. This can cause - inconsistencies and can be avoided by unifying the two sets, rather - than merely overwriting the old value. - -.. bro:attr:: &priority - - Specifies the execution priority (as a signed integer) of a hook or - event handler. Higher values are executed before lower ones. The - default value is 0. - -.. bro:attr:: &group - - Groups event handlers such that those in the same group can be - jointly activated or deactivated. - -.. bro:attr:: &log - - Writes a record field to the associated log stream. - -.. bro:attr:: &error_handler - - Internally set on the events that are associated with the reporter - framework: :bro:id:`reporter_info`, :bro:id:`reporter_warning`, and - :bro:id:`reporter_error`. It prevents any handlers of those events - from being able to generate reporter messages that go through any of - those events (i.e., it prevents an infinite event recursion). Instead, - such nested reporter messages are output to stderr. - -.. bro:attr:: &type_column - - Used by the input framework. It can be used on columns of type - :bro:type:`port` and specifies the name of an additional column in - the input file which specifies the protocol of the port (tcp/udp/icmp). From 026233d1f25ea49a9215388ea2b221d039b4df5d Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 4 Aug 2014 11:15:42 -0700 Subject: [PATCH 02/71] change SSL log to contain a boolean flag signaling if a session was resumed instead of the (usually not really that useful) session ID the client sent. --- scripts/base/protocols/ssl/main.bro | 57 ++++++++++++++++++++++++++--- 1 file changed, 52 insertions(+), 5 deletions(-) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index f1315f8c85..61e6e438db 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -12,7 +12,7 @@ export { ## Time when the SSL connection was first detected. ts: time &log; ## Unique ID for the connection. - uid: string &log; + uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. id: conn_id &log; ## SSL/TLS version that the server offered. @@ -25,7 +25,20 @@ export { ## indicates the server name that the client was requesting. server_name: string &log &optional; ## Session ID offered by the client for session resumption. - session_id: string &log &optional; + ## Not used for logging. + session_id: string &optional; + ## Flag to indicate if the session was resumed and re-used + ## the key material exchanged in an earlier connection. + resumed: bool &log &default=F; + ## Flag to indicate if we saw a non-empty session ticket being + ## sent by the client using an empty session ID. This value + ## is used to determine if a session is being resumed and is + ## not logged + client_ticket_empty_session_seen: bool &default=F; + ## Flag to indicate if we saw a client key exchange message sent + ## by the client. This value is used to determine if a session + ## is being resumed and is not logged. + client_key_exchange_seen: bool &default=F; ## Last alert that was seen during the connection. last_alert: string &log &optional; @@ -36,11 +49,11 @@ export { ## Flag to indicate if this ssl session has been established ## succesfully, or if it was aborted during the handshake. - established: bool &log &default=F; + established: bool &log &default=F; ## Flag to indicate if this record already has been logged, to ## prevent duplicates. - logged: bool &default=F; + logged: bool &default=F; }; ## The default root CA bundle. By default, the mozilla-ca-list.bro @@ -149,8 +162,11 @@ event ssl_client_hello(c: connection, version: count, possible_ts: time, client_ set_session(c); # Save the session_id if there is one set. - if ( session_id != /^\x00{32}$/ ) + if ( |session_id| > 0 && session_id != /^\x00{32}$/ ) + { c$ssl$session_id = bytestring_to_hexstr(session_id); + c$ssl$client_ticket_empty_session_seen = F; + } } event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=5 @@ -159,6 +175,9 @@ event ssl_server_hello(c: connection, version: count, possible_ts: time, server_ c$ssl$version = version_strings[version]; c$ssl$cipher = cipher_desc[cipher]; + + if ( c$ssl?$session_id && c$ssl$session_id == bytestring_to_hexstr(session_id) ) + c$ssl$resumed = T; } event ssl_server_curve(c: connection, curve: count) &priority=5 @@ -180,6 +199,34 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) } } +event ssl_handshake_message(c: connection, is_orig: bool, msg_type: count, length: count) &priority=5 + { + set_session(c); + + if ( is_orig && msg_type == SSL::CLIENT_KEY_EXCHANGE ) + c$ssl$client_key_exchange_seen = T; + } + +# extension event is fired _before_ the respective client or server hello. +# Important for client_ticket_empty_session_seen +event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5 + { + set_session(c); + + if ( is_orig && SSL::extensions[code] == "SessionTicket TLS" && |val| > 0 ) + # in this case, we might have an empty ID. Set back to F in client_hello event + # if it is not empty after all + c$ssl$client_ticket_empty_session_seen = T; + } + +event ssl_change_cipher_spec(c: connection, is_orig: bool) &priority=5 + { + set_session(c); + + if ( is_orig && c$ssl$client_ticket_empty_session_seen && !c$ssl$client_key_exchange_seen ) + c$ssl$resumed = T; + } + event ssl_alert(c: connection, is_orig: bool, level: count, desc: count) &priority=5 { set_session(c); From 14d265482a3793c65f3c9461fdfb7f790ab509b2 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Mon, 4 Aug 2014 22:16:09 -0700 Subject: [PATCH 03/71] add information about server chosen protocol to ssl.log, if provided by alpn. This is e.g. used to negotiate spdy or http/2 --- scripts/base/protocols/ssl/main.bro | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/base/protocols/ssl/main.bro b/scripts/base/protocols/ssl/main.bro index 61e6e438db..e491c915fa 100644 --- a/scripts/base/protocols/ssl/main.bro +++ b/scripts/base/protocols/ssl/main.bro @@ -41,6 +41,9 @@ export { client_key_exchange_seen: bool &default=F; ## Last alert that was seen during the connection. last_alert: string &log &optional; + ## Next protocol the server chose using the application layer + ## next protocol extension, if present. + next_protocol: string &log &optional; ## The analyzer ID used for the analyzer instance attached ## to each connection. It is not used for logging since it's a @@ -199,6 +202,17 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec) } } +event ssl_extension_application_layer_protocol_negotiation(c: connection, is_orig: bool, protocols: string_vec) + { + set_session(c); + + if ( is_orig ) + return; + + if ( |protocols| > 0 ) + c$ssl$next_protocol = protocols[0]; + } + event ssl_handshake_message(c: connection, is_orig: bool, msg_type: count, length: count) &priority=5 { set_session(c); From 33053cca3aa8d3670f96fd430b4725915030cde4 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 8 Aug 2014 09:26:11 -0700 Subject: [PATCH 04/71] Mark everything below 2048 bit as a weak key (Browsers will stop accepting 1024 bits soon, so we can be of that opinion too). Also - fix notice suppression. :/ --- scripts/policy/protocols/ssl/weak-keys.bro | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/scripts/policy/protocols/ssl/weak-keys.bro b/scripts/policy/protocols/ssl/weak-keys.bro index f11fb9da5e..e849c3c06c 100644 --- a/scripts/policy/protocols/ssl/weak-keys.bro +++ b/scripts/policy/protocols/ssl/weak-keys.bro @@ -22,7 +22,7 @@ export { ## The minimal key length in bits that is considered to be safe. Any shorter ## (non-EC) key lengths will trigger the notice. - const notify_minimal_key_length = 1024 &redef; + const notify_minimal_key_length = 2048 &redef; ## Warn if the DH key length is smaller than the certificate key length. This is ## potentially unsafe because it gives a wrong impression of safety due to the @@ -56,7 +56,7 @@ event ssl_established(c: connection) &priority=3 NOTICE([$note=Weak_Key, $msg=fmt("Host uses weak certificate with %d bit key", key_length), $conn=c, $suppress_for=1day, - $identifier=cat(c$id$orig_h, c$id$orig_p, key_length) + $identifier=cat(c$id$resp_h, c$id$resp_h, key_length) ]); } @@ -66,12 +66,12 @@ event ssl_dh_server_params(c: connection, p: string, q: string, Ys: string) &pri return; local key_length = |Ys| * 8; # key length in bits - + if ( key_length < notify_minimal_key_length ) NOTICE([$note=Weak_Key, $msg=fmt("Host uses weak DH parameters with %d key bits", key_length), $conn=c, $suppress_for=1day, - $identifier=cat(c$id$orig_h, c$id$orig_p, key_length) + $identifier=cat(c$id$resp_h, c$id$resp_p, key_length) ]); if ( notify_dh_length_shorter_cert_length && @@ -86,7 +86,7 @@ event ssl_dh_server_params(c: connection, p: string, q: string, Ys: string) &pri $msg=fmt("DH key length of %d bits is smaller certificate key length of %d bits", key_length, c$ssl$cert_chain[0]$x509$certificate$key_length), $conn=c, $suppress_for=1day, - $identifier=cat(c$id$orig_h, c$id$orig_p) + $identifier=cat(c$id$resp_h, c$id$resp_p) ]); } } From 675b12c960b9f264a5c61aa1fa1821371e61be67 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Fri, 8 Aug 2014 10:14:11 -0700 Subject: [PATCH 05/71] update test baselines --- .../scripts.base.protocols.ftp.gridftp/ssl.log | 12 ++++++------ .../scripts.base.protocols.pop3.starttls/ssl.log | 10 +++++----- .../scripts.base.protocols.smtp.starttls/ssl.log | 10 +++++----- .../scripts.base.protocols.ssl.basic/ssl.log | 10 +++++----- .../Baseline/scripts.base.protocols.ssl.dhe/ssl.log | 10 +++++----- .../scripts.base.protocols.ssl.ecdhe/ssl.log | 10 +++++----- .../ssl.log | 10 +++++----- .../scripts.base.protocols.ssl.tls-1.2/ssl.log | 10 +++++----- .../scripts.policy.protocols.ssl.known-certs/ssl.log | 12 ++++++------ .../ssl.log | 12 ++++++------ .../ssl.log | 10 +++++----- 11 files changed, 58 insertions(+), 58 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log index 5fb15d53ae..130ca05bca 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.ftp.gridftp/ssl.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-26-16-44-47 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1348168976.508038 CXWv6p3arKYeMETxOg 192.168.57.103 60108 192.168.57.101 2811 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - - - T FBtbj87tgpyeDSj31,F8TfgZ31c1dFu8Kt2k FVNYOh2BeQBb7MpCPe,FwjBou1e5DbpE0eOgk,FbYQmk4x4M4Bx3PZme CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid -1348168976.551422 CjhGID4nQcgTWjvg4c 192.168.57.103 35391 192.168.57.101 55968 TLSv10 TLS_RSA_WITH_NULL_SHA - - - - T F4SSqN31HDIrrH5Q8h,FJHp5Pf6VLQsRQK3,FHACqa3dX9BXRV2av,FNnDVT1NURRWeoLLN3 FFWYVj4BcvQb35WIaf,Fj16G835fnJgnVlKU6,FGONoc1Nj0Ka5zlxDa CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid -#close 2014-04-26-16-44-47 +#open 2014-08-08-17-13-20 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1348168976.508038 CXWv6p3arKYeMETxOg 192.168.57.103 60108 192.168.57.101 2811 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - F - - T FBtbj87tgpyeDSj31,F8TfgZ31c1dFu8Kt2k FVNYOh2BeQBb7MpCPe,FwjBou1e5DbpE0eOgk,FbYQmk4x4M4Bx3PZme CN=host/alpha,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=Globus Simple CA,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid +1348168976.551422 CjhGID4nQcgTWjvg4c 192.168.57.103 35391 192.168.57.101 55968 TLSv10 TLS_RSA_WITH_NULL_SHA - - F - - T F4SSqN31HDIrrH5Q8h,FJHp5Pf6VLQsRQK3,FHACqa3dX9BXRV2av,FNnDVT1NURRWeoLLN3 FFWYVj4BcvQb35WIaf,Fj16G835fnJgnVlKU6,FGONoc1Nj0Ka5zlxDa CN=932373381,CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=917532944,CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid CN=Jon Siwek,OU=local,OU=simpleCA-alpha,OU=GlobusTest,O=Grid +#close 2014-08-08-17-13-20 diff --git a/testing/btest/Baseline/scripts.base.protocols.pop3.starttls/ssl.log b/testing/btest/Baseline/scripts.base.protocols.pop3.starttls/ssl.log index 1eab1092ed..632dcddfb0 100644 --- a/testing/btest/Baseline/scripts.base.protocols.pop3.starttls/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.pop3.starttls/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-05-15-17-23-07 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1400173552.424910 CXWv6p3arKYeMETxOg 192.168.4.149 54775 192.168.4.149 110 TLSv12 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 - - - - T FEdAw24VSam39HNlY5 (empty) emailAddress=postmaster@lilawelt.de,CN=chimaera.lilawelt.de,OU=Servers,O=Lilawelt,L=Munich,C=DE emailAddress=postmaster@lilawelt.de,CN=Lilawelt,OU=Lilawelt CA,O=Lilawelt,L=Munich,C=DE - - -#close 2014-05-15-17-23-07 +#open 2014-08-08-17-13-27 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1400173552.424910 CXWv6p3arKYeMETxOg 192.168.4.149 54775 192.168.4.149 110 TLSv12 TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 - - F - - T FEdAw24VSam39HNlY5 (empty) emailAddress=postmaster@lilawelt.de,CN=chimaera.lilawelt.de,OU=Servers,O=Lilawelt,L=Munich,C=DE emailAddress=postmaster@lilawelt.de,CN=Lilawelt,OU=Lilawelt CA,O=Lilawelt,L=Munich,C=DE - - +#close 2014-08-08-17-13-27 diff --git a/testing/btest/Baseline/scripts.base.protocols.smtp.starttls/ssl.log b/testing/btest/Baseline/scripts.base.protocols.smtp.starttls/ssl.log index cec018c589..5e9aa80cb5 100644 --- a/testing/btest/Baseline/scripts.base.protocols.smtp.starttls/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.smtp.starttls/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-05-15-16-56-36 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1400168397.019290 CXWv6p3arKYeMETxOg 192.168.4.149 54170 74.125.142.26 25 TLSv12 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 secp256r1 - - - T FUE0dj3SWjQASC0bk3,FbPkr51wrSMIUT5Hib,FVW1o23Jjs8yenOhzb (empty) CN=mx.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - -#close 2014-05-15-16-56-36 +#open 2014-08-08-17-13-38 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1400168397.019290 CXWv6p3arKYeMETxOg 192.168.4.149 54170 74.125.142.26 25 TLSv12 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 secp256r1 - F - - T FUE0dj3SWjQASC0bk3,FbPkr51wrSMIUT5Hib,FVW1o23Jjs8yenOhzb (empty) CN=mx.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - +#close 2014-08-08-17-13-38 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log index 7834e74868..bea2210a13 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.basic/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-26-16-45-01 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1335538392.319381 CXWv6p3arKYeMETxOg 192.168.1.105 62045 74.125.224.79 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 ssl.gstatic.com - - T F6wfNWn8LR755SYo7,FJl60T1mOolaez9T0h (empty) CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority,O=Google Inc,C=US - - -#close 2014-04-26-16-45-01 +#open 2014-08-08-17-13-42 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1335538392.319381 CXWv6p3arKYeMETxOg 192.168.1.105 62045 74.125.224.79 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 ssl.gstatic.com F - - T F6wfNWn8LR755SYo7,FJl60T1mOolaez9T0h (empty) CN=*.gstatic.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority,O=Google Inc,C=US - - +#close 2014-08-08-17-13-42 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.dhe/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.dhe/ssl.log index 652f3b3df7..0eb4ee4100 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ssl.dhe/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.dhe/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-27-00-52-03 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1398558136.319509 CXWv6p3arKYeMETxOg 192.168.18.50 62277 162.219.2.166 443 TLSv12 TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - - - - T F6fLv13PBYz8MNqx68,F8cTDl1penwXxGu4K7 (empty) emailAddress=denicadmmail@arcor.de,CN=www.lilawelt.net,C=US CN=StartCom Class 1 Primary Intermediate Server CA,OU=Secure Digital Certificate Signing,O=StartCom Ltd.,C=IL - - -#close 2014-04-27-00-52-03 +#open 2014-08-08-17-13-45 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1398558136.319509 CXWv6p3arKYeMETxOg 192.168.18.50 62277 162.219.2.166 443 TLSv12 TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA - - F - - T F6fLv13PBYz8MNqx68,F8cTDl1penwXxGu4K7 (empty) emailAddress=denicadmmail@arcor.de,CN=www.lilawelt.net,C=US CN=StartCom Class 1 Primary Intermediate Server CA,OU=Secure Digital Certificate Signing,O=StartCom Ltd.,C=IL - - +#close 2014-08-08-17-13-45 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.ecdhe/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.ecdhe/ssl.log index 66ea42be70..2571efbbff 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ssl.ecdhe/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.ecdhe/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-26-16-39-57 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1398529018.678827 CXWv6p3arKYeMETxOg 192.168.18.50 56981 74.125.239.97 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA secp256r1 - - - T FDy6ve1m58lwPRfhE9,FnGjwc1EVGk5x0WZk5,F2T07R1XZFCmeWafv2 (empty) CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - -#close 2014-04-26-16-39-57 +#open 2014-08-08-17-13-48 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1398529018.678827 CXWv6p3arKYeMETxOg 192.168.18.50 56981 74.125.239.97 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA secp256r1 - F - - T FDy6ve1m58lwPRfhE9,FnGjwc1EVGk5x0WZk5,F2T07R1XZFCmeWafv2 (empty) CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - +#close 2014-08-08-17-13-48 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-handshake-failure/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-handshake-failure/ssl.log index 082106e89e..1bee20aaff 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-handshake-failure/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2-handshake-failure/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-26-16-45-16 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1393957586.786031 CXWv6p3arKYeMETxOg 192.168.4.149 53525 74.125.239.37 443 - - - - - handshake_failure F - - - - - - -#close 2014-04-26-16-45-16 +#open 2014-08-08-17-13-53 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1393957586.786031 CXWv6p3arKYeMETxOg 192.168.4.149 53525 74.125.239.37 443 - - - - F handshake_failure - F - - - - - - +#close 2014-08-08-17-13-53 diff --git a/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log index ab1345d0cc..4e51c53d27 100644 --- a/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log +++ b/testing/btest/Baseline/scripts.base.protocols.ssl.tls-1.2/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-26-16-45-09 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1357328848.549370 CXWv6p3arKYeMETxOg 10.0.0.80 56637 68.233.76.12 443 TLSv12 TLS_RSA_WITH_RC4_128_MD5 - - - - T FlnQzb2dJK4p9jXwmd,FaDzX22O4j3kFF6Jqg,F9Tsjm3OdCmGGw43Yh (empty) CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - -#close 2014-04-26-16-45-09 +#open 2014-08-08-17-13-51 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1357328848.549370 CXWv6p3arKYeMETxOg 10.0.0.80 56637 68.233.76.12 443 TLSv12 TLS_RSA_WITH_RC4_128_MD5 - - F - - T FlnQzb2dJK4p9jXwmd,FaDzX22O4j3kFF6Jqg,F9Tsjm3OdCmGGw43Yh (empty) CN=*.taleo.net,OU=Comodo PremiumSSL Wildcard,OU=Web,O=Taleo Inc.,street=4140 Dublin Boulevard,street=Suite 400,L=Dublin,ST=CA,postalCode=94568,C=US CN=COMODO High-Assurance Secure Server CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - +#close 2014-08-08-17-13-51 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/ssl.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/ssl.log index b09bd04350..031f205aab 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/ssl.log +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.known-certs/ssl.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-27-06-48-05 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string -1394747126.855035 CXWv6p3arKYeMETxOg 192.168.4.149 60623 74.125.239.129 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 secp256r1 - - - T FlaIzV19yTmBYwWwc6,F0BeiV3cMsGkNML0P2,F6PfYi2WUoPdIJrhpg (empty) CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - -1394747129.505622 CjhGID4nQcgTWjvg4c 192.168.4.149 60624 74.125.239.129 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 secp256r1 - - - T FOye6a4kt8a7QChqw3,FytlLr3jOQenFAVtYi,FEmnxy4DGbxkmtQJS1 (empty) CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - -#close 2014-04-27-06-48-05 +#open 2014-08-08-17-13-55 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string +1394747126.855035 CXWv6p3arKYeMETxOg 192.168.4.149 60623 74.125.239.129 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 secp256r1 - F - - T FlaIzV19yTmBYwWwc6,F0BeiV3cMsGkNML0P2,F6PfYi2WUoPdIJrhpg (empty) CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - +1394747129.505622 CjhGID4nQcgTWjvg4c 192.168.4.149 60624 74.125.239.129 443 TLSv12 TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 secp256r1 - F - - T FOye6a4kt8a7QChqw3,FytlLr3jOQenFAVtYi,FEmnxy4DGbxkmtQJS1 (empty) CN=*.google.com,O=Google Inc,L=Mountain View,ST=California,C=US CN=Google Internet Authority G2,O=Google Inc,C=US - - +#close 2014-08-08-17-13-55 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log index 7965e3be89..a464c64670 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-certs/ssl.log @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-04-26-16-45-32 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string string -1394745602.951961 CXWv6p3arKYeMETxOg 192.168.4.149 60539 87.98.220.10 443 TLSv10 TLS_DHE_RSA_WITH_AES_256_CBC_SHA - - - - T F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl (empty) CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - certificate has expired -1394745618.791420 CjhGID4nQcgTWjvg4c 192.168.4.149 60540 122.1.240.204 443 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - - - T F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h (empty) CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - ok -#close 2014-04-26-16-45-32 +#open 2014-08-08-17-13-58 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer validation_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1394745602.951961 CXWv6p3arKYeMETxOg 192.168.4.149 60539 87.98.220.10 443 TLSv10 TLS_DHE_RSA_WITH_AES_256_CBC_SHA - - F - - T F1fX1R2cDOzbvg17ye,FqPEQR2eytAQybroyl (empty) CN=www.spidh.org,OU=COMODO SSL,OU=Domain Control Validated CN=COMODO SSL CA,O=COMODO CA Limited,L=Salford,ST=Greater Manchester,C=GB - - certificate has expired +1394745618.791420 CjhGID4nQcgTWjvg4c 192.168.4.149 60540 122.1.240.204 443 TLSv10 TLS_RSA_WITH_AES_256_CBC_SHA - - F - - T F6NAbK127LhNBaEe5c,FDhmPt28vyXlGMTxP7,F0ROCKibhE1KntJ1h (empty) CN=www.tobu-estate.com,OU=Terms of use at www.verisign.com/rpa (c)05,O=TOBU RAILWAY Co.\,Ltd.,L=Sumida-ku,ST=Tokyo,C=JP CN=VeriSign Class 3 Secure Server CA - G3,OU=Terms of use at https://www.verisign.com/rpa (c)10,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - ok +#close 2014-08-08-17-13-58 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl.log index 33b589d9ac..7fc4b5c636 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl.log +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-05-30-22-37-19 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string string -1398367809.790512 CXWv6p3arKYeMETxOg 192.168.4.149 56253 131.253.61.82 443 TLSv10 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA secp384r1 - - - T Fr1vuhmDOykX05Vj1,FlFGqI1PyTt7Vuo8E9,FSASzpV1NMIvbQ1W9 (empty) CN=login.live.com,OU=MSA,O=Microsoft Corporation,street=1 Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=Private Organization,1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 CN=VeriSign Class 3 Extended Validation SSL SGC CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - good -#close 2014-05-30-22-37-19 +#open 2014-08-08-17-14-03 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1398367809.790512 CXWv6p3arKYeMETxOg 192.168.4.149 56253 131.253.61.82 443 TLSv10 TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA secp384r1 - F - - T Fr1vuhmDOykX05Vj1,FlFGqI1PyTt7Vuo8E9,FSASzpV1NMIvbQ1W9 (empty) CN=login.live.com,OU=MSA,O=Microsoft Corporation,street=1 Microsoft Way,L=Redmond,ST=Washington,postalCode=98052,C=US,serialNumber=600413485,businessCategory=Private Organization,1.3.6.1.4.1.311.60.2.1.2=#130A57617368696E67746F6E,1.3.6.1.4.1.311.60.2.1.3=#13025553 CN=VeriSign Class 3 Extended Validation SSL SGC CA,OU=Terms of use at https://www.verisign.com/rpa (c)06,OU=VeriSign Trust Network,O=VeriSign\, Inc.,C=US - - good +#close 2014-08-08-17-14-03 From d5513a07575b990a87e994a3ae4873bc02adaf69 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 26 Aug 2014 17:50:28 -0500 Subject: [PATCH 06/71] Improve multipart HTTP/MIME entity file analysis. Singular CR or LF characters in multipart body content are no longer converted to a full CRLF (thus corrupting the file) and it also no longer considers the CRLF before the multipart boundary as part of the content. Addresses BIT-1235. --- src/analyzer/protocol/http/HTTP.cc | 21 +++++++++++++ src/analyzer/protocol/http/HTTP.h | 2 ++ src/analyzer/protocol/mime/MIME.cc | 26 ++++++++++++++-- src/analyzer/protocol/mime/MIME.h | 1 + src/analyzer/protocol/tcp/ContentLine.cc | 7 +++-- src/analyzer/protocol/tcp/ContentLine.h | 5 ++++ .../1-file | 2 +- .../2-file | 2 +- .../3-file | 2 +- .../out | 30 +++++++++---------- .../out | 8 ++--- .../thefile0 | 1 - .../all-events.log | 6 ++-- 13 files changed, 82 insertions(+), 31 deletions(-) diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index 02b6947b9f..857cb35980 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -463,6 +463,20 @@ void HTTP_Entity::SubmitAllHeaders() if ( DEBUG_http ) DEBUG_MSG("%.6f end of headers\n", network_time); + if ( Parent() && + Parent()->MIMEContentType() == mime::CONTENT_TYPE_MULTIPART ) + { + // Don't treat single \r or \n characters in the multipart body content + // as lines because the MIME_Entity code will implicitly add back a + // \r\n for each line it receives. We do this instead of setting + // plain delivery mode for the content line analyzer because + // the size of the content to deliver "plainly" may be unknown + // and just leaving it in that mode indefinitely screws up the + // detection of multipart boundaries. + http_message->content_line->SupressWeirds(true); + http_message->content_line->SetCRLFAsEOL(0); + } + // The presence of a message-body in a request is signaled by // the inclusion of a Content-Length or Transfer-Encoding // header field in the request's message-headers. @@ -664,6 +678,13 @@ void HTTP_Message::EndEntity(mime::MIME_Entity* entity) current_entity = (HTTP_Entity*) entity->Parent(); + if ( entity->Parent() && + entity->Parent()->MIMEContentType() == mime::CONTENT_TYPE_MULTIPART ) + { + content_line->SupressWeirds(false); + content_line->SetCRLFAsEOL(); + } + // It is necessary to call Done when EndEntity is triggered by // SubmitAllHeaders (through EndOfData). if ( entity == top_level ) diff --git a/src/analyzer/protocol/http/HTTP.h b/src/analyzer/protocol/http/HTTP.h index 5785d93198..075e6f4dba 100644 --- a/src/analyzer/protocol/http/HTTP.h +++ b/src/analyzer/protocol/http/HTTP.h @@ -99,6 +99,8 @@ enum { // HTTP_MessageDone -> {Request,Reply}Made class HTTP_Message : public mime::MIME_Message { +friend class HTTP_Entity; + public: HTTP_Message(HTTP_Analyzer* analyzer, tcp::ContentLine_Analyzer* cl, bool is_orig, int expect_body, int64_t init_header_length); diff --git a/src/analyzer/protocol/mime/MIME.cc b/src/analyzer/protocol/mime/MIME.cc index 6f992c9256..0e54fb7826 100644 --- a/src/analyzer/protocol/mime/MIME.cc +++ b/src/analyzer/protocol/mime/MIME.cc @@ -552,6 +552,7 @@ void MIME_Entity::init() data_buf_offset = -1; message = 0; + delay_adding_implicit_CRLF = false; } MIME_Entity::~MIME_Entity() @@ -1005,12 +1006,33 @@ void MIME_Entity::DecodeDataLine(int len, const char* data, int trailing_CRLF) void MIME_Entity::DecodeBinary(int len, const char* data, int trailing_CRLF) { + if ( delay_adding_implicit_CRLF ) + { + delay_adding_implicit_CRLF = false; + DataOctet(CR); + DataOctet(LF); + } + DataOctets(len, data); if ( trailing_CRLF ) { - DataOctet(CR); - DataOctet(LF); + if ( Parent() && + Parent()->MIMEContentType() == mime::CONTENT_TYPE_MULTIPART ) + { + // For multipart body content, we want to keep all implicit CRLFs + // except for the last because that one belongs to the multipart + // boundary delimiter, not the content. Simply delaying the + // addition of implicit CRLFs until another chunk of content + // data comes in is a way to prevent the CRLF before the final + // message boundary from being accidentally added to the content. + delay_adding_implicit_CRLF = true; + } + else + { + DataOctet(CR); + DataOctet(LF); + } } } diff --git a/src/analyzer/protocol/mime/MIME.h b/src/analyzer/protocol/mime/MIME.h index 2b2f88105d..1790d0faaa 100644 --- a/src/analyzer/protocol/mime/MIME.h +++ b/src/analyzer/protocol/mime/MIME.h @@ -172,6 +172,7 @@ protected: int data_buf_offset; MIME_Message* message; + bool delay_adding_implicit_CRLF; }; // The reason I separate MIME_Message as an abstract class is to diff --git a/src/analyzer/protocol/tcp/ContentLine.cc b/src/analyzer/protocol/tcp/ContentLine.cc index 72314dd45d..f5dd7aaf07 100644 --- a/src/analyzer/protocol/tcp/ContentLine.cc +++ b/src/analyzer/protocol/tcp/ContentLine.cc @@ -32,6 +32,7 @@ void ContentLine_Analyzer::InitState() seq_to_skip = 0; plain_delivery_length = 0; is_plain = 0; + suppress_weirds = false; InitBuffer(0); } @@ -258,7 +259,7 @@ int ContentLine_Analyzer::DoDeliverOnce(int len, const u_char* data) else { - if ( Conn()->FlagEvent(SINGULAR_LF) ) + if ( ! suppress_weirds && Conn()->FlagEvent(SINGULAR_LF) ) Conn()->Weird("line_terminated_with_single_LF"); buf[offset++] = c; } @@ -277,7 +278,7 @@ int ContentLine_Analyzer::DoDeliverOnce(int len, const u_char* data) } if ( last_char == '\r' ) - if ( Conn()->FlagEvent(SINGULAR_CR) ) + if ( ! suppress_weirds && Conn()->FlagEvent(SINGULAR_CR) ) Conn()->Weird("line_terminated_with_single_CR"); last_char = c; @@ -307,7 +308,7 @@ void ContentLine_Analyzer::CheckNUL() ; // Ignore it. else { - if ( Conn()->FlagEvent(NUL_IN_LINE) ) + if ( ! suppress_weirds && Conn()->FlagEvent(NUL_IN_LINE) ) Conn()->Weird("NUL_in_line"); flag_NULs = 0; } diff --git a/src/analyzer/protocol/tcp/ContentLine.h b/src/analyzer/protocol/tcp/ContentLine.h index 93c473c47c..7a5a6b996e 100644 --- a/src/analyzer/protocol/tcp/ContentLine.h +++ b/src/analyzer/protocol/tcp/ContentLine.h @@ -15,6 +15,9 @@ public: ContentLine_Analyzer(Connection* conn, bool orig); ~ContentLine_Analyzer(); + void SupressWeirds(bool enable) + { suppress_weirds = enable; } + // If enabled, flag (first) line with embedded NUL. Default off. void SetIsNULSensitive(bool enable) { flag_NULs = enable; } @@ -96,6 +99,8 @@ protected: // Don't deliver further data. int skip_deliveries; + bool suppress_weirds; + // If true, flag (first) line with embedded NUL. unsigned int flag_NULs:1; diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file index 77356c3140..30d74d2584 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/1-file @@ -1 +1 @@ -test +test \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file index ac2a9e002d..d606037cb2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/2-file @@ -1 +1 @@ -test2 +test2 \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file index ae48ec8c20..29f446afe2 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/3-file @@ -1 +1 @@ -test3 +test3 \ No newline at end of file diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out index b22c8fe886..0bf8d6a0c9 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.http.multipart/out @@ -1,39 +1,39 @@ FILE_NEW file #0, 0, 0 FILE_BOF_BUFFER -test^M^J +test FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -file #0, 6, 0 +file #0, 4, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] source: HTTP -MD5: 9f06243abcb89c70e0c331c61d871fa7 -SHA1: fde773a18bb29f5ed65e6f0a7aa717fd1fa485d4 -SHA256: 837ccb607e312b170fac7383d7ccfd61fa5072793f19a25e75fbacb56539b86b +MD5: 098f6bcd4621d373cade4e832627b4f6 +SHA1: a94a8fe5ccb19ba61c4c0873d391e987982fbbd3 +SHA256: 9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08 FILE_NEW file #1, 0, 0 FILE_BOF_BUFFER -test2^M^J +test2 FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -file #1, 7, 0 +file #1, 5, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] source: HTTP -MD5: d68af81ef370b3873d50f09140068810 -SHA1: 51a7b6f2d91f6a87822dc04560f2972bc14fc97e -SHA256: de0edd0ac4a705aff70f34734e90a1d0a1d8b76abe4bb53f3ea934bc105b3b17 +MD5: ad0234829205b9033196ba818f7a872b +SHA1: 109f4b3c50d7b0df729d299bc6f8e9ef9066971f +SHA256: 60303ae22b998861bce3b28f33eec1be758a213c86c93c076dbe9f558c11c752 FILE_NEW file #2, 0, 0 FILE_BOF_BUFFER -test3^M^J +test3 FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -file #2, 7, 0 +file #2, 5, 0 [orig_h=141.142.228.5, orig_p=57262/tcp, resp_h=54.243.88.146, resp_p=80/tcp] source: HTTP -MD5: 1a3d75d44753ad246f0bd333cdaf08b0 -SHA1: 4f98809ab09272dfcc58266e3f23ae2393f70e76 -SHA256: 018c67a2c30ed9977e1dddfe98cac542165dac355cf9764c91a362613e752933 +MD5: 8ad8757baa8564dc136c1e07507f4a98 +SHA1: 3ebfa301dc59196f18593c45e519287a23297589 +SHA256: fd61a03af4f77d870fc21e05e7e80678095c92d808cfb3b5c279ee04c74aca13 FILE_NEW file #3, 0, 0 FILE_BOF_BUFFER diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out index 1d54e9a2ac..44c240c7ee 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/out @@ -6,12 +6,12 @@ MIME_TYPE text/plain FILE_OVER_NEW_CONNECTION FILE_STATE_REMOVE -file #0, 79, 0 +file #0, 77, 0 [orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp] source: SMTP -MD5: 92bca2e6cdcde73647125da7dccbdd07 -SHA1: b7e497be8a9f5e2c4b6980fceb015360f98f4a13 -SHA256: 785a8a044d1454ec88837108f443bbb30cc4f529393ffd57118261036bfe59f5 +MD5: 58aff3af22807bc5f4b6357c0038256c +SHA1: c39dc8cd0f8d8b1f7fc8b362c41e69fdf20f668a +SHA256: 8d057f3af311c20675eea767a9df5fa31ff3597c6d5d50fd0cdc34766c40204d FILE_NEW file #1, 0, 0 FILE_BOF_BUFFER diff --git a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile0 b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile0 index f4dd7d22f4..0b84e1fd86 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile0 +++ b/testing/btest/Baseline/scripts.base.frameworks.file-analysis.smtp/thefile0 @@ -10,4 +10,3 @@ Find the attachment GPS - diff --git a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log index b8f576e497..6508792b36 100644 --- a/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log +++ b/testing/btest/Baseline/scripts.policy.misc.dump-events/all-events.log @@ -305,15 +305,15 @@ [2] is_orig: bool = T 1254722770.692743 file_new - [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=, u2_events=] + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=, u2_events=] 1254722770.692743 file_over_new_connection - [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^J}, rx_hosts={^J^J}, conn_uids={^J^J}, source=SMTP, depth=0, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=] + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^J}, rx_hosts={^J^J}, conn_uids={^J^J}, source=SMTP, depth=0, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=] [1] c: connection = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^ISMTP^J}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^J}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^J}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=] [2] is_orig: bool = F 1254722770.692743 file_state_remove - [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=79, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=] + [0] f: fa_file = [id=Fel9gs4OtNEV6gUJZ5, parent_id=, source=SMTP, is_orig=F, conns={^J^I[[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp]] = [id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], orig=[size=1610, state=4, num_pkts=9, num_bytes_ip=518, flow_label=0], resp=[size=462, state=4, num_pkts=10, num_bytes_ip=870, flow_label=0], start_time=1254722767.529046, duration=3.163697, service={^J^I^ISMTP^J^I}, addl=, hot=0, history=ShAdDa, uid=CjhGID4nQcgTWjvg4c, tunnel=, dpd=, conn=, extract_orig=F, extract_resp=F, dhcp=, dnp3=, dns=, dns_state=, ftp=, ftp_data_reuse=F, ssl=, http=, http_state=, irc=, modbus=, radius=, snmp=, smtp=[ts=1254722768.219663, uid=CjhGID4nQcgTWjvg4c, id=[orig_h=10.10.1.4, orig_p=1470/tcp, resp_h=74.53.140.153, resp_p=25/tcp], trans_depth=1, helo=GP, mailfrom=, rcptto={^J^I^I^J^I}, date=Mon, 5 Oct 2009 11:36:07 +0530, from="Gurpartap Singh" , to={^J^I^I^J^I}, reply_to=, msg_id=<000301ca4581$ef9e57f0$cedb07d0$@in>, in_reply_to=, subject=SMTP, x_originating_ip=, first_received=, second_received=, last_reply=354 Enter message, ending with "." on a line by itself, path=[74.53.140.153, 10.10.1.4], user_agent=Microsoft Office Outlook 12.0, tls=F, process_received_from=T, has_client_activity=T, entity=, fuids=[Fel9gs4OtNEV6gUJZ5]], smtp_state=[helo=GP, messages_transferred=0, pending_messages=, mime_depth=3], socks=, ssh=, syslog=]^J}, last_active=1254722770.692743, seen_bytes=77, total_bytes=, missing_bytes=0, overflow_bytes=0, timeout_interval=2.0 mins, bof_buffer_size=1024, bof_buffer=Hello^M^J^M^J ^M^J^M^JI send u smtp pcap file ^M^J^M^JFind the attachment^M^J^M^J ^M^J^M^JGPS^M^J^M^J, mime_type=text/plain, mime_types=[[strength=-20, mime=text/plain]], info=[ts=1254722770.692743, fuid=Fel9gs4OtNEV6gUJZ5, tx_hosts={^J^I74.53.140.153^J}, rx_hosts={^J^I10.10.1.4^J}, conn_uids={^J^ICjhGID4nQcgTWjvg4c^J}, source=SMTP, depth=3, analyzers={^J^J}, mime_type=text/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=] 1254722770.692743 get_file_handle [0] tag: enum = Analyzer::ANALYZER_SMTP From 675fba3fdee0a391cfb6fc52d07b08caaca96c76 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 28 Aug 2014 13:13:30 -0500 Subject: [PATCH 07/71] Remove timeouts from remote communication loop. The select() now blocks until there's work to do instead of relying on a small timeout value which can cause unproductive use of cpu cycles. --- src/CMakeLists.txt | 2 + src/ChunkedIO.cc | 46 ++++++++++++++++++++++- src/ChunkedIO.h | 16 +++++++- src/DNS_Mgr.cc | 5 ++- src/DNS_Mgr.h | 3 +- src/Flare.cc | 29 +++++++++++++++ src/Flare.h | 45 +++++++++++++++++++++++ src/FlowSrc.cc | 5 ++- src/FlowSrc.h | 3 +- src/IOSource.cc | 47 ++++++++++++++++++------ src/IOSource.h | 13 +++++-- src/Pipe.cc | 79 ++++++++++++++++++++++++++++++++++++++++ src/Pipe.h | 57 +++++++++++++++++++++++++++++ src/PktSrc.cc | 5 ++- src/PktSrc.h | 3 +- src/RemoteSerializer.cc | 47 +++++++++++------------- src/RemoteSerializer.h | 3 +- src/Serializer.cc | 5 ++- src/Serializer.h | 3 +- src/threading/Manager.cc | 3 +- src/threading/Manager.h | 3 +- 21 files changed, 364 insertions(+), 58 deletions(-) create mode 100644 src/Flare.cc create mode 100644 src/Flare.h create mode 100644 src/Pipe.cc create mode 100644 src/Pipe.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04867b7189..3764533b66 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -279,6 +279,7 @@ set(bro_SRCS EventRegistry.cc Expr.cc File.cc + Flare.cc FlowSrc.cc Frag.cc Frame.cc @@ -299,6 +300,7 @@ set(bro_SRCS OSFinger.cc PacketFilter.cc PersistenceSerializer.cc + Pipe.cc PktSrc.cc PolicyFile.cc PrefixTable.cc diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc index 54e2e59575..a94eb98748 100644 --- a/src/ChunkedIO.cc +++ b/src/ChunkedIO.cc @@ -210,6 +210,7 @@ bool ChunkedIOFd::WriteChunk(Chunk* chunk, bool partial) else pending_head = pending_tail = q; + write_flare.Fire(); return Flush(); } @@ -232,6 +233,7 @@ bool ChunkedIOFd::PutIntoWriteBuffer(Chunk* chunk) write_len += len; delete chunk; + write_flare.Fire(); if ( network_time - last_flush > 0.005 ) FlushWriteBuffer(); @@ -269,6 +271,10 @@ bool ChunkedIOFd::FlushWriteBuffer() if ( unsigned(written) == len ) { write_pos = write_len = 0; + + if ( ! pending_head ) + write_flare.Extinguish(); + return true; } @@ -318,7 +324,12 @@ bool ChunkedIOFd::Flush() } } - return FlushWriteBuffer(); + bool rval = FlushWriteBuffer(); + + if ( ! pending_head && write_len == 0 ) + write_flare.Extinguish(); + + return rval; } uint32 ChunkedIOFd::ChunkAvailable() @@ -394,6 +405,9 @@ bool ChunkedIOFd::Read(Chunk** chunk, bool may_block) #ifdef DEBUG_COMMUNICATION AddToBuffer("", true); #endif + if ( ! ChunkAvailable() ) + read_flare.Extinguish(); + return false; } @@ -402,9 +416,15 @@ bool ChunkedIOFd::Read(Chunk** chunk, bool may_block) #ifdef DEBUG_COMMUNICATION AddToBuffer("", true); #endif + read_flare.Extinguish(); return true; } + if ( ChunkAvailable() ) + read_flare.Fire(); + else + read_flare.Extinguish(); + #ifdef DEBUG if ( *chunk ) DBG_LOG(DBG_CHUNKEDIO, "read of size %d %s[%s]", @@ -481,6 +501,9 @@ bool ChunkedIOFd::ReadChunk(Chunk** chunk, bool may_block) read_pos = 0; read_len = bytes_left; + if ( ! ChunkAvailable() ) + read_flare.Extinguish(); + // If allowed, wait a bit for something to read. if ( may_block ) { @@ -607,6 +630,14 @@ bool ChunkedIOFd::IsFillingUp() return stats.pending > MAX_BUFFERED_CHUNKS_SOFT; } +std::vector ChunkedIOFd::FdSupplements() const + { + std::vector rval; + rval.push_back(write_flare.FD()); + rval.push_back(read_flare.FD()); + return rval; + } + void ChunkedIOFd::Clear() { while ( pending_head ) @@ -618,6 +649,9 @@ void ChunkedIOFd::Clear() } pending_head = pending_tail = 0; + + if ( write_len == 0 ) + write_flare.Extinguish(); } const char* ChunkedIOFd::Error() @@ -830,6 +864,7 @@ bool ChunkedIOSSL::Write(Chunk* chunk) else write_head = write_tail = q; + write_flare.Fire(); Flush(); return true; } @@ -935,6 +970,7 @@ bool ChunkedIOSSL::Flush() write_state = LEN; } + write_flare.Extinguish(); return true; } @@ -1104,6 +1140,13 @@ bool ChunkedIOSSL::IsFillingUp() return false; } +std::vector ChunkedIOSSL::FdSupplements() const + { + std::vector rval; + rval.push_back(write_flare.FD()); + return rval; + } + void ChunkedIOSSL::Clear() { while ( write_head ) @@ -1114,6 +1157,7 @@ void ChunkedIOSSL::Clear() write_head = next; } write_head = write_tail = 0; + write_flare.Extinguish(); } const char* ChunkedIOSSL::Error() diff --git a/src/ChunkedIO.h b/src/ChunkedIO.h index a9865e4c05..c640e529b8 100644 --- a/src/ChunkedIO.h +++ b/src/ChunkedIO.h @@ -6,8 +6,9 @@ #include "config.h" #include "List.h" #include "util.h" - +#include "Flare.h" #include +#include #ifdef NEED_KRB5_H # include @@ -95,6 +96,11 @@ public: // Returns underlying fd if available, -1 otherwise. virtual int Fd() { return -1; } + // Returns supplementary file descriptors that become read-ready in order + // to signal that there is some work that can be performed. + virtual std::vector FdSupplements() const + { return std::vector(); } + // Makes sure that no additional protocol data is written into // the output stream. If this is activated, the output cannot // be read again by any of these classes! @@ -177,6 +183,7 @@ public: virtual void Clear(); virtual bool Eof() { return eof; } virtual int Fd() { return fd; } + virtual std::vector FdSupplements() const; virtual void Stats(char* buffer, int length); private: @@ -240,6 +247,8 @@ private: ChunkQueue* pending_tail; pid_t pid; + bro::Flare write_flare; + bro::Flare read_flare; }; // Chunked I/O using an SSL connection. @@ -262,6 +271,7 @@ public: virtual void Clear(); virtual bool Eof() { return eof; } virtual int Fd() { return socket; } + virtual std::vector FdSupplements() const; virtual void Stats(char* buffer, int length); private: @@ -303,6 +313,8 @@ private: // One SSL for all connections. static SSL_CTX* ctx; + + bro::Flare write_flare; }; #include @@ -328,6 +340,8 @@ public: virtual bool Eof() { return io->Eof(); } virtual int Fd() { return io->Fd(); } + virtual std::vector FdSupplements() const + { return io->FdSupplements(); } virtual void Stats(char* buffer, int length); void EnableCompression(int level) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 9188d61b96..9fb5c8bb87 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1217,9 +1217,10 @@ void DNS_Mgr::IssueAsyncRequests() } } -void DNS_Mgr::GetFds(int* read, int* write, int* except) +void DNS_Mgr::GetFds(std::vector* read, std::vector* write, + std::vector* except) { - *read = nb_dns_fd(nb_dns); + read->push_back(nb_dns_fd(nb_dns)); } double DNS_Mgr::NextTimestamp(double* network_time) diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index 7864505add..fa19914add 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -132,7 +132,8 @@ protected: void DoProcess(bool flush); // IOSource interface. - virtual void GetFds(int* read, int* write, int* except); + virtual void GetFds(std::vector* read, std::vector* write, + std::vector* except); virtual double NextTimestamp(double* network_time); virtual void Process(); virtual const char* Tag() { return "DNS_Mgr"; } diff --git a/src/Flare.cc b/src/Flare.cc new file mode 100644 index 0000000000..8a0418f631 --- /dev/null +++ b/src/Flare.cc @@ -0,0 +1,29 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Flare.h" +#include "util.h" +#include +#include +#include + +using namespace bro; + +Flare::Flare() + : pipe(FD_CLOEXEC, FD_CLOEXEC, O_NONBLOCK, O_NONBLOCK) + { + } + +void Flare::Fire() + { + char tmp; + safe_write(pipe.WriteFD(), &tmp, 1); + } + +void Flare::Extinguish() + { + char tmp[256]; + + for ( ; ; ) + if ( read(pipe.ReadFD(), &tmp, sizeof(tmp)) == -1 && errno == EAGAIN ) + break; + } diff --git a/src/Flare.h b/src/Flare.h new file mode 100644 index 0000000000..4e6378847a --- /dev/null +++ b/src/Flare.h @@ -0,0 +1,45 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef BRO_FLARE_H +#define BRO_FLARE_H + +#include "Pipe.h" + +namespace bro { + +class Flare { +public: + + /** + * Create a flare object that can be used to signal a "ready" status via + * a file descriptor that may be integrated with select(), poll(), etc. + * Not thread-safe, but that should only require Fire()/Extinguish() calls + * to be made mutually exclusive (across all copies of a Flare). + */ + Flare(); + + /** + * @return a file descriptor that will become ready if the flare has been + * Fire()'d and not yet Extinguished()'d. + */ + int FD() const + { return pipe.ReadFD(); } + + /** + * Put the object in the "ready" state. + */ + void Fire(); + + /** + * Take the object out of the "ready" state. + */ + void Extinguish(); + +private: + + Pipe pipe; +}; + +} // namespace bro + +#endif // BRO_FLARE_H diff --git a/src/FlowSrc.cc b/src/FlowSrc.cc index 8eed94fcea..4999d9cb97 100644 --- a/src/FlowSrc.cc +++ b/src/FlowSrc.cc @@ -28,10 +28,11 @@ FlowSrc::~FlowSrc() delete netflow_analyzer; } -void FlowSrc::GetFds(int* read, int* write, int* except) +void FlowSrc::GetFds(std::vector* read, std::vector* write, + std::vector* except) { if ( selectable_fd >= 0 ) - *read = selectable_fd; + read->push_back(selectable_fd); } double FlowSrc::NextTimestamp(double* network_time) diff --git a/src/FlowSrc.h b/src/FlowSrc.h index 03dda2761d..ee927604e1 100644 --- a/src/FlowSrc.h +++ b/src/FlowSrc.h @@ -34,7 +34,8 @@ public: // IOSource interface: bool IsReady(); - void GetFds(int* read, int* write, int* except); + void GetFds(std::vector* read, std::vector* write, + std::vector* except); double NextTimestamp(double* network_time); void Process(); diff --git a/src/IOSource.cc b/src/IOSource.cc index d47007caad..540b797162 100644 --- a/src/IOSource.cc +++ b/src/IOSource.cc @@ -24,6 +24,15 @@ void IOSourceRegistry::RemoveAll() dont_counts = sources.size(); } +static void fd_vector_set(const std::vector& fds, fd_set* set, int* max) + { + for ( size_t i = 0; i < fds.size(); ++i ) + { + FD_SET(fds[i], set); + *max = ::max(fds[i], *max); + } + } + IOSource* IOSourceRegistry::FindSoonest(double* ts) { // Remove sources which have gone dry. For simplicity, we only @@ -94,16 +103,14 @@ IOSource* IOSourceRegistry::FindSoonest(double* ts) // be ready. continue; - src->fd_read = src->fd_write = src->fd_except = 0; + src->fd_read.clear(); + src->fd_write.clear(); + src->fd_except.clear(); src->src->GetFds(&src->fd_read, &src->fd_write, &src->fd_except); - FD_SET(src->fd_read, &fd_read); - FD_SET(src->fd_write, &fd_write); - FD_SET(src->fd_except, &fd_except); - - maxx = max(src->fd_read, maxx); - maxx = max(src->fd_write, maxx); - maxx = max(src->fd_except, maxx); + fd_vector_set(src->fd_read, &fd_read, &maxx); + fd_vector_set(src->fd_write, &fd_write, &maxx); + fd_vector_set(src->fd_except, &fd_except, &maxx); } // We can't block indefinitely even when all sources are dry: @@ -143,9 +150,7 @@ IOSource* IOSourceRegistry::FindSoonest(double* ts) if ( ! src->src->IsIdle() ) continue; - if ( FD_ISSET(src->fd_read, &fd_read) || - FD_ISSET(src->fd_write, &fd_write) || - FD_ISSET(src->fd_except, &fd_except) ) + if ( src->Ready(&fd_read, &fd_write, &fd_except) ) { double local_network_time = 0; double ts = src->src->NextTimestamp(&local_network_time); @@ -174,3 +179,23 @@ void IOSourceRegistry::Register(IOSource* src, bool dont_count) ++dont_counts; return sources.push_back(s); } + +static bool fd_vector_ready(const std::vector& fds, fd_set* set) + { + for ( size_t i = 0; i < fds.size(); ++i ) + if ( FD_ISSET(fds[i], set) ) + return true; + + return false; + } + +bool IOSourceRegistry::Source::Ready(fd_set* read, fd_set* write, + fd_set* except) const + { + if ( fd_vector_ready(fd_read, read) || + fd_vector_ready(fd_write, write) || + fd_vector_ready(fd_except, except) ) + return true; + + return false; + } diff --git a/src/IOSource.h b/src/IOSource.h index db50bbd2a9..3da70af568 100644 --- a/src/IOSource.h +++ b/src/IOSource.h @@ -4,6 +4,8 @@ #define iosource_h #include +#include +#include #include "Timer.h" using namespace std; @@ -22,7 +24,8 @@ public: // Returns select'able fds (leaves args untouched if we don't have // selectable fds). - virtual void GetFds(int* read, int* write, int* except) = 0; + virtual void GetFds(std::vector* read, std::vector* write, + std::vector* except) = 0; // The following two methods are only called when either IsIdle() // returns false or select() on one of the fds indicates that there's @@ -89,9 +92,11 @@ protected: struct Source { IOSource* src; - int fd_read; - int fd_write; - int fd_except; + std::vector fd_read; + std::vector fd_write; + std::vector fd_except; + + bool Ready(fd_set* read, fd_set* write, fd_set* except) const; }; typedef list SourceList; diff --git a/src/Pipe.cc b/src/Pipe.cc new file mode 100644 index 0000000000..51298d07b6 --- /dev/null +++ b/src/Pipe.cc @@ -0,0 +1,79 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#include "Pipe.h" +#include "Reporter.h" +#include +#include +#include +#include + +using namespace bro; + +static void pipe_fail(int eno) + { + char tmp[256]; + strerror_r(eno, tmp, sizeof(tmp)); + reporter->FatalError("Pipe failure: %s", tmp); + } + +static void set_flags(int fd, int flags) + { + if ( flags ) + fcntl(fd, F_SETFD, fcntl(fd, F_GETFD) | flags); + } + +static void set_status_flags(int fd, int flags) + { + if ( flags ) + fcntl(fd, F_SETFL, fcntl(fd, F_GETFL) | flags); + } + +static int dup_or_fail(int fd, int flags) + { + int rval = dup(fd); + + if ( rval < 0 ) + pipe_fail(errno); + + set_flags(fd, flags); + return rval; + } + +Pipe::Pipe(int flags0, int flags1, int status_flags0, int status_flags1) + { + // pipe2 can set flags atomically, but not yet available everywhere. + if ( ::pipe(fds) ) + pipe_fail(errno); + + flags[0] = flags0; + flags[1] = flags1; + + set_flags(fds[0], flags[0]); + set_flags(fds[1], flags[1]); + set_status_flags(fds[0], status_flags0); + set_status_flags(fds[1], status_flags1); + } + +Pipe::~Pipe() + { + close(fds[0]); + close(fds[1]); + } + +Pipe::Pipe(const Pipe& other) + { + fds[0] = dup_or_fail(other.fds[0], other.flags[0]); + fds[1] = dup_or_fail(other.fds[1], other.flags[1]); + } + +Pipe& Pipe::operator=(const Pipe& other) + { + if ( this == &other ) + return *this; + + close(fds[0]); + close(fds[1]); + fds[0] = dup_or_fail(other.fds[0], other.flags[0]); + fds[1] = dup_or_fail(other.fds[1], other.flags[1]); + return *this; + } diff --git a/src/Pipe.h b/src/Pipe.h new file mode 100644 index 0000000000..493169e615 --- /dev/null +++ b/src/Pipe.h @@ -0,0 +1,57 @@ +// See the file "COPYING" in the main distribution directory for copyright. + +#ifndef BRO_PIPE_H +#define BRO_PIPE_H + +namespace bro { + +class Pipe { +public: + + /** + * Create a pair of file descriptors via pipe(), or aborts if it cannot. + * @param flags0 file descriptor flags to set on read end of pipe. + * @param flags1 file descriptor flags to set on write end of pipe. + * @param status_flags0 descriptor status flags to set on read end of pipe. + * @param status_flags1 descriptor status flags to set on write end of pipe. + */ + Pipe(int flags0 = 0, int flags1 = 0, int status_flags0 = 0, + int status_flags1 = 0); + + /** + * Close the pair of file descriptors owned by the object. + */ + ~Pipe(); + + /** + * Make a copy of another Pipe object (file descriptors are dup'd). + */ + Pipe(const Pipe& other); + + /** + * Assign a Pipe object by closing file descriptors and duping those of + * the other. + */ + Pipe& operator=(const Pipe& other); + + /** + * @return the file descriptor associated with the read-end of the pipe. + */ + int ReadFD() const + { return fds[0]; } + + /** + * @return the file descriptor associated with the write-end of the pipe. + */ + int WriteFD() const + { return fds[1]; } + +private: + + int fds[2]; + int flags[2]; +}; + +} // namespace bro + +#endif // BRO_PIPE_H diff --git a/src/PktSrc.cc b/src/PktSrc.cc index b5ac3a5d69..04b7b7d552 100644 --- a/src/PktSrc.cc +++ b/src/PktSrc.cc @@ -51,7 +51,8 @@ PktSrc::~PktSrc() delete [] readfile; } -void PktSrc::GetFds(int* read, int* write, int* except) +void PktSrc::GetFds(std::vector* read, std::vector* write, + std::vector* except) { if ( pseudo_realtime ) { @@ -62,7 +63,7 @@ void PktSrc::GetFds(int* read, int* write, int* except) } if ( selectable_fd >= 0 ) - *read = selectable_fd; + read->push_back(selectable_fd); } int PktSrc::ExtractNextPacket() diff --git a/src/PktSrc.h b/src/PktSrc.h index 70eef4dd00..0d4be12b43 100644 --- a/src/PktSrc.h +++ b/src/PktSrc.h @@ -98,7 +98,8 @@ public: // IOSource interface bool IsReady(); - void GetFds(int* read, int* write, int* except); + void GetFds(std::vector* read, std::vector* write, + std::vector* except); double NextTimestamp(double* local_network_time); void Process(); const char* Tag() { return "PktSrc"; } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 3e46c5a1d2..34c5f1abce 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -1368,12 +1368,17 @@ void RemoteSerializer::Unregister(ID* id) } } -void RemoteSerializer::GetFds(int* read, int* write, int* except) +void RemoteSerializer::GetFds(std::vector* read, std::vector* write, + std::vector* except) { - *read = io->Fd(); + read->push_back(io->Fd()); + std::vector supp = io->FdSupplements(); + + for ( size_t i = 0; i < supp.size(); ++i ) + read->push_back(supp[i]); if ( io->CanWrite() ) - *write = io->Fd(); + write->push_back(io->Fd()); } double RemoteSerializer::NextTimestamp(double* local_network_time) @@ -3356,6 +3361,15 @@ SocketComm::~SocketComm() static unsigned int first_rtime = 0; +static void fd_vector_set(const std::vector& fds, fd_set* set, int* max) + { + for ( size_t i = 0; i < fds.size(); ++i ) + { + FD_SET(fds[i], set); + *max = ::max(fds[i], *max); + } + } + void SocketComm::Run() { first_rtime = (unsigned int) current_time(true); @@ -3381,6 +3395,7 @@ void SocketComm::Run() FD_SET(io->Fd(), &fd_read); max_fd = io->Fd(); + fd_vector_set(io->FdSupplements(), &fd_read, &max_fd); loop_over_list(peers, i) { @@ -3389,6 +3404,7 @@ void SocketComm::Run() FD_SET(peers[i]->io->Fd(), &fd_read); if ( peers[i]->io->Fd() > max_fd ) max_fd = peers[i]->io->Fd(); + fd_vector_set(peers[i]->io->FdSupplements(), &fd_read, &max_fd); } else { @@ -3439,38 +3455,17 @@ void SocketComm::Run() if ( ! io->IsFillingUp() && shutting_conns_down ) shutting_conns_down = false; - // We cannot rely solely on select() as the there may - // be some data left in our input/output queues. So, we use - // a small timeout for select and check for data - // manually afterwards. - static long selects = 0; static long canwrites = 0; - static long timeouts = 0; ++selects; if ( io->CanWrite() ) ++canwrites; - // FIXME: Fine-tune this (timeouts, flush, etc.) - struct timeval small_timeout; - small_timeout.tv_sec = 0; - small_timeout.tv_usec = - io->CanWrite() || io->CanRead() ? 1 : 10; - -#if 0 - if ( ! io->CanWrite() ) - usleep(10); -#endif - - int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, - &small_timeout); - - if ( a == 0 ) - ++timeouts; + int a = select(max_fd + 1, &fd_read, &fd_write, &fd_except, 0); if ( selects % 100000 == 0 ) - Log(fmt("selects=%ld canwrites=%ld timeouts=%ld", selects, canwrites, timeouts)); + Log(fmt("selects=%ld canwrites=%ld", selects, canwrites)); if ( a < 0 ) // Ignore errors for now. diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index 9dbfbd9dae..3aa4f91bb0 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -140,7 +140,8 @@ public: void Finish(); // Overidden from IOSource: - virtual void GetFds(int* read, int* write, int* except); + virtual void GetFds(std::vector* read, std::vector* write, + std::vector* except); virtual double NextTimestamp(double* local_network_time); virtual void Process(); virtual TimerMgr::Tag* GetCurrentTag(); diff --git a/src/Serializer.cc b/src/Serializer.cc index 36b1c74000..0ea79cfafb 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -1067,9 +1067,10 @@ void EventPlayer::GotFunctionCall(const char* name, double time, // We don't replay function calls. } -void EventPlayer::GetFds(int* read, int* write, int* except) +void EventPlayer::GetFds(std::vector* read, std::vector* write, + std::vector* except) { - *read = fd; + read->push_back(fd); } double EventPlayer::NextTimestamp(double* local_network_time) diff --git a/src/Serializer.h b/src/Serializer.h index 543797a7af..0524906d48 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -355,7 +355,8 @@ public: EventPlayer(const char* file); virtual ~EventPlayer(); - virtual void GetFds(int* read, int* write, int* except); + virtual void GetFds(std::vector* read, std::vector* write, + std::vector* except); virtual double NextTimestamp(double* local_network_time); virtual void Process(); virtual const char* Tag() { return "EventPlayer"; } diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 4491cd42b5..c16b9f4351 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -65,7 +65,8 @@ void Manager::AddMsgThread(MsgThread* thread) msg_threads.push_back(thread); } -void Manager::GetFds(int* read, int* write, int* except) +void Manager::GetFds(std::vector* read, std::vector* write, + std::vector* except) { } diff --git a/src/threading/Manager.h b/src/threading/Manager.h index e839749a91..4f0e53928e 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -103,7 +103,8 @@ protected: /** * Part of the IOSource interface. */ - virtual void GetFds(int* read, int* write, int* except); + virtual void GetFds(std::vector* read, std::vector* write, + std::vector* except); /** * Part of the IOSource interface. From 77955d76772eea871f238e4c4cb4da9a3896db43 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 3 Sep 2014 09:51:34 -0500 Subject: [PATCH 08/71] Fix possible abort on writing to a full pipe. --- src/Flare.cc | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/Flare.cc b/src/Flare.cc index 8a0418f631..960e66cbf4 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -1,7 +1,6 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "Flare.h" -#include "util.h" #include #include #include @@ -16,7 +15,21 @@ Flare::Flare() void Flare::Fire() { char tmp; - safe_write(pipe.WriteFD(), &tmp, 1); + + for ( ; ; ) + { + int n = write(pipe.WriteFD(), &tmp, 1); + + if ( n > 0 ) + // Success -- wrote a byte to pipe. + break; + + if ( n < 0 && errno == EAGAIN ) + // Success -- pipe is full and just need at least one byte in it. + break; + + // Loop because either the byte wasn't written or got EINTR error. + } } void Flare::Extinguish() @@ -25,5 +38,6 @@ void Flare::Extinguish() for ( ; ; ) if ( read(pipe.ReadFD(), &tmp, sizeof(tmp)) == -1 && errno == EAGAIN ) + // Pipe is now drained. break; } From 09214652975c5f9f5b92bcee7c12f325a3a7b37b Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 3 Sep 2014 16:23:13 -0500 Subject: [PATCH 09/71] Fix Pipe copy/assignment to make a copy of flags. --- src/Pipe.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Pipe.cc b/src/Pipe.cc index 51298d07b6..3f60409fdb 100644 --- a/src/Pipe.cc +++ b/src/Pipe.cc @@ -64,6 +64,8 @@ Pipe::Pipe(const Pipe& other) { fds[0] = dup_or_fail(other.fds[0], other.flags[0]); fds[1] = dup_or_fail(other.fds[1], other.flags[1]); + flags[0] = other.flags[0]; + flags[1] = other.flags[1]; } Pipe& Pipe::operator=(const Pipe& other) @@ -75,5 +77,7 @@ Pipe& Pipe::operator=(const Pipe& other) close(fds[1]); fds[0] = dup_or_fail(other.fds[0], other.flags[0]); fds[1] = dup_or_fail(other.fds[1], other.flags[1]); + flags[0] = other.flags[0]; + flags[1] = other.flags[1]; return *this; } From 5c9a7a92a49d5854a2732c03b72d5517bbf91888 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Thu, 4 Sep 2014 13:32:24 -0500 Subject: [PATCH 10/71] Add more script language reference documentation Added new sections on operators, statements, and directives. Also improved the documentation on types and attributes by providing more examples and added a chart on the top of each page with links to each type and attribute for easier access to the information. --- doc/ext/bro.py | 7 + doc/script-reference/attributes.rst | 179 +++++++-- doc/script-reference/directives.rst | 173 ++++++++ doc/script-reference/index.rst | 3 + doc/script-reference/operators.rst | 179 +++++++++ doc/script-reference/statements.rst | 602 ++++++++++++++++++++++++++++ doc/script-reference/types.rst | 523 +++++++++++++----------- 7 files changed, 1385 insertions(+), 281 deletions(-) create mode 100644 doc/script-reference/directives.rst create mode 100644 doc/script-reference/operators.rst create mode 100644 doc/script-reference/statements.rst diff --git a/doc/ext/bro.py b/doc/ext/bro.py index 9295c63312..1df4a518c2 100644 --- a/doc/ext/bro.py +++ b/doc/ext/bro.py @@ -176,6 +176,10 @@ class BroIdentifier(BroGeneric): def get_index_text(self, objectname, name): return name +class BroKeyword(BroGeneric): + def get_index_text(self, objectname, name): + return name + class BroAttribute(BroGeneric): def get_index_text(self, objectname, name): return _('%s (attribute)') % (name) @@ -213,6 +217,7 @@ class BroDomain(Domain): 'type': ObjType(l_('type'), 'type'), 'namespace': ObjType(l_('namespace'), 'namespace'), 'id': ObjType(l_('id'), 'id'), + 'keyword': ObjType(l_('keyword'), 'keyword'), 'enum': ObjType(l_('enum'), 'enum'), 'attr': ObjType(l_('attr'), 'attr'), } @@ -221,6 +226,7 @@ class BroDomain(Domain): 'type': BroGeneric, 'namespace': BroNamespace, 'id': BroIdentifier, + 'keyword': BroKeyword, 'enum': BroEnum, 'attr': BroAttribute, } @@ -229,6 +235,7 @@ class BroDomain(Domain): 'type': XRefRole(), 'namespace': XRefRole(), 'id': XRefRole(), + 'keyword': XRefRole(), 'enum': XRefRole(), 'attr': XRefRole(), 'see': XRefRole(), diff --git a/doc/script-reference/attributes.rst b/doc/script-reference/attributes.rst index ca66ab2112..0160499fb9 100644 --- a/doc/script-reference/attributes.rst +++ b/doc/script-reference/attributes.rst @@ -1,38 +1,120 @@ Attributes ========== -Attributes occur at the end of type or event declarations and change their -behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T: -set[count] &read_expire=5min`` or ``event foo() &priority=-3``. The Bro -scripting language supports the following attributes. +The Bro scripting language supports the following attributes. -.. bro:attr:: &optional ++-----------------------------+-----------------------------------------------+ +| Name | Description | ++=============================+===============================================+ +| :bro:attr:`&redef` |Redefine a global constant or extend a type. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&priority` |Specify priority for event handler or hook. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&log` |Mark a record field to be written to a log. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&optional` |Allow a record field value to be missing. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&default` |Specifies a default value. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&add_func` |Specify a function to call for each "redef +=".| ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&delete_func` |Same as "&add_func", except for "redef -=". | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&expire_func` |Specify a function to call when container | +| |element expires. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&read_expire` |Specify a read timeout interval. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&write_expire` |Specify a write timeout interval. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&create_expire` |Specify a creation timeout interval. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&synchronized` |Synchronize a variable across nodes. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&persistent` |Make a variable persistent (written to disk). | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&rotate_interval`|Rotate a file after specified interval. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&rotate_size` |Rotate a file after specified file size. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&encrypt` |Encrypt a file when writing to disk. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&raw_output` |Open file in raw mode (chars. are not escaped).| ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&mergeable` |Prefer set union for synchronized state. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&group` |Group event handlers to activate/deactivate. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&error_handler` |Used internally for reporter framework events. | ++-----------------------------+-----------------------------------------------+ +| :bro:attr:`&type_column` |Used by input framework for "port" type. | ++-----------------------------+-----------------------------------------------+ - Allows a record field to be missing. For example the type ``record { - a: addr; b: port &optional; }`` could be instantiated both as - singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``. - -.. bro:attr:: &default - - Uses a default value for a record field, a function/hook/event - parameter, or container elements. For example, ``table[int] of - string &default="foo"`` would create a table that returns the - :bro:type:`string` ``"foo"`` for any non-existing index. +Here is a more detailed explanation of each attribute: .. bro:attr:: &redef - Allows for redefinition of initial object values. This is typically - used with constants, for example, ``const clever = T &redef;`` would - allow the constant to be redefined at some later point during script - execution. + Allows for redefinition of initial values of global objects declared as + constant. -.. bro:attr:: &rotate_interval + In this example, the constant (assuming it is global) can be redefined + with a :bro:keyword:`redef` at some later point:: - Rotates a file after a specified interval. + const clever = T &redef; -.. bro:attr:: &rotate_size +.. bro:attr:: &priority - Rotates a file after it has reached a given size in bytes. + Specifies the execution priority (as a signed integer) of a hook or + event handler. Higher values are executed before lower ones. The + default value is 0. Example:: + + event bro_init() &priority=10 + { + print "high priority"; + } + +.. bro:attr:: &log + + Writes a :bro:type:`record` field to the associated log stream. + +.. bro:attr:: &optional + + Allows a record field value to be missing (i.e., neither initialized nor + ever assigned a value). + + In this example, the record could be instantiated with either + "myrec($a=127.0.0.1)" or "myrec($a=127.0.0.1, $b=80/tcp)":: + + type myrec: record { a: addr; b: port &optional; }; + + The ``?$`` operator can be used to check if a record field has a value or + not (it returns a ``bool`` value of ``T`` if the field has a value, + and ``F`` if not). + +.. bro:attr:: &default + + Specifies a default value for a record field, container element, or a + function/hook/event parameter. + + In this example, the record could be instantiated with either + "myrec($a=5, $c=3.14)" or "myrec($a=5, $b=53/udp, $c=3.14)":: + + type myrec: record { a: count; b: port &default=80/tcp; c: double; }; + + In this example, the table will return the string ``"foo"`` for any + attempted access to a non-existing index:: + + global mytable: table[count] of string &default="foo"; + + When used with function/hook/event parameters, all of the parameters + with the "&default" attribute must come after all other parameters. + For example, the following function could be called either as "myfunc(5)" + or as "myfunc(5, 53/udp)":: + + function myfunc(a: count, b: port &default=80/tcp) + { + print a, b; + } .. bro:attr:: &add_func @@ -46,8 +128,8 @@ scripting language supports the following attributes. .. bro:attr:: &delete_func - Same as &add_func, except for "redef" declarations that use the "-=" - operator. + Same as :bro:attr:`&add_func`, except for :bro:keyword:`redef` declarations + that use the "-=" operator. .. bro:attr:: &expire_func @@ -76,23 +158,29 @@ scripting language supports the following attributes. is, the element expires after the given amount of time since it has been inserted into the container, regardless of any reads or writes. -.. bro:attr:: &persistent - - Makes a variable persistent, i.e., its value is written to disk (per - default at shutdown time). - .. bro:attr:: &synchronized Synchronizes variable accesses across nodes. The value of a ``&synchronized`` variable is automatically propagated to all peers when it changes. +.. bro:attr:: &persistent + + Makes a variable persistent, i.e., its value is written to disk (per + default at shutdown time). + +.. bro:attr:: &rotate_interval + + Rotates a file after a specified interval. + +.. bro:attr:: &rotate_size + + Rotates a file after it has reached a given size in bytes. + .. bro:attr:: &encrypt Encrypts files right before writing them to disk. -.. TODO: needs to be documented in more detail. - .. bro:attr:: &raw_output Opens a file in raw mode, i.e., non-ASCII characters are not @@ -108,21 +196,11 @@ scripting language supports the following attributes. inconsistencies and can be avoided by unifying the two sets, rather than merely overwriting the old value. -.. bro:attr:: &priority - - Specifies the execution priority (as a signed integer) of a hook or - event handler. Higher values are executed before lower ones. The - default value is 0. - .. bro:attr:: &group Groups event handlers such that those in the same group can be jointly activated or deactivated. -.. bro:attr:: &log - - Writes a record field to the associated log stream. - .. bro:attr:: &error_handler Internally set on the events that are associated with the reporter @@ -135,5 +213,20 @@ scripting language supports the following attributes. .. bro:attr:: &type_column Used by the input framework. It can be used on columns of type - :bro:type:`port` and specifies the name of an additional column in + :bro:type:`port` (such a column only contains the port number) and + specifies the name of an additional column in the input file which specifies the protocol of the port (tcp/udp/icmp). + + In the following example, the input file would contain four columns + named "ip", "srcp", "proto", and "msg":: + + type Idx: record { + ip: addr; + }; + + + type Val: record { + srcp: port &type_column = "proto"; + msg: string; + }; + diff --git a/doc/script-reference/directives.rst b/doc/script-reference/directives.rst new file mode 100644 index 0000000000..e513e93911 --- /dev/null +++ b/doc/script-reference/directives.rst @@ -0,0 +1,173 @@ +Directives +========== + +The Bro scripting language supports a number of directives that can +affect which scripts will be loaded or which lines in a script will be +executed. Directives are evaluated before script execution begins. + +.. bro:keyword:: @DEBUG + + TODO + + +.. bro:keyword:: @DIR + + Expands to the directory pathname where the current script is located. + + Example:: + + print "Directory:", @DIR + + +.. bro:keyword:: @FILENAME + + Expands to the filename of the current script. + + Example:: + + print "File:", @FILENAME + +.. bro:keyword:: @load + + Loads the specified Bro script, specified as the relative pathname + of the file (relative to one of the directories in Bro's file search path). + If the Bro script filename ends with ".bro", then you don't need to + specify the file extension. The filename cannot contain any whitespace. + + In this example, Bro will try to load a script + "policy/misc/capture-loss.bro" by looking in each directory in the file + search path (the file search path can be changed by setting the BROPATH + environment variable):: + + @load policy/misc/capture-loss + + If you specify the name of a directory instead of a filename, then + Bro will try to load a file in that directory called "__load__.bro" + (presumably that file will contain additional "@load" directives). + + In this example, Bro will try to load a file "tuning/defaults/__load__.bro" + by looking in each directory in the file search path:: + + @load tuning/defaults + + The purpose of this directive is to ensure that all script dependencies + are satisfied, and to avoid having to list every needed Bro script + on the command-line. Bro keeps track of which scripts have been + loaded, so it is not an error to load a script more than once (once + a script has been loaded, any subsequent "@load" directives + for that script are ignored). + + +.. bro:keyword:: @load-sigs + + This works similarly to "@load", except that in this case the filename + represents a signature file (not a Bro script). If the signature filename + ends with ".sig", then you don't need to specify the file extension + in the "@load-sigs" directive. The filename cannot contain any + whitespace. + + In this example, Bro will try to load a signature file + "base/protocols/ssl/dpd.sig":: + + @load-sigs base/protocols/ssl/dpd + + The format for a signature file is explained in the documentation for the + `Signature Framework <../frameworks/signatures.html>`_. + + +.. bro:keyword:: @unload + + This specifies a Bro script that we don't want to load (so a subsequent + attempt to load the specified script will be skipped). However, + if the specified script has already been loaded, then this directive + has no affect. + + In the following example, if the "policy/misc/capture-loss.bro" script + has not been loaded yet, then Bro will not load it:: + + @unload policy/misc/capture-loss + + +.. bro:keyword:: @prefixes + + Specifies a filename prefix to use when looking for script files + to load automatically. The prefix cannot contain any whitespace. + + In the following example, the prefix "cluster" is used and all prefixes + that were previously specified are not used:: + + @prefixes = cluster + + In the following example, the prefix "cluster-manager" is used in + addition to any previously-specified prefixes:: + + @prefixes += cluster-manager + + The way this works is that after Bro parses all script files, then for each + loaded script Bro will take the absolute path of the script and then + it removes the portion of the directory path that is in Bro's file + search path. Then it replaces each "/" character with a period "." + and then prepends the prefix (specified in the "@prefixes" directive) + followed by a period. The resulting filename is searched for in each + directory in Bro's file search path. If a matching file is found, then + the file is automatically loaded. + + For example, if a script called "local.bro" has been loaded, and a prefix + of "test" was specified, then Bro will look for a file named + "test.local.bro" in each directory of Bro's file search path. + + An alternative way to specify prefixes is to use the "-p" Bro + command-line option. + +.. bro:keyword:: @if + + The specified expression must evaluate to type :bro:type:`bool`. If the + value is true, then the following script lines (up to the next "@else" + or "@endif") are available to be executed. + + Example:: + + @if ( ver == 2 ) + print "version 2 detected"; + @endif + +.. bro:keyword:: @ifdef + + This works like "@if", except that the result is true if the specified + identifier is defined. + + Example:: + + @ifdef ( pi ) + print "pi is defined"; + @endif + +.. bro:keyword:: @ifndef + + This works exactly like "@ifdef", except that the result is true if the + specified identifier is not defined. + + Example:: + + @ifndef ( pi ) + print "pi is not defined"; + @endif + +.. bro:keyword:: @else + + This directive is optional after an "@if", "@ifdef", or + "@ifndef". If present, it provides an else clause. + + Example:: + + @ifdef ( pi ) + print "pi is defined"; + @else + print "pi is not defined"; + @endif + +.. bro:keyword:: @endif + + This directive is required to terminate each "@if", "@ifdef", or + "@ifndef". + diff --git a/doc/script-reference/index.rst b/doc/script-reference/index.rst index a2c6f0a24f..dc11447c5f 100644 --- a/doc/script-reference/index.rst +++ b/doc/script-reference/index.rst @@ -5,8 +5,11 @@ Script Reference .. toctree:: :maxdepth: 1 + operators types attributes + statements + directives notices proto-analyzers file-analyzers diff --git a/doc/script-reference/operators.rst b/doc/script-reference/operators.rst new file mode 100644 index 0000000000..7fa52cf4b2 --- /dev/null +++ b/doc/script-reference/operators.rst @@ -0,0 +1,179 @@ +Operators +========= + +The Bro scripting language supports the following operators. Note that +each data type only supports a subset of these operators. For more +details, see the documentation about the `data types `_. + +Relational operators +-------------------- + +The relational operators evaluate to type :bro:type:`bool`. + ++------------------------------+--------------+ +| Name | Syntax | ++==============================+==============+ +| Equality | *a* == *b* | ++------------------------------+--------------+ +| Inequality | *a* != *b* | ++------------------------------+--------------+ +| Less than | *a* < *b* | ++------------------------------+--------------+ +| Less than or equal | *a* <= *b* | ++------------------------------+--------------+ +| Greater than | *a* > *b* | ++------------------------------+--------------+ +| Greater than or equal | *a* >= *b* | ++------------------------------+--------------+ + + +Logical operators +----------------- + +The logical operators require operands of type :bro:type:`bool`, and +evaluate to type :bro:type:`bool`. + ++------------------------------+--------------+ +| Name | Syntax | ++==============================+==============+ +| Logical AND | *a* && *b* | ++------------------------------+--------------+ +| Logical OR | *a* \|\| *b* | ++------------------------------+--------------+ +| Logical NOT | ! *a* | ++------------------------------+--------------+ + + +Arithmetic operators +-------------------- + ++------------------------------+-------------+-------------------------------+ +| Name | Syntax | Notes | ++==============================+=============+===============================+ +| Addition | *a* + *b* | If operands are strings, then | +| | | this performs string | +| | | concatenation. | ++------------------------------+-------------+-------------------------------+ +| Subtraction | *a* - *b* | | ++------------------------------+-------------+-------------------------------+ +| Multiplication | *a* \* *b* | | ++------------------------------+-------------+-------------------------------+ +| Division | *a* / *b* | | ++------------------------------+-------------+-------------------------------+ +| Modulo | *a* % *b* | Operand types cannot be | +| | | double. | ++------------------------------+-------------+-------------------------------+ +| Unary plus | \+ *a* | | ++------------------------------+-------------+-------------------------------+ +| Unary minus | \- *a* | | ++------------------------------+-------------+-------------------------------+ +| Pre-increment | ++ *a* | Operand type cannot be | +| | | double. | ++------------------------------+-------------+-------------------------------+ +| Pre-decrement | ``--`` *a* | Operand type cannot be | +| | | double. | ++------------------------------+-------------+-------------------------------+ +| Absolute value | \| *a* \| | If operand is string, set, | +| | | table, or vector, this | +| | | evaluates to number | +| | | of elements. | ++------------------------------+-------------+-------------------------------+ + + +Assignment operators +-------------------- + +The assignment operators evaluate to the result of the assignment. + ++------------------------------+-------------+ +| Name | Syntax | ++==============================+=============+ +| Assignment | *a* = *b* | ++------------------------------+-------------+ +| Addition assignment | *a* += *b* | ++------------------------------+-------------+ +| Subtraction assignment | *a* -= *b* | ++------------------------------+-------------+ + + +Record field operators +---------------------- + +The record field operators take a :bro:type:`record` as the first operand, +and a field name as the second operand. For both operators, the specified +field name must be in the declaration of the record type. + ++------------------------------+-------------+-------------------------------+ +| Name | Syntax | Notes | ++==============================+=============+===============================+ +| Field access | *a* $ *b* | | ++------------------------------+-------------+-------------------------------+ +| Field value existence test | *a* ?$ *b* | Evaluates to type "bool". | +| | | True if the specified field | +| | | has been assigned a value, or | +| | | false if not. | ++------------------------------+-------------+-------------------------------+ + + +Other operators +--------------- + ++--------------------------------+-------------------+------------------------+ +| Name | Syntax | Notes | ++================================+===================+========================+ +| Membership test | *a* in *b* |Evaluates to type | +| | |"bool". Do not | +| | |confuse this use of "in"| +| | |with that used in a | +| | |:bro:keyword:`for` | +| | |statement. | ++--------------------------------+-------------------+------------------------+ +| Non-membership test | *a* !in *b* |This is the logical NOT | +| | |of the "in" operator. | +| | |For example: "a !in b" | +| | |is equivalent to | +| | |"!(a in b)". | ++--------------------------------+-------------------+------------------------+ +| Table or vector element access | *a* [ *b* ] |This operator can also | +| | |be used with a set, but | +| | |only with the | +| | |:bro:keyword:`add` or | +| | |:bro:keyword:`delete` | +| | |statement. | ++--------------------------------+-------------------+------------------------+ +| Substring extraction | *a* [ *b* : *c* ] |See the | +| | |:bro:type:`string` type | +| | |for more details. | ++--------------------------------+-------------------+------------------------+ +| Create a deep copy | copy ( *a* ) |This is relevant only | +| | |for data types that are | +| | |assigned by reference, | +| | |such as "vector", "set",| +| | |"table", and "record". | ++--------------------------------+-------------------+------------------------+ +| Module namespace access | *a* \:\: *b* |The first operand is the| +| | |module name, and the | +| | |second operand is an | +| | |identifier that refers | +| | |to a global variable, | +| | |enumeration constant, or| +| | |user-defined type that | +| | |was exported from the | +| | |module. | ++--------------------------------+-------------------+------------------------+ +| Conditional | *a* ? *b* : *c* |The first operand must | +| | |evaluate to a "bool" | +| | |type. If true, then the| +| | |second expression is | +| | |evaluated and is the | +| | |result of the entire | +| | |expression. Otherwise, | +| | |the third expression is | +| | |evaluated and is the | +| | |result of the entire | +| | |expression. The types of| +| | |the second and third | +| | |operands must be | +| | |compatible. | ++--------------------------------+-------------------+------------------------+ + diff --git a/doc/script-reference/statements.rst b/doc/script-reference/statements.rst new file mode 100644 index 0000000000..9c8b8de20d --- /dev/null +++ b/doc/script-reference/statements.rst @@ -0,0 +1,602 @@ +Declarations and Statements +=========================== + +The Bro scripting language supports the following declarations and +statements. + + +Declarations +~~~~~~~~~~~~ + ++----------------------------+-----------------------------+ +| Name | Description | ++============================+=============================+ +| :bro:keyword:`module` | Change the current module | ++----------------------------+-----------------------------+ +| :bro:keyword:`export` | Export identifiers from the | +| | current module | ++----------------------------+-----------------------------+ +| :bro:keyword:`global` | Declare a global variable | ++----------------------------+-----------------------------+ +| :bro:keyword:`const` | Declare a constant | ++----------------------------+-----------------------------+ +| :bro:keyword:`type` | Declare a user-defined type | ++----------------------------+-----------------------------+ +| :bro:keyword:`redef` | Redefine a global value or | +| | extend a user-defined type | ++----------------------------+-----------------------------+ +| `function/event/hook`_ | Declare a function, event | +| | handler, or hook | ++----------------------------+-----------------------------+ + +Statements +~~~~~~~~~~ + ++----------------------------+------------------------+ +| Name | Description | ++============================+========================+ +| :bro:keyword:`local` | Declare a local | +| | variable | ++----------------------------+------------------------+ +| :bro:keyword:`add`, | Add or delete | +| :bro:keyword:`delete` | elements | ++----------------------------+------------------------+ +| :bro:keyword:`print` | Print to stdout or a | +| | file | ++----------------------------+------------------------+ +| :bro:keyword:`for`, | Loop over each | +| :bro:keyword:`next`, | element in a container | +| :bro:keyword:`break` | object | ++----------------------------+------------------------+ +| :bro:keyword:`if` | Evaluate boolean | +| | expression and if true,| +| | execute a statement | ++----------------------------+------------------------+ +| :bro:keyword:`switch`, | Evaluate expression | +| :bro:keyword:`break`, | and execute statement | +| :bro:keyword:`fallthrough` | with a matching value | ++----------------------------+------------------------+ +| :bro:keyword:`when` | Asynchronous execution | ++----------------------------+------------------------+ +| :bro:keyword:`event`, | Invoke or schedule | +| :bro:keyword:`schedule` | an event handler | ++----------------------------+------------------------+ +| :bro:keyword:`return` | Return from function, | +| | hook, or event handler | ++----------------------------+------------------------+ + +Declarations +------------ + +The following global declarations cannot occur within a function, hook, or +event handler. Also, these declarations cannot appear after any statements +that are outside of a function, hook, or event handler. + +.. bro:keyword:: module + + The "module" keyword is used to change the current module. This + affects the scope of any subsequently declared global identifiers. + + Example:: + + module mymodule; + + If a global identifier is declared after a "module" declaration, + then its scope ends at the end of the current Bro script or at the + next "module" declaration, whichever comes first. However, if a + global identifier is declared after a "module" declaration, but inside + an :bro:keyword:`export` block, then its scope ends at the end of the + last loaded Bro script, but it must be referenced using the namespace + operator (``::``) in other modules. + + There can be any number of "module" declarations in a Bro script. + The same "module" declaration can appear in any number of different + Bro scripts. + + +.. bro:keyword:: export + + An "export" block contains one or more declarations + (no statements are allowed in an "export" block) that the current + module is exporting. This enables these global identifiers to be visible + in other modules (but not prior to their declaration) via the namespace + operator (``::``). See the :bro:keyword:`module` keyword for a more + detailed explanation. + + Example:: + + export { + redef enum Log::ID += { LOG }; + + type Info: record { + ts: time &log; + uid: string &log; + }; + + const conntime = 30sec &redef; + } + + Note that the braces in an "export" block are always required + (they do not indicate a compound statement). Also, no semicolon is + needed to terminate an "export" block. + +.. bro:keyword:: global + + Variables declared with the "global" keyword will be global. + If a type is not specified, then an initializer is required so that + the type can be inferred. Likewise, if an initializer is not supplied, + then the type must be specified. Example:: + + global pi = 3.14; + global hosts: set[addr]; + global ciphers: table[string] of string = table(); + + Variable declarations outside of any function, hook, or event handler are + required to use this keyword (unless they are declared with the + :bro:keyword:`const` keyword). Definitions of functions, hooks, and + event handlers are not allowed to use the "global" + keyword (they already have global scope), except function declarations + where no function body is supplied use the "global" keyword. + + The scope of a global variable begins where the declaration is located, + and extends through all remaining Bro scripts that are loaded (however, + see the :bro:keyword:`module` keyword for an explanation of how modules + change the visibility of global identifiers). + + +.. bro:keyword:: const + + A variable declared with the "const" keyword will be constant. + Variables declared as constant are required to be initialized at the + time of declaration. Example:: + + const pi = 3.14; + const ssh_port: port = 22/tcp; + + The value of a constant cannot be changed later (the only + exception is if the variable is global and has the :bro:attr:`&redef` + attribute, then its value can be changed only with a :bro:keyword:`redef`). + + The scope of a constant is local if the declaration is in a + function, hook, or event handler, and global otherwise. + Note that the "const" keyword cannot be used with either the "local" + or "global" keywords (i.e., "const" replaces "local" and "global"). + + +.. bro:keyword:: type + + The "type" keyword is used to declare a user-defined type. The name + of this new type has global scope and can be used anywhere a built-in + type name can occur. + + The "type" keyword is most commonly used when defining a + :bro:type:`record` or an :bro:type:`enum`, but is also useful when + dealing with more complex types. + + Example:: + + type mytype: table[count] of table[addr, port] of string; + global myvar: mytype; + +.. bro:keyword:: redef + + There are three ways that "redef" can be used: to change the value of + a global variable, to extend a record type or enum type, or to specify + a new event handler body that replaces all those that were previously + defined. + + If you're using "redef" to change a global variable (defined using either + :bro:keyword:`const` or :bro:keyword:`global`), then the variable that you + want to change must have the :bro:attr:`&redef` attribute. If the variable + you're changing is a table, set, or pattern, you can use ``+=`` to add + new elements, or you can use ``=`` to specify a new value (all previous + contents of the object are removed). If the variable you're changing is a + set or table, then you can use the ``-=`` operator to remove the + specified elements (nothing happens for specified elements that don't + exist). If the variable you are changing is not a table, set, or pattern, + then you must use the ``=`` operator. + + Examples:: + + redef pi = 3.14; + + If you're using "redef" to extend a record or enum, then you must + use the ``+=`` assignment operator. + For an enum, you can add more enumeration constants, and for a record + you can add more record fields (however, each record field in the "redef" + must have either the :bro:attr:`&optional` or :bro:attr:`&default` + attribute). + + Examples:: + + redef enum color += { Blue, Red }; + redef record MyRecord += { n2:int &optional; s2:string &optional; }; + + If you're using "redef" to specify a new event handler body that + replaces all those that were previously defined (i.e., any subsequently + defined event handler body will not be affected by this "redef"), then + the syntax is the same as a regular event handler definition except for + the presence of the "redef" keyword. + + Example:: + + redef event myevent(s:string) { print "Redefined", s; } + + +.. _function/event/hook: + +**function/event/hook** + For details on how to declare a :bro:type:`function`, + :bro:type:`event` handler, or :bro:type:`hook`, + see the documentation for those types. + + +Statements +---------- + +Each statement in a Bro script must be terminated with a semicolon (with a +few exceptions noted below). An individual statement can span multiple +lines. + +All statements (except those contained within a function, hook, or event +handler) must appear after all global declarations. + +Here are the statements that the Bro scripting language supports. + +.. bro:keyword:: add + + The "add" statement is used to add an element to a :bro:type:`set`. + Nothing happens if the specified element already exists in the set. + + Example:: + + local myset: set[string]; + add myset["test"]; + +.. bro:keyword:: break + + The "break" statement is used to break out of a :bro:keyword:`switch` or + :bro:keyword:`for` statement. + + +.. bro:keyword:: delete + + The "delete" statement is used to remove an element from a + :bro:type:`set` or :bro:type:`table`. Nothing happens if the + specified element does not exist in the set or table. + + Example:: + + local myset = set("this", "test"); + local mytable = table(["key1"] = 80/tcp, ["key2"] = 53/udp); + delete myset["test"]; + delete mytable["key1"]; + +.. bro:keyword:: event + + The "event" statement immediately queues invocation of an event handler. + + Example:: + + event myevent("test", 5); + +.. bro:keyword:: fallthrough + + The "fallthrough" statement can be used as the last statement in a + "case" block to indicate that execution should continue into the + next "case" or "default" label. + + For an example, see the :bro:keyword:`switch` statement. + +.. bro:keyword:: for + + A "for" loop iterates over each element in a string, set, vector, or + table and executes a statement for each iteration. + + For each iteration of the loop, a loop variable will be assigned to an + element if the expression evaluates to a string or set, or an index if + the expression evaluates to a vector or table. Then the statement + is executed. However, the statement will not be executed if the expression + evaluates to an object with no elements. + + If the expression is a table or a set with more than one index, then the + loop variable must be specified as a comma-separated list of different + loop variables (one for each index), enclosed in brackets. + + A :bro:keyword:`break` statement can be used at any time to immediately + terminate the "for" loop, and a :bro:keyword:`next` statement can be + used to skip to the next loop iteration. + + Note that the loop variable in a "for" statement is not allowed to be + a global variable, and it does not need to be declared prior to the "for" + statement. The type will be inferred from the elements of the + expression. + + Example:: + + local myset = set(80/tcp, 81/tcp); + local mytable = table([10.0.0.1, 80/tcp]="s1", [10.0.0.2, 81/tcp]="s2"); + + for (p in myset) + print p; + + for ([i,j] in mytable) { + if (mytable[i,j] == "done") + break; + if (mytable[i,j] == "skip") + next; + print i,j; + } + + +.. bro:keyword:: if + + Evaluates a given expression, which must yield a :bro:type:`bool` value. + If true, then a specified statement is executed. If false, then + the statement is not executed. Example:: + + if ( x == 2 ) print "x is 2"; + + + However, if the expression evaluates to false and if an "else" is + provided, then the statement following the "else" is executed. Example:: + + if ( x == 2 ) + print "x is 2"; + else + print "x is not 2"; + +.. bro:keyword:: local + + A variable declared with the "local" keyword will be local. If a type + is not specified, then an initializer is required so that the type can + be inferred. Likewise, if an initializer is not supplied, then the + type must be specified. + + Examples:: + + local x1 = 5.7; + local x2: double; + local x3: double = 5.7; + + Variable declarations inside a function, hook, or event handler are + required to use this keyword (the only two exceptions are variables + declared with :bro:keyword:`const`, and variables implicitly declared in a + :bro:keyword:`for` statement). + + The scope of a local variable starts at the location where it is declared + and persists to the end of the function, hook, + or event handler in which it is declared (this is true even if the + local variable was declared within a `compound statement`_ or is the loop + variable in a "for" statement). + + +.. bro:keyword:: next + + The "next" statement can only appear within a :bro:keyword:`for` loop. + It causes execution to skip to the next iteration. + + For an example, see the :bro:keyword:`for` statement. + +.. bro:keyword:: print + + The "print" statement takes a comma-separated list of one or more + expressions. Each expression in the list is evaluated and then converted + to a string. Then each string is printed, with each string separated by + a comma in the output. + + Examples:: + + print 3.14; + print "Results", x, y; + + By default, the "print" statement writes to the standard + output (stdout). However, if the first expression is of type + :bro:type:`file`, then "print" writes to that file. + + If a string contains non-printable characters (i.e., byte values that are + not in the range 32 - 126), then the "print" statement converts each + non-printable character to an escape sequence before it is printed. + + For more control over how the strings are formatted, see the :bro:id:`fmt` + function. + +.. bro:keyword:: return + + The "return" statement immediately exits the current function, hook, or + event handler. For a function, the specified expression (if any) is + evaluated and returned. A "return" statement in a hook or event handler + cannot return a value because event handlers and hooks do not have + return types. + + Examples:: + + function my_func(): string + { + return "done"; + } + + event my_event(n: count) + { + if ( n == 0 ) return; + + print n; + } + + There is a special form of the "return" statement that is only allowed + in functions. Syntactically, it looks like a :bro:keyword:`when` statement + immediately preceded by the "return" keyword. This form of the "return" + statement is used to specify a function that delays its result (such a + function can only be called in the expression of a :bro:keyword:`when` + statement). The function returns at the time the "when" + statement's condition becomes true, and the function returns the value + that the "when" statement's body returns (or if the condition does + not become true within the specified timeout interval, then the function + returns the value that the "timeout" block returns). + + Example:: + + global X: table[string] of count; + + function a() : count + { + # This delays until condition becomes true. + return when ( "a" in X ) + { + return X["a"]; + } + timeout 30 sec + { + return 0; + } + } + + event bro_init() + { + # Installs a trigger which fires if a() returns 42. + when ( a() == 42 ) + print "expected result"; + + print "Waiting for a() to return..."; + X["a"] = 42; + } + + +.. bro:keyword:: schedule + + The "schedule" statement is used to raise a specified event with + specified parameters at a later time specified as an :bro:type:`interval`. + + Example:: + + schedule 30sec { myevent(x, y, z) }; + + Note that the braces are always required (they do not indicate a + `compound statement`_). + + Note that "schedule" is actually an expression that returns a value + of type "timer", but in practice the return value is not used. + +.. bro:keyword:: switch + + A "switch" statement evaluates a given expression and jumps to + the first "case" label which contains a matching value (the result of the + expression must be type-compatible with all of the values in all of the + "case" labels). If there is no matching value, then execution jumps to + the "default" label instead, and if there is no "default" label then + execution jumps out of the "switch" block. + + Here is an example (assuming that "get_day_of_week" is a + function that returns a string):: + + switch get_day_of_week() + { + case "Sa", "Su": + print "weekend"; + fallthrough; + case "Mo", "Tu", "We", "Th", "Fr": + print "valid result"; + break; + default: + print "invalid result"; + break; + } + + A "switch" block can have any number of "case" labels, and one + optional "default" label. + + A "case" label can have a comma-separated list of + more than one value. A value in a "case" label can be an expression, + but it must be a constant expression (i.e., the expression can consist + only of constants). + + Each "case" and the "default" block must + end with either a :bro:keyword:`break`, :bro:keyword:`fallthrough`, or + :bro:keyword:`return` statement (although "return" is allowed only + if the "switch" statement is inside a function, hook, or event handler). + If a "case" (or "default") block contain more than one statement, then + there is no need to wrap them in braces. + + Note that the braces in a "switch" statement are always required (these + do not indicate the presence of a `compound statement`_), and that no + semicolon is needed at the end of a "switch" statement. + + +.. bro:keyword:: when + + Evaluates a given expression, which must result in a value of type + :bro:type:`bool`. When the value of the expression becomes available + and if the result is true, then a specified statement is executed. + + In the following example, if the expression evaluates to true, then + the "print" statement is executed:: + + when ( (local x = foo()) && x == 42 ) + print x; + + However, if a timeout is specified, and if the expression does not + evaluate to true within the specified timeout interval, then the + statement following the "timeout" keyword is executed:: + + when ( (local x = foo()) && x == 42 ) + print x; + timeout 5sec { + print "timeout"; + } + + Note that when a timeout is specified the braces are + always required (these do not indicate a `compound statement`_). + + The expression in a "when" statement can contain a declaration of a local + variable but only if the declaration is written in the form + "local *var* = *init*" (example: "local x = myfunction()"). This form + of a local declaration is actually an expression, the result of which + is always a boolean true value. + + The expression in a "when" statement can contain an asynchronous function + call such as :bro:id:`lookup_hostname` (in fact, this is the only place + such a function can be called), but it can also contain an ordinary + function call. When an asynchronous function call is in the expression, + then Bro will continue processing statements in the script following + the "when" statement, and when the result of the function call is available + Bro will finish evaluating the expression in the "when" statement. + See the :bro:keyword:`return` statement for an explanation of how to + create an asynchronous function in a Bro script. + + +.. _compound statement: + +**compound statement** + A compound statement is created by wrapping zero or more statements in + braces ``{ }``. Individual statements inside the braces need to be + terminated by a semicolon, but a semicolon is not needed at the end + (outside of the braces) of a compound statement. + + A compound statement is required in order to execute more than one + statement in the body of a :bro:keyword:`for`, :bro:keyword:`if`, or + :bro:keyword:`when` statement. + + Example:: + + if ( x == 2 ) { + print "x is 2"; + ++x; + } + + Note that there are other places in the Bro scripting language that use + braces, but that do not indicate the presence of a compound + statement (these are noted in the documentation). + +.. _null: + +**null statement** + The null statement (executing it has no effect) consists of just a + semicolon. This might be useful during testing or debugging a Bro script + in places where a statement is required, but it is probably not useful + otherwise. + + Example:: + + if ( x == 2 ) + ; + diff --git a/doc/script-reference/types.rst b/doc/script-reference/types.rst index 049b43c04a..75988e0fb5 100644 --- a/doc/script-reference/types.rst +++ b/doc/script-reference/types.rst @@ -1,89 +1,114 @@ Types ===== -Every value in a Bro script has a type (see below for a list of all built-in -types). Although Bro variables have static types (meaning that their type -is fixed), their type is inferred from the value to which they are -initially assigned when the variable is declared without an explicit type -name. +The Bro scripting language supports the following built-in types: -Automatic conversions happen when a binary operator has operands of -different types. Automatic conversions are limited to converting between -numeric types. The numeric types are ``int``, ``count``, and ``double`` -(``bool`` is not a numeric type). -When an automatic conversion occurs, values are promoted to the "highest" -type in the expression. In general, this promotion follows a simple -hierarchy: ``double`` is highest, ``int`` comes next, and ``count`` is -lowest. ++-----------------------+--------------------+ +| Name | Description | ++=======================+====================+ +| :bro:type:`bool` | Boolean | ++-----------------------+--------------------+ +| :bro:type:`count`, | Numeric types | +| :bro:type:`int`, | | +| :bro:type:`double` | | ++-----------------------+--------------------+ +| :bro:type:`time`, | Time types | +| :bro:type:`interval` | | ++-----------------------+--------------------+ +| :bro:type:`string` | String | ++-----------------------+--------------------+ +| :bro:type:`pattern` | Regular expression | ++-----------------------+--------------------+ +| :bro:type:`port`, | Network types | +| :bro:type:`addr`, | | +| :bro:type:`subnet` | | ++-----------------------+--------------------+ +| :bro:type:`enum` | Enumeration | +| | (user-defined type)| ++-----------------------+--------------------+ +| :bro:type:`table`, | Container types | +| :bro:type:`set`, | | +| :bro:type:`vector`, | | +| :bro:type:`record` | | ++-----------------------+--------------------+ +| :bro:type:`function`, | Executable types | +| :bro:type:`event`, | | +| :bro:type:`hook` | | ++-----------------------+--------------------+ +| :bro:type:`file` | File type (only | +| | for writing) | ++-----------------------+--------------------+ +| :bro:type:`opaque` | Opaque type (for | +| | some built-in | +| | functions) | ++-----------------------+--------------------+ +| :bro:type:`any` | Any type (for | +| | functions or | +| | containers) | ++-----------------------+--------------------+ -The Bro scripting language supports the following built-in types. - -.. bro:type:: void - - An internal Bro type (i.e., "void" is not a reserved keyword in the Bro - scripting language) representing the absence of a return type for a - function. +Here is a more detailed description of each type: .. bro:type:: bool Reflects a value with one of two meanings: true or false. The two - ``bool`` constants are ``T`` and ``F``. + "bool" constants are ``T`` and ``F``. - The ``bool`` type supports the following operators: equality/inequality + The "bool" type supports the following operators: equality/inequality (``==``, ``!=``), logical and/or (``&&``, ``||``), logical - negation (``!``), and absolute value (where ``|T|`` is 1, and ``|F|`` is 0). + negation (``!``), and absolute value (where ``|T|`` is 1, and ``|F|`` is 0, + and in both cases the result type is :bro:type:`count`). .. bro:type:: int - A numeric type representing a 64-bit signed integer. An ``int`` constant - is a string of digits preceded by a ``+`` or ``-`` sign, e.g. + A numeric type representing a 64-bit signed integer. An "int" constant + is a string of digits preceded by a "+" or "-" sign, e.g. ``-42`` or ``+5`` (the "+" sign is optional but see note about type - inferencing below). An ``int`` constant can also be written in + inferencing below). An "int" constant can also be written in hexadecimal notation (in which case "0x" must be between the sign and the hex digits), e.g. ``-0xFF`` or ``+0xabc123``. - The ``int`` type supports the following operators: arithmetic + The "int" type supports the following operators: arithmetic operators (``+``, ``-``, ``*``, ``/``, ``%``), comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), assignment operators (``=``, ``+=``, ``-=``), pre-increment (``++``), pre-decrement - (``--``), and absolute value (e.g., ``|-3|`` is 3). + (``--``), unary plus and minus (``+``, ``-``), and absolute value + (e.g., ``|-3|`` is 3, but the result type is :bro:type:`count`). When using type inferencing use care so that the - intended type is inferred, e.g. ``local size_difference = 0`` will - infer :bro:type:`count`, while ``local size_difference = +0`` - will infer :bro:type:`int`. + intended type is inferred, e.g. "local size_difference = 0" will + infer ":bro:type:`count`", while "local size_difference = +0" + will infer "int". .. bro:type:: count - A numeric type representing a 64-bit unsigned integer. A ``count`` - constant is a string of digits, e.g. ``1234`` or ``0``. A ``count`` + A numeric type representing a 64-bit unsigned integer. A "count" + constant is a string of digits, e.g. ``1234`` or ``0``. A "count" can also be written in hexadecimal notation (in which case "0x" must precede the hex digits), e.g. ``0xff`` or ``0xABC123``. - The ``count`` type supports the same operators as the :bro:type:`int` - type. A unary plus or minus applied to a ``count`` results in an ``int``. - -.. bro:type:: counter - - An alias to :bro:type:`count`. + The "count" type supports the same operators as the ":bro:type:`int`" + type, but a unary plus or minus applied to a "count" results in an + "int". .. bro:type:: double A numeric type representing a double-precision floating-point number. Floating-point constants are written as a string of digits with an optional decimal point, optional scale-factor in scientific - notation, and optional ``+`` or ``-`` sign. Examples are ``-1234``, + notation, and optional "+" or "-" sign. Examples are ``-1234``, ``-1234e0``, ``3.14159``, and ``.003E-23``. - The ``double`` type supports the following operators: arithmetic + The "double" type supports the following operators: arithmetic operators (``+``, ``-``, ``*``, ``/``), comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), assignment operators - (``=``, ``+=``, ``-=``), and absolute value (e.g., ``|-3.14|`` is 3.14). + (``=``, ``+=``, ``-=``), unary plus and minus (``+``, ``-``), and + absolute value (e.g., ``|-3.14|`` is 3.14). When using type inferencing use care so that the - intended type is inferred, e.g. ``local size_difference = 5`` will - infer :bro:type:`count`, while ``local size_difference = 5.0`` - will infer :bro:type:`double`. + intended type is inferred, e.g. "local size_difference = 5" will + infer ":bro:type:`count`", while "local size_difference = 5.0" + will infer "double". .. bro:type:: time @@ -94,10 +119,10 @@ The Bro scripting language supports the following built-in types. Time values support the comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``). A ``time`` value can be subtracted from - another ``time`` value to produce an ``interval`` value. An ``interval`` - value can be added to, or subtracted from, a ``time`` value to produce a - ``time`` value. The absolute value of a ``time`` value is a ``double`` - with the same numeric value. + another ``time`` value to produce an :bro:type:`interval` value. An + ``interval`` value can be added to, or subtracted from, a ``time`` value + to produce a ``time`` value. The absolute value of a ``time`` value is + a :bro:type:`double` with the same numeric value. .. bro:type:: interval @@ -112,52 +137,58 @@ The Bro scripting language supports the following built-in types. ``3.5mins``. An ``interval`` can also be negated, for example ``-12 hr`` represents "twelve hours in the past". - Intervals support addition and subtraction. Intervals also support - division (in which case the result is a ``double`` value), the - comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), - and the assignment operators (``=``, ``+=``, ``-=``). Also, an - ``interval`` can be multiplied or divided by an arithmetic type - (``count``, ``int``, or ``double``) to produce an ``interval`` value. - The absolute value of an ``interval`` is a ``double`` value equal to the - number of seconds in the ``interval`` (e.g., ``|-1 min|`` is 60). + Intervals support addition and subtraction, the comparison operators + (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``), the assignment + operators (``=``, ``+=``, ``-=``), and unary plus and minus (``+``, ``-``). + + Intervals also support division (in which case the result is a + :bro:type:`double` value). An ``interval`` can be multiplied or divided + by an arithmetic type (``count``, ``int``, or ``double``) to produce + an ``interval`` value. The absolute value of an ``interval`` is a + ``double`` value equal to the number of seconds in the ``interval`` + (e.g., ``|-1 min|`` is 60.0). .. bro:type:: string - A type used to hold character-string values which represent text. - String constants are created by enclosing text in double quotes (") - and the backslash character (\\) introduces escape sequences (all of - the C-style escape sequences are supported). + A type used to hold character-string values which represent text, although + strings in a Bro script can actually contain any arbitrary binary data. + + String constants are created by enclosing text within a pair of double + quotes ("). A string constant cannot span multiple lines in a Bro script. + The backslash character (\\) introduces escape sequences. The + following escape sequences are recognized: ``\n``, ``\t``, ``\v``, ``\b``, + ``\r``, ``\f``, ``\a``, ``\ooo`` (where each 'o' is an octal digit), + ``\xhh`` (where each 'h' is a hexadecimal digit). For escape sequences + that don't match any of these, Bro will just remove the backslash (so + to represent a literal backslash in a string constant, you just use + two consecutive backslashes). Strings support concatenation (``+``), and assignment (``=``, ``+=``). Strings also support the comparison operators (``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``). The number of characters in a string can be found by enclosing the string within pipe characters (e.g., ``|"abc"|`` - is 3). - - The subscript operator can extract an individual character or a substring - of a string (string indexing is zero-based, but an index of - -1 refers to the last character in the string, and -2 refers to the - second-to-last character, etc.). When extracting a substring, the - starting and ending index values are separated by a colon. For example:: - - local orig = "0123456789"; - local third_char = orig[2]; - local last_char = orig[-1]; - local first_three_chars = orig[0:2]; - - Substring searching can be performed using the "in" or "!in" + is 3). Substring searching can be performed using the "in" or "!in" operators (e.g., "bar" in "foobar" yields true). - Note that Bro represents strings internally as a count and vector of - bytes rather than a NUL-terminated byte string (although string - constants are also automatically NUL-terminated). This is because - network traffic can easily introduce NULs into strings either by - nature of an application, inadvertently, or maliciously. And while - NULs are allowed in Bro strings, when present in strings passed as - arguments to many functions, a run-time error can occur as their - presence likely indicates a sort of problem. In that case, the - string will also only be represented to the user as the literal - "" string. + The subscript operator can extract a substring of a string. To do this, + specify the starting index to extract (if the starting index is omitted, + then zero is assumed), followed by a colon and index + one past the last character to extract (if the last index is omitted, + then the extracted substring will go to the end of the original string). + However, if both the colon and last index are omitted, then a string of + length one is extracted. String indexing is zero-based, but an index + of -1 refers to the last character in the string, and -2 refers to the + second-to-last character, etc. Here are a few examples:: + + local orig = "0123456789"; + local second_char = orig[1]; + local last_char = orig[-1]; + local first_two_chars = orig[:2]; + local last_two_chars = orig[8:]; + local no_first_and_last = orig[1:9]; + + Note that the subscript operator cannot be used to modify a string (i.e., + it cannot be on the left side of an assignment operator). .. bro:type:: pattern @@ -171,7 +202,7 @@ The Bro scripting language supports the following built-in types. and embedded. In exact matching the ``==`` equality relational operator is used - with one :bro:type:`pattern` operand and one :bro:type:`string` + with one "pattern" operand and one ":bro:type:`string`" operand (order of operands does not matter) to check whether the full string exactly matches the pattern. In exact matching, the ``^`` beginning-of-line and ``$`` end-of-line anchors are redundant since @@ -187,8 +218,8 @@ The Bro scripting language supports the following built-in types. yields false. The ``!=`` operator would yield the negation of ``==``. In embedded matching the ``in`` operator is used with one - :bro:type:`pattern` operand (which must be on the left-hand side) and - one :bro:type:`string` operand, but tests whether the pattern + "pattern" operand (which must be on the left-hand side) and + one ":bro:type:`string`" operand, but tests whether the pattern appears anywhere within the given string. For example:: /foo|bar/ in "foobar" @@ -200,27 +231,12 @@ The Bro scripting language supports the following built-in types. is false since "oob" does not appear at the start of "foobar". The ``!in`` operator would yield the negation of ``in``. -.. bro:type:: enum - - A type allowing the specification of a set of related values that - have no further structure. An example declaration: - - .. code:: bro - - type color: enum { Red, White, Blue, }; - - The last comma after ``Blue`` is optional. - - The only operations allowed on enumerations are equality comparisons - (``==``, ``!=``) and assignment (``=``). - Enumerations do not have associated values or ordering. - .. bro:type:: port - A type representing transport-level port numbers. Besides TCP and + A type representing transport-level port numbers (besides TCP and UDP ports, there is a concept of an ICMP "port" where the source port is the ICMP message type and the destination port the ICMP - message code. A ``port`` constant is written as an unsigned integer + message code). A ``port`` constant is written as an unsigned integer followed by one of ``/tcp``, ``/udp``, ``/icmp``, or ``/unknown``. Ports support the comparison operators (``==``, ``!=``, ``<``, ``<=``, @@ -252,14 +268,6 @@ The Bro scripting language supports the following built-in types. address) are treated internally as IPv4 addresses (for example, ``[::ffff:192.168.1.100]`` is equal to ``192.168.1.100``). - Hostname constants can also be used, but since a hostname can - correspond to multiple IP addresses, the type of such a variable is a - :bro:type:`set` of :bro:type:`addr` elements. For example: - - .. code:: bro - - local a = www.google.com; - Addresses can be compared for equality (``==``, ``!=``), and also for ordering (``<``, ``<=``, ``>``, ``>=``). The absolute value of an address gives the size in bits (32 for IPv4, and 128 for IPv6). @@ -282,8 +290,16 @@ The Bro scripting language supports the following built-in types. if ( a in s ) print "true"; - Note that you can check if a given ``addr`` is IPv4 or IPv6 using + You can check if a given ``addr`` is IPv4 or IPv6 using the :bro:id:`is_v4_addr` and :bro:id:`is_v6_addr` built-in functions. + + Note that hostname constants can also be used, but since a hostname can + correspond to multiple IP addresses, the type of such a variable is + "set[addr]". For example: + + .. code:: bro + + local a = www.google.com; .. bro:type:: subnet @@ -293,13 +309,24 @@ The Bro scripting language supports the following built-in types. number. For example, ``192.168.0.0/16`` or ``[fe80::]/64``. Subnets can be compared for equality (``==``, ``!=``). An - :bro:type:`addr` can be checked for inclusion in a subnet using - the "in" or "!in" operators. + "addr" can be checked for inclusion in a subnet using + the ``in`` or ``!in`` operators. -.. bro:type:: any +.. bro:type:: enum - Used to bypass strong typing. For example, a function can take an - argument of type ``any`` when it may be of different types. + A type allowing the specification of a set of related values that + have no further structure. An example declaration: + + .. code:: bro + + type color: enum { Red, White, Blue, }; + + The last comma after ``Blue`` is optional. Both the type name ``color`` + and the individual values (``Red``, etc.) have global scope. + + Enumerations do not have associated values or ordering. + The only operations allowed on enumerations are equality comparisons + (``==``, ``!=``) and assignment (``=``). .. bro:type:: table @@ -313,24 +340,25 @@ The Bro scripting language supports the following built-in types. table [ type^+ ] of type - where *type^+* is one or more types, separated by commas. For example: + where *type^+* is one or more types, separated by commas. + For example: .. code:: bro global a: table[count] of string; - declares a table indexed by :bro:type:`count` values and yielding - :bro:type:`string` values. The yield type can also be more complex: + declares a table indexed by "count" values and yielding + "string" values. The yield type can also be more complex: .. code:: bro global a: table[count] of table[addr, port] of string; - which declares a table indexed by :bro:type:`count` and yielding - another :bro:type:`table` which is indexed by an :bro:type:`addr` - and :bro:type:`port` to yield a :bro:type:`string`. + which declares a table indexed by "count" and yielding + another "table" which is indexed by an "addr" + and "port" to yield a "string". - Initialization of tables occurs by enclosing a set of initializers within + One way to initialize a table is by enclosing a set of initializers within braces, for example: .. code:: bro @@ -340,18 +368,17 @@ The Bro scripting language supports the following built-in types. [5] = "five", }; - A table constructor (equivalent to above example) can also be used - to create a table: + A table constructor can also be used to create a table: .. code:: bro - global t2: table[count] of string = table( - [11] = "eleven", - [5] = "five" + global t2 = table( + [192.168.0.2, 22/tcp] = "ssh", + [192.168.0.3, 80/tcp] = "http" ); Table constructors can also be explicitly named by a type, which is - useful for when a more complex index type could otherwise be + useful when a more complex index type could otherwise be ambiguous: .. code:: bro @@ -378,17 +405,7 @@ The Bro scripting language supports the following built-in types. if ( 13 in t ) ... - - Iterate over tables with a ``for`` loop: - - .. code:: bro - - local t: table[count] of string; - for ( n in t ) - ... - - local services: table[addr, port] of string; - for ( [a, p] in services ) + if ( [192.168.0.2, 22/tcp] in t2 ) ... Add or overwrite individual table elements by assignment: @@ -397,7 +414,7 @@ The Bro scripting language supports the following built-in types. t[13] = "thirteen"; - Remove individual table elements with ``delete``: + Remove individual table elements with :bro:keyword:`delete`: .. code:: bro @@ -413,6 +430,9 @@ The Bro scripting language supports the following built-in types. |t| + See the :bro:keyword:`for` statement for info on how to iterate over + the elements in a table. + .. bro:type:: set A set is like a :bro:type:`table`, but it is a collection of indices @@ -423,25 +443,22 @@ The Bro scripting language supports the following built-in types. where *type^+* is one or more types separated by commas. - Sets are initialized by listing elements enclosed by curly braces: + Sets can be initialized by listing elements enclosed by curly braces: .. code:: bro global s: set[port] = { 21/tcp, 23/tcp, 80/tcp, 443/tcp }; global s2: set[port, string] = { [21/tcp, "ftp"], [23/tcp, "telnet"] }; - The types are explicitly shown in the example above, but they could - have been left to type inference. - A set constructor (equivalent to above example) can also be used to create a set: .. code:: bro - global s3: set[port] = set(21/tcp, 23/tcp, 80/tcp, 443/tcp); + global s3 = set(21/tcp, 23/tcp, 80/tcp, 443/tcp); Set constructors can also be explicitly named by a type, which is - useful for when a more complex index type could otherwise be + useful when a more complex index type could otherwise be ambiguous: .. code:: bro @@ -462,18 +479,10 @@ The Bro scripting language supports the following built-in types. if ( 21/tcp in s ) ... - if ( 21/tcp !in s ) + if ( [21/tcp, "ftp"] !in s2 ) ... - Iterate over a set with a ``for`` loop: - - .. code:: bro - - local s: set[port]; - for ( p in s ) - ... - - Elements are added with ``add``: + Elements are added with :bro:keyword:`add`: .. code:: bro @@ -482,7 +491,7 @@ The Bro scripting language supports the following built-in types. Nothing happens if the element with value ``22/tcp`` was already present in the set. - And removed with ``delete``: + And removed with :bro:keyword:`delete`: .. code:: bro @@ -498,6 +507,9 @@ The Bro scripting language supports the following built-in types. |s| + See the :bro:keyword:`for` statement for info on how to iterate over + the elements in a set. + .. bro:type:: vector A vector is like a :bro:type:`table`, except it's always indexed by a @@ -512,7 +524,7 @@ The Bro scripting language supports the following built-in types. .. code:: bro - global v: vector of string = vector("one", "two", "three"); + local v = vector("one", "two", "three"); Vector constructors can also be explicitly named by a type, which is useful for when a more complex yield type could otherwise be @@ -536,14 +548,6 @@ The Bro scripting language supports the following built-in types. print v[2]; - Iterate over a vector with a ``for`` loop: - - .. code:: bro - - local v: vector of string; - for ( n in v ) - ... - An element can be added to a vector by assigning the value (a value that already exists at that index will be overwritten): @@ -574,11 +578,17 @@ The Bro scripting language supports the following built-in types. The resulting vector of bool is the logical "and" (or logical "or") of each element of the operand vectors. + See the :bro:keyword:`for` statement for info on how to iterate over + the elements in a vector. + .. bro:type:: record - A ``record`` is a collection of values. Each value has a field name + A "record" is a collection of values. Each value has a field name and a type. Values do not need to have the same type and the types - have no restrictions. An example record type definition: + have no restrictions. Field names must follow the same syntax as + regular variable names (except that field names are allowed to be the + same as local or global variables). An example record type + definition: .. code:: bro @@ -587,85 +597,44 @@ The Bro scripting language supports the following built-in types. s: string &optional; }; - Access to a record field uses the dollar sign (``$``) operator: - - .. code:: bro - - global r: MyRecordType; - r$c = 13; - - Record assignment can be done field by field or as a whole like: - - .. code:: bro - - r = [$c = 13, $s = "thirteen"]; - + Records can be initialized or assigned as a whole in three different ways. When assigning a whole record value, all fields that are not :bro:attr:`&optional` or have a :bro:attr:`&default` attribute must - be specified. - - To test for existence of a field that is :bro:attr:`&optional`, use the - ``?$`` operator: + be specified. First, there's a constructor syntax: .. code:: bro - if ( r?$s ) - ... - - Records can also be created using a constructor syntax: - - .. code:: bro - - global r2: MyRecordType = record($c = 7); + local r: MyRecordType = record($c = 7); And the constructor can be explicitly named by type, too, which - is arguably more readable code: + is arguably more readable: .. code:: bro - global r3 = MyRecordType($c = 42); + local r = MyRecordType($c = 42); -.. bro:type:: opaque - - A data type whose actual representation/implementation is - intentionally hidden, but whose values may be passed to certain - functions that can actually access the internal/hidden resources. - Opaque types are differentiated from each other by qualifying them - like ``opaque of md5`` or ``opaque of sha1``. Any valid identifier - can be used as the type qualifier. - - An example use of this type is the set of built-in functions which - perform hashing: + And the third way is like this: .. code:: bro - local handle: opaque of md5 = md5_hash_init(); - md5_hash_update(handle, "test"); - md5_hash_update(handle, "testing"); - print md5_hash_finish(handle); + local r: MyRecordType = [$c = 13, $s = "thirteen"]; - Here the opaque type is used to provide a handle to a particular - resource which is calculating an MD5 checksum incrementally over - time, but the details of that resource aren't relevant, it's only - necessary to have a handle as a way of identifying it and - distinguishing it from other such resources. - -.. bro:type:: file - - Bro supports writing to files, but not reading from them. Files - can be opened using either the :bro:id:`open` or :bro:id:`open_for_append` - built-in functions, and closed using the :bro:id:`close` built-in - function. For example, declare, open, and write to a file - and finally close it like: + Access to a record field uses the dollar sign (``$``) operator, and + record fields can be assigned with this: .. code:: bro - global f: file = open("myfile"); - print f, "hello, world"; - close(f); + local r: MyRecordType; + r$c = 13; - Writing to files like this for logging usually isn't recommended, for better - logging support see :doc:`/frameworks/logging`. + To test if a field that is :bro:attr:`&optional` has been assigned a + value, use the ``?$`` operator (it returns a :bro:type:`bool` value of + ``T`` if the field has been assigned a value, or ``F`` if not): + + .. code:: bro + + if ( r ?$ s ) + ... .. bro:type:: function @@ -697,6 +666,16 @@ The Bro scripting language supports the following built-in types. type, but when it is, the return type and argument list (including the name of each argument) must match exactly. + Here is an example function that takes no parameters and does not + return a value: + + .. code:: bro + + function my_func() + { + print "my_func"; + } + Function types don't need to have a name and can be assigned anonymously: .. code:: bro @@ -739,9 +718,20 @@ The Bro scripting language supports the following built-in types. Event handlers are nearly identical in both syntax and semantics to a :bro:type:`function`, with the two differences being that event handlers have no return type since they never return a value, and - you cannot call an event handler. Instead of directly calling an - event handler from a script, event handler bodies are executed when - they are invoked by one of three different methods: + you cannot call an event handler. + + Example: + + .. code:: bro + + event my_event(r: bool, s: string) + { + print "my_event", r, s; + } + + Instead of directly calling an event handler from a script, event + handler bodies are executed when they are invoked by one of three + different methods: - From the event engine @@ -762,7 +752,7 @@ The Bro scripting language supports the following built-in types. This assumes that ``password_exposed`` was previously declared as an event handler type with compatible arguments. - - Via the ``schedule`` expression in a script + - Via the :bro:keyword:`schedule` expression in a script This delays the invocation of event handlers until some time in the future. For example: @@ -786,8 +776,8 @@ The Bro scripting language supports the following built-in types. immediate and they do not get scheduled through an event queue. Also, a unique feature of a hook is that a given hook handler body can short-circuit the execution of remaining hook handlers simply by - exiting from the body as a result of a ``break`` statement (as - opposed to a ``return`` or just reaching the end of the body). + exiting from the body as a result of a :bro:keyword:`break` statement (as + opposed to a :bro:keyword:`return` or just reaching the end of the body). A hook type is declared like:: @@ -856,3 +846,60 @@ The Bro scripting language supports the following built-in types. executed due to one handler body exiting as a result of a ``break`` statement. +.. bro:type:: file + + Bro supports writing to files, but not reading from them (to read from + files see the :doc:`/frameworks/input`). Files + can be opened using either the :bro:id:`open` or :bro:id:`open_for_append` + built-in functions, and closed using the :bro:id:`close` built-in + function. For example, declare, open, and write to a file and finally + close it like: + + .. code:: bro + + local f = open("myfile"); + print f, "hello, world"; + close(f); + + Writing to files like this for logging usually isn't recommended, for better + logging support see :doc:`/frameworks/logging`. + +.. bro:type:: opaque + + A data type whose actual representation/implementation is + intentionally hidden, but whose values may be passed to certain + built-in functions that can actually access the internal/hidden resources. + Opaque types are differentiated from each other by qualifying them + like "opaque of md5" or "opaque of sha1". + + An example use of this type is the set of built-in functions which + perform hashing: + + .. code:: bro + + local handle = md5_hash_init(); + md5_hash_update(handle, "test"); + md5_hash_update(handle, "testing"); + print md5_hash_finish(handle); + + Here the opaque type is used to provide a handle to a particular + resource which is calculating an MD5 hash incrementally over + time, but the details of that resource aren't relevant, it's only + necessary to have a handle as a way of identifying it and + distinguishing it from other such resources. + +.. bro:type:: any + + Used to bypass strong typing. For example, a function can take an + argument of type ``any`` when it may be of different types. + The only operation allowed on a variable of type ``any`` is assignment. + + Note that users aren't expected to use this type. It's provided mainly + for use by some built-in functions and scripts included with Bro. + +.. bro:type:: void + + An internal Bro type (i.e., "void" is not a reserved keyword in the Bro + scripting language) representing the absence of a return type for a + function. + From 59c54a0fc62025e128f02eae718052c6c24ce532 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 9 Sep 2014 16:28:04 -0500 Subject: [PATCH 11/71] Add a simple FD_Set wrapper/helper class. --- src/ChunkedIO.cc | 14 +++---- src/ChunkedIO.h | 14 +++---- src/DNS_Mgr.cc | 6 +-- src/DNS_Mgr.h | 4 +- src/RemoteSerializer.cc | 22 +++++------ src/RemoteSerializer.h | 4 +- src/Serializer.cc | 6 +-- src/Serializer.h | 4 +- src/iosource/FD_Set.h | 83 ++++++++++++++++++++++++++++++++++++++++ src/iosource/IOSource.h | 6 +-- src/iosource/Manager.cc | 39 ++++--------------- src/iosource/Manager.h | 19 ++++++--- src/iosource/PktSrc.cc | 6 +-- src/iosource/PktSrc.h | 4 +- src/threading/Manager.cc | 4 +- src/threading/Manager.h | 4 +- 16 files changed, 149 insertions(+), 90 deletions(-) create mode 100644 src/iosource/FD_Set.h diff --git a/src/ChunkedIO.cc b/src/ChunkedIO.cc index a94eb98748..722b209bcd 100644 --- a/src/ChunkedIO.cc +++ b/src/ChunkedIO.cc @@ -630,11 +630,11 @@ bool ChunkedIOFd::IsFillingUp() return stats.pending > MAX_BUFFERED_CHUNKS_SOFT; } -std::vector ChunkedIOFd::FdSupplements() const +iosource::FD_Set ChunkedIOFd::ExtraReadFDs() const { - std::vector rval; - rval.push_back(write_flare.FD()); - rval.push_back(read_flare.FD()); + iosource::FD_Set rval; + rval.Insert(write_flare.FD()); + rval.Insert(read_flare.FD()); return rval; } @@ -1140,10 +1140,10 @@ bool ChunkedIOSSL::IsFillingUp() return false; } -std::vector ChunkedIOSSL::FdSupplements() const +iosource::FD_Set ChunkedIOSSL::ExtraReadFDs() const { - std::vector rval; - rval.push_back(write_flare.FD()); + iosource::FD_Set rval; + rval.Insert(write_flare.FD()); return rval; } diff --git a/src/ChunkedIO.h b/src/ChunkedIO.h index c640e529b8..b590453a72 100644 --- a/src/ChunkedIO.h +++ b/src/ChunkedIO.h @@ -7,8 +7,8 @@ #include "List.h" #include "util.h" #include "Flare.h" +#include "iosource/FD_Set.h" #include -#include #ifdef NEED_KRB5_H # include @@ -98,8 +98,8 @@ public: // Returns supplementary file descriptors that become read-ready in order // to signal that there is some work that can be performed. - virtual std::vector FdSupplements() const - { return std::vector(); } + virtual iosource::FD_Set ExtraReadFDs() const + { return iosource::FD_Set(); } // Makes sure that no additional protocol data is written into // the output stream. If this is activated, the output cannot @@ -183,7 +183,7 @@ public: virtual void Clear(); virtual bool Eof() { return eof; } virtual int Fd() { return fd; } - virtual std::vector FdSupplements() const; + virtual iosource::FD_Set ExtraReadFDs() const; virtual void Stats(char* buffer, int length); private: @@ -271,7 +271,7 @@ public: virtual void Clear(); virtual bool Eof() { return eof; } virtual int Fd() { return socket; } - virtual std::vector FdSupplements() const; + virtual iosource::FD_Set ExtraReadFDs() const; virtual void Stats(char* buffer, int length); private: @@ -340,8 +340,8 @@ public: virtual bool Eof() { return io->Eof(); } virtual int Fd() { return io->Fd(); } - virtual std::vector FdSupplements() const - { return io->FdSupplements(); } + virtual iosource::FD_Set ExtraReadFDs() const + { return io->ExtraReadFDs(); } virtual void Stats(char* buffer, int length); void EnableCompression(int level) diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index e7f7f218c0..2c049ba803 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -1216,10 +1216,10 @@ void DNS_Mgr::IssueAsyncRequests() } } -void DNS_Mgr::GetFds(std::vector* read, std::vector* write, - std::vector* except) +void DNS_Mgr::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) { - read->push_back(nb_dns_fd(nb_dns)); + read->Insert(nb_dns_fd(nb_dns)); } double DNS_Mgr::NextTimestamp(double* network_time) diff --git a/src/DNS_Mgr.h b/src/DNS_Mgr.h index d4071a3a0d..d8f420e6cc 100644 --- a/src/DNS_Mgr.h +++ b/src/DNS_Mgr.h @@ -132,8 +132,8 @@ protected: void DoProcess(bool flush); // IOSource interface. - virtual void GetFds(std::vector* read, std::vector* write, - std::vector* except); + virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except); virtual double NextTimestamp(double* network_time); virtual void Process(); virtual const char* Tag() { return "DNS_Mgr"; } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index e23ea775dc..8762f491e4 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -1367,17 +1367,14 @@ void RemoteSerializer::Unregister(ID* id) } } -void RemoteSerializer::GetFds(std::vector* read, std::vector* write, - std::vector* except) +void RemoteSerializer::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) { - read->push_back(io->Fd()); - std::vector supp = io->FdSupplements(); - - for ( size_t i = 0; i < supp.size(); ++i ) - read->push_back(supp[i]); + read->Insert(io->Fd()); + read->Aggregate(io->ExtraReadFDs()); if ( io->CanWrite() ) - write->push_back(io->Fd()); + write->Insert(io->Fd()); } double RemoteSerializer::NextTimestamp(double* local_network_time) @@ -3390,11 +3387,9 @@ void SocketComm::Run() FD_ZERO(&fd_write); FD_ZERO(&fd_except); - int max_fd = 0; - + int max_fd = io->Fd(); FD_SET(io->Fd(), &fd_read); - max_fd = io->Fd(); - fd_vector_set(io->FdSupplements(), &fd_read, &max_fd); + max_fd = std::max(max_fd, io->ExtraReadFDs().Set(&fd_read)); loop_over_list(peers, i) { @@ -3403,7 +3398,8 @@ void SocketComm::Run() FD_SET(peers[i]->io->Fd(), &fd_read); if ( peers[i]->io->Fd() > max_fd ) max_fd = peers[i]->io->Fd(); - fd_vector_set(peers[i]->io->FdSupplements(), &fd_read, &max_fd); + max_fd = std::max(max_fd, + peers[i]->io->ExtraReadFDs().Set(&fd_read)); } else { diff --git a/src/RemoteSerializer.h b/src/RemoteSerializer.h index ebc990f243..2af7610a7c 100644 --- a/src/RemoteSerializer.h +++ b/src/RemoteSerializer.h @@ -140,8 +140,8 @@ public: void Finish(); // Overidden from IOSource: - virtual void GetFds(std::vector* read, std::vector* write, - std::vector* except); + virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except); virtual double NextTimestamp(double* local_network_time); virtual void Process(); virtual TimerMgr::Tag* GetCurrentTag(); diff --git a/src/Serializer.cc b/src/Serializer.cc index 1a637f6576..7306b0ded0 100644 --- a/src/Serializer.cc +++ b/src/Serializer.cc @@ -1068,10 +1068,10 @@ void EventPlayer::GotFunctionCall(const char* name, double time, // We don't replay function calls. } -void EventPlayer::GetFds(std::vector* read, std::vector* write, - std::vector* except) +void EventPlayer::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) { - read->push_back(fd); + read->Insert(fd); } double EventPlayer::NextTimestamp(double* local_network_time) diff --git a/src/Serializer.h b/src/Serializer.h index 6640afc722..558dce2086 100644 --- a/src/Serializer.h +++ b/src/Serializer.h @@ -355,8 +355,8 @@ public: EventPlayer(const char* file); virtual ~EventPlayer(); - virtual void GetFds(std::vector* read, std::vector* write, - std::vector* except); + virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except); virtual double NextTimestamp(double* local_network_time); virtual void Process(); virtual const char* Tag() { return "EventPlayer"; } diff --git a/src/iosource/FD_Set.h b/src/iosource/FD_Set.h new file mode 100644 index 0000000000..43e7c37fc4 --- /dev/null +++ b/src/iosource/FD_Set.h @@ -0,0 +1,83 @@ +#ifndef BRO_FD_SET_H +#define BRO_FD_SET_H + +#include +#include + +namespace iosource { + +/** + * A container holding a set of file descriptors. + */ +class FD_Set { +public: + + /** + * Constructor. The set is initially empty. + */ + FD_Set() + : max(-1), fds() + { } + + /** + * Insert a file descriptor in to the set. + * @param fd the fd to insert in the set. + * @return false if fd was already in the set, else true. + */ + bool Insert(int fd) + { + if ( max < fd ) max = fd; + return fds.insert(fd).second; + } + + /** + * Inserts all the file descriptors from another set in to this one. + * @param other a file descriptor set to merge in to this one. + */ + void Aggregate(const FD_Set& other) + { + for ( std::set::const_iterator it = other.fds.begin(); + it != other.fds.end(); ++it ) + Insert(*it); + } + + /** + * Empties the set. + */ + void Clear() + { max = -1; fds.clear(); } + + /** + * Insert file descriptors in to a fd_set for use with select(). + * @return the greatest file descriptor inserted. + */ + int Set(fd_set* set) const + { + for ( std::set::const_iterator it = fds.begin(); it != fds.end(); + ++it ) + FD_SET(*it, set); + return max; + } + + /** + * @return Whether a file descriptor belonging to this set is within the + * fd_set arugment. + */ + bool Ready(fd_set* set) const + { + for ( std::set::const_iterator it = fds.begin(); it != fds.end(); + ++it ) + if ( FD_ISSET(*it, set) ) + return true; + return false; + } + +private: + + int max; + std::set fds; +}; + +} // namespace bro + +#endif // BRO_FD_SET_H diff --git a/src/iosource/IOSource.h b/src/iosource/IOSource.h index 630a7fcf11..df82012268 100644 --- a/src/iosource/IOSource.h +++ b/src/iosource/IOSource.h @@ -8,8 +8,7 @@ extern "C" { } #include -#include - +#include "FD_Set.h" #include "Timer.h" namespace iosource { @@ -62,8 +61,7 @@ public: * * @param except Pointer to container where to insert a except descriptor. */ - virtual void GetFds(std::vector* read, std::vector* write, - std::vector* except) = 0; + virtual void GetFds(FD_Set* read, FD_Set* write, FD_Set* except) = 0; /** * Returns the timestamp (in \a global network time) associated with diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 4259087e40..41118bdbfe 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -44,15 +44,6 @@ void Manager::RemoveAll() dont_counts = sources.size(); } -static void fd_vector_set(const std::vector& fds, fd_set* set, int* max) - { - for ( size_t i = 0; i < fds.size(); ++i ) - { - FD_SET(fds[i], set); - *max = ::max(fds[i], *max); - } - } - IOSource* Manager::FindSoonest(double* ts) { // Remove sources which have gone dry. For simplicity, we only @@ -124,14 +115,9 @@ IOSource* Manager::FindSoonest(double* ts) // be ready. continue; - src->fd_read.clear(); - src->fd_write.clear(); - src->fd_except.clear(); + src->Clear(); src->src->GetFds(&src->fd_read, &src->fd_write, &src->fd_except); - - fd_vector_set(src->fd_read, &fd_read, &maxx); - fd_vector_set(src->fd_write, &fd_write, &maxx); - fd_vector_set(src->fd_except, &fd_except, &maxx); + src->SetFds(&fd_read, &fd_write, &fd_except, &maxx); } // We can't block indefinitely even when all sources are dry: @@ -316,21 +302,10 @@ PktDumper* Manager::OpenPktDumper(const string& path, bool append) return pd; } -static bool fd_vector_ready(const std::vector& fds, fd_set* set) +void Manager::Source::SetFds(fd_set* read, fd_set* write, fd_set* except, + int* maxx) const { - for ( size_t i = 0; i < fds.size(); ++i ) - if ( FD_ISSET(fds[i], set) ) - return true; - - return false; - } - -bool Manager::Source::Ready(fd_set* read, fd_set* write, fd_set* except) const - { - if ( fd_vector_ready(fd_read, read) || - fd_vector_ready(fd_write, write) || - fd_vector_ready(fd_except, except) ) - return true; - - return false; + *maxx = std::max(*maxx, fd_read.Set(read)); + *maxx = std::max(*maxx, fd_write.Set(write)); + *maxx = std::max(*maxx, fd_except.Set(except)); } diff --git a/src/iosource/Manager.h b/src/iosource/Manager.h index a18d433d66..288ec74352 100644 --- a/src/iosource/Manager.h +++ b/src/iosource/Manager.h @@ -5,8 +5,7 @@ #include #include -#include -#include +#include "iosource/FD_Set.h" namespace iosource { @@ -115,11 +114,19 @@ private: struct Source { IOSource* src; - std::vector fd_read; - std::vector fd_write; - std::vector fd_except; + FD_Set fd_read; + FD_Set fd_write; + FD_Set fd_except; - bool Ready(fd_set* read, fd_set* write, fd_set* except) const; + bool Ready(fd_set* read, fd_set* write, fd_set* except) const + { return fd_read.Ready(read) || fd_write.Ready(write) || + fd_except.Ready(except); } + + void SetFds(fd_set* read, fd_set* write, fd_set* except, + int* maxx) const; + + void Clear() + { fd_read.Clear(); fd_write.Clear(); fd_except.Clear(); } }; typedef std::list SourceList; diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 963f37cb83..6ee95a48a1 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -218,8 +218,8 @@ void PktSrc::Done() Close(); } -void PktSrc::GetFds(std::vector* read, std::vector* write, - std::vector* except) +void PktSrc::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) { if ( pseudo_realtime ) { @@ -230,7 +230,7 @@ void PktSrc::GetFds(std::vector* read, std::vector* write, } if ( IsOpen() && props.selectable_fd >= 0 ) - read->push_back(props.selectable_fd); + read->Insert(props.selectable_fd); } double PktSrc::NextTimestamp(double* local_network_time) diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 8705ec48f3..9c05115257 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -388,8 +388,8 @@ private: // IOSource interface implementation. virtual void Init(); virtual void Done(); - virtual void GetFds(std::vector* read, std::vector* write, - std::vector* except); + virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except); virtual double NextTimestamp(double* local_network_time); virtual void Process(); virtual const char* Tag(); diff --git a/src/threading/Manager.cc b/src/threading/Manager.cc index 19ee948a1c..449f2a8ad1 100644 --- a/src/threading/Manager.cc +++ b/src/threading/Manager.cc @@ -65,8 +65,8 @@ void Manager::AddMsgThread(MsgThread* thread) msg_threads.push_back(thread); } -void Manager::GetFds(std::vector* read, std::vector* write, - std::vector* except) +void Manager::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) { } diff --git a/src/threading/Manager.h b/src/threading/Manager.h index e208bb64c1..70e592fa10 100644 --- a/src/threading/Manager.h +++ b/src/threading/Manager.h @@ -103,8 +103,8 @@ protected: /** * Part of the IOSource interface. */ - virtual void GetFds(std::vector* read, std::vector* write, - std::vector* except); + virtual void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except); /** * Part of the IOSource interface. From 36efc8253df9b462e46cf0a40359f835f6b3f1e5 Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Mon, 15 Sep 2014 10:57:32 -0500 Subject: [PATCH 12/71] New page for List of Log files, linked to script-reference --- doc/logs/index.rst | 45 ++-------------------- doc/script-reference/index.rst | 2 + doc/script-reference/list-of-log-files.rst | 38 ++++++++++++++++++ 3 files changed, 43 insertions(+), 42 deletions(-) create mode 100644 doc/script-reference/list-of-log-files.rst diff --git a/doc/logs/index.rst b/doc/logs/index.rst index 7c7006054f..a8fb951c80 100644 --- a/doc/logs/index.rst +++ b/doc/logs/index.rst @@ -111,7 +111,9 @@ default, including: such "crud" that is usually not worth following up on. As you can see, some log files are specific to a particular protocol, -while others aggregate information across different types of activity. +while others aggregate information across different types of activity. +For a complete list of log files and a description of its purpose, +see :doc:`List of Log Files <../script-reference/list-of-log-files>`. .. _bro-cut: @@ -250,44 +252,3 @@ protocol, it can have multiple ``GET``/``POST``/etc requests in a stream and Bro is able to extract and track that information for you, giving you an in-depth and structured view into HTTP traffic on your network. - ------------------------ -Common Log Files ------------------------ -As a monitoring tool, Bro records a detailed view of the traffic inspected -and the events generated in a series of relevant log files. These files can -later be reviewed for monitoring, auditing and troubleshooting purposes. - -In this section we present a brief explanation of the most commonly used log -files generated by Bro including links to descriptions of some of the fields -for each log type. - -+-----------------+---------------------------------------+------------------------------+ -| Log File | Description | Field Descriptions | -+=================+=======================================+==============================+ -| http.log | Shows all HTTP requests and replies | :bro:type:`HTTP::Info` | -+-----------------+---------------------------------------+------------------------------+ -| ftp.log | Records FTP activity | :bro:type:`FTP::Info` | -+-----------------+---------------------------------------+------------------------------+ -| ssl.log | Records SSL sessions including | :bro:type:`SSL::Info` | -| | certificates used | | -+-----------------+---------------------------------------+------------------------------+ -| known_certs.log | Includes SSL certificates used | :bro:type:`Known::CertsInfo` | -+-----------------+---------------------------------------+------------------------------+ -| smtp.log | Summarizes SMTP traffic on a network | :bro:type:`SMTP::Info` | -+-----------------+---------------------------------------+------------------------------+ -| dns.log | Shows all DNS activity on a network | :bro:type:`DNS::Info` | -+-----------------+---------------------------------------+------------------------------+ -| conn.log | Records all connections seen by Bro | :bro:type:`Conn::Info` | -+-----------------+---------------------------------------+------------------------------+ -| dpd.log | Shows network activity on | :bro:type:`DPD::Info` | -| | non-standard ports | | -+-----------------+---------------------------------------+------------------------------+ -| files.log | Records information about all files | :bro:type:`Files::Info` | -| | transmitted over the network | | -+-----------------+---------------------------------------+------------------------------+ -| weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` | -| | activity | | -+-----------------+---------------------------------------+------------------------------+ - - diff --git a/doc/script-reference/index.rst b/doc/script-reference/index.rst index bd600e4a97..fb2b24efa7 100644 --- a/doc/script-reference/index.rst +++ b/doc/script-reference/index.rst @@ -12,3 +12,5 @@ Script Reference packages scripts Broxygen Example Script + list-of-log-files + diff --git a/doc/script-reference/list-of-log-files.rst b/doc/script-reference/list-of-log-files.rst new file mode 100644 index 0000000000..cae276eb95 --- /dev/null +++ b/doc/script-reference/list-of-log-files.rst @@ -0,0 +1,38 @@ +================= +List of Log Files +================= + +As a monitoring tool, Bro records a detailed view of the traffic inspected +and the events generated in a series of relevant log files. These files can +later be reviewed for monitoring, auditing and troubleshooting purposes. + +Listed below are the log files generated by Bro, a brief description of the +log file, and links to descriptions of some of the fields for each log type. + ++-----------------+---------------------------------------+------------------------------+ +| Log File | Description | Field Descriptions | ++=================+=======================================+==============================+ +| http.log | Shows all HTTP requests and replies | :bro:type:`HTTP::Info` | ++-----------------+---------------------------------------+------------------------------+ +| ftp.log | Records FTP activity | :bro:type:`FTP::Info` | ++-----------------+---------------------------------------+------------------------------+ +| ssl.log | Records SSL sessions including | :bro:type:`SSL::Info` | +| | certificates used | | ++-----------------+---------------------------------------+------------------------------+ +| known_certs.log | Includes SSL certificates used | :bro:type:`Known::CertsInfo` | ++-----------------+---------------------------------------+------------------------------+ +| smtp.log | Summarizes SMTP traffic on a network | :bro:type:`SMTP::Info` | ++-----------------+---------------------------------------+------------------------------+ +| dns.log | Shows all DNS activity on a network | :bro:type:`DNS::Info` | ++-----------------+---------------------------------------+------------------------------+ +| conn.log | Records all connections seen by Bro | :bro:type:`Conn::Info` | ++-----------------+---------------------------------------+------------------------------+ +| dpd.log | Shows network activity on | :bro:type:`DPD::Info` | +| | non-standard ports | | ++-----------------+---------------------------------------+------------------------------+ +| files.log | Records information about all files | :bro:type:`Files::Info` | +| | transmitted over the network | | ++-----------------+---------------------------------------+------------------------------+ +| weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` | +| | activity | | ++-----------------+---------------------------------------+------------------------------+ From 401ec39ce2c0ed68f80e1cb254c4d328d85d9bae Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Tue, 16 Sep 2014 09:49:48 -0500 Subject: [PATCH 13/71] Changing name of file --- doc/script-reference/{list-of-log-files.rst => log-files.rst} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename doc/script-reference/{list-of-log-files.rst => log-files.rst} (100%) diff --git a/doc/script-reference/list-of-log-files.rst b/doc/script-reference/log-files.rst similarity index 100% rename from doc/script-reference/list-of-log-files.rst rename to doc/script-reference/log-files.rst From d342cde22c3ae936f5f9d3831ff425ecef7d28e5 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 16 Sep 2014 10:09:46 -0500 Subject: [PATCH 14/71] Improve error message when failing to activate a plugin. Also fix a unit test helper script that checks plugin availability. --- src/main.cc | 8 +++++++- testing/scripts/has-writer | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/main.cc b/src/main.cc index ddd65fafd5..55afe34c79 100644 --- a/src/main.cc +++ b/src/main.cc @@ -850,9 +850,15 @@ int main(int argc, char** argv) file_mgr->InitPreScript(); broxygen_mgr->InitPreScript(); + bool missing_plugin = false; + for ( set::const_iterator i = requested_plugins.begin(); i != requested_plugins.end(); i++ ) - plugin_mgr->ActivateDynamicPlugin(*i); + if ( ! plugin_mgr->ActivateDynamicPlugin(*i) ) + missing_plugin = true; + + if ( missing_plugin ) + reporter->FatalError("Failed to activate requested dynamic plugins."); plugin_mgr->ActivateDynamicPlugins(! bare_mode); diff --git a/testing/scripts/has-writer b/testing/scripts/has-writer index 4c5f38a6bb..d6cdf28d12 100755 --- a/testing/scripts/has-writer +++ b/testing/scripts/has-writer @@ -3,4 +3,4 @@ # Returns true if Bro has been compiled with support for writer type # $1. The type name must match the plugin name that "bro -N" prints. -bro -N $1 >/dev/null +bro -N | grep -q $1 >/dev/null From 14940c2d8947d6ec7a8f9af944e8efb1e7b8df1a Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Mon, 22 Sep 2014 10:59:05 -0500 Subject: [PATCH 15/71] More updates to log files page: descriptions --- doc/script-reference/index.rst | 5 +- doc/script-reference/log-files.rst | 138 ++++++++++++++++++++++------- 2 files changed, 109 insertions(+), 34 deletions(-) diff --git a/doc/script-reference/index.rst b/doc/script-reference/index.rst index fb2b24efa7..ee73ca84ee 100644 --- a/doc/script-reference/index.rst +++ b/doc/script-reference/index.rst @@ -5,6 +5,7 @@ Script Reference .. toctree:: :maxdepth: 1 + log-files notices proto-analyzers file-analyzers @@ -12,5 +13,5 @@ Script Reference packages scripts Broxygen Example Script - list-of-log-files - + + diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index cae276eb95..d4c3cee02e 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -1,38 +1,112 @@ -================= -List of Log Files -================= +========= +Log Files +========= As a monitoring tool, Bro records a detailed view of the traffic inspected and the events generated in a series of relevant log files. These files can later be reviewed for monitoring, auditing and troubleshooting purposes. -Listed below are the log files generated by Bro, a brief description of the -log file, and links to descriptions of some of the fields for each log type. +Listed below are the log files generated by Bro, including a brief description +of the log file and links to descriptions of some of the fields for each log type. -+-----------------+---------------------------------------+------------------------------+ -| Log File | Description | Field Descriptions | -+=================+=======================================+==============================+ -| http.log | Shows all HTTP requests and replies | :bro:type:`HTTP::Info` | -+-----------------+---------------------------------------+------------------------------+ -| ftp.log | Records FTP activity | :bro:type:`FTP::Info` | -+-----------------+---------------------------------------+------------------------------+ -| ssl.log | Records SSL sessions including | :bro:type:`SSL::Info` | -| | certificates used | | -+-----------------+---------------------------------------+------------------------------+ -| known_certs.log | Includes SSL certificates used | :bro:type:`Known::CertsInfo` | -+-----------------+---------------------------------------+------------------------------+ -| smtp.log | Summarizes SMTP traffic on a network | :bro:type:`SMTP::Info` | -+-----------------+---------------------------------------+------------------------------+ -| dns.log | Shows all DNS activity on a network | :bro:type:`DNS::Info` | -+-----------------+---------------------------------------+------------------------------+ -| conn.log | Records all connections seen by Bro | :bro:type:`Conn::Info` | -+-----------------+---------------------------------------+------------------------------+ -| dpd.log | Shows network activity on | :bro:type:`DPD::Info` | -| | non-standard ports | | -+-----------------+---------------------------------------+------------------------------+ -| files.log | Records information about all files | :bro:type:`Files::Info` | -| | transmitted over the network | | -+-----------------+---------------------------------------+------------------------------+ -| weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` | -| | activity | | -+-----------------+---------------------------------------+------------------------------+ ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| app_stats.log | Info about web apps in use on network | :bro:type:`AppStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| barnyard2.log | Alerts received from Barnyard2 | :bro:type:`Barnyard2::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| capture_loss.log | Packet loss rate | :bro:type:`CaptureLoss::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| cluster.log | Cluster messages | :bro:type:`Cluster::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| communication.log | Connections to remote Bro or Broccoli | :bro:type:`Communication::Info` | +| | instances | | ++----------------------------+---------------------------------------+---------------------------------+ +| conn.log  | Connection info | :bro:type:`Conn::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dhcp.log  | DHCP leases | :bro:type:`DHCP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dnp3.log | Requests and replies using DNP3 | :bro:type:`DNP3::Info` | +| | protocol | | ++----------------------------+---------------------------------------+---------------------------------+ +| dns.log  | DNS activity | :bro:type:`DNS::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dpd.log | Network activity on non-standard | :bro:type:`DPD::Info` | +| | ports | | ++----------------------------+---------------------------------------+---------------------------------+ +| files.log | Info about files transmitted over the | :bro:type:`Files::Info` | +| | network | | ++----------------------------+---------------------------------------+---------------------------------+ +| ftp.log | FTP activity | :bro:type:`FTP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| http.log | HTTP requests and replies | :bro:type:`HTTP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| intel.log | Details about the intelligence | :bro:type:`Intel::Info` | +| | framework | | ++----------------------------+---------------------------------------+---------------------------------+ +| irc.log | IRC commands and responses | :bro:type:`IRC::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_certs.log | SSL certificates used | :bro:type:`Known::CertsInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_devices.log | MAC addresses of devices on the | :bro:type:`Known::DevicesInfo` | +| | network | | ++----------------------------+---------------------------------------+---------------------------------+ +| known_hosts.log | Daily record of completed TCP | :bro:type:`Known::HostsInfo` | +| | handshakes | | ++----------------------------+---------------------------------------+---------------------------------+ +| known_modbus.log | Modbus masters and workers | :bro:type:`Known::ModbusInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_services.log | Tracks services and protocols used | :bro:type:`Known::ServicesInfo` | +| | during a session | | ++----------------------------+---------------------------------------+---------------------------------+ +| loaded_scripts.log | Shows all scripts loaded by Bro | :bro:type:`LoadedScripts::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| modbus.log | Modbus protocol data | :bro:type:`Modbus::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| modbus_register_change.log | | | ++----------------------------+---------------------------------------+---------------------------------+ +| notice.log | Bro notices | :bro:type:`Notice::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| notice_alarm.log | The alarm stream | :bro:type:`Notice::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| packetfilter.log | Status of packet filters | :bro:type:`PacketFilter::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| radius.log  | RADIUS authentication attempts | :bro:type:`RADIUS::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| reporter.log | Records error messages, location, | :bro:type:`Reporter::Info` | +| | and severity | | ++----------------------------+---------------------------------------+---------------------------------+ +| signatures.log | Tracks signatures used on TCP | :bro:type:`Signatures::Info` | +| | connections | | ++----------------------------+---------------------------------------+---------------------------------+ +| smtp.log | SMTP traffic on a network | :bro:type:`SMTP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| snmp.log  | SNMP traffic on a network | :bro:type:`SNMP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| socks.log | SOCKS proxy requests | :bro:type:`SOCKS::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| software.log | Software being used on the network | :bro:type:`Software::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| ssh.log  | SSH connections | :bro:type:`SSH::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| ssl.log  | SSL/TLS handshake info | :bro:type:`SSL::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| stats.log | Shows log memory/packet/lag | :bro:type:`Stats::Info` | +| | statistics | | ++----------------------------+---------------------------------------+---------------------------------+ +| syslog.log  | Syslog messages and data | :bro:type:`Syslog::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| traceroute.log | Address and protocol data of a given | :bro:type:`Traceroute::Info` | +| | traceroute | | ++----------------------------+---------------------------------------+---------------------------------+ +| tunnel.log | Tunnel data | :bro:type:`Tunnel::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| unified2.log | Interprets Snort's unified output | :bro:type:`Unified2::Info` | +| | format | | ++----------------------------+---------------------------------------+---------------------------------+ +| weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` | +| | activity | | ++----------------------------+---------------------------------------+---------------------------------+ +| x509.log | Tracks X.509 certificates | :bro:type:`X509::Info` | ++----------------------------+---------------------------------------+---------------------------------+ From e402a224d8530c0cb813e0b4b79a6a94982df13c Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Tue, 23 Sep 2014 08:53:54 -0500 Subject: [PATCH 16/71] Adding deatils for modbus_register_change.log --- doc/script-reference/log-files.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index d4c3cee02e..5c2c761a07 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -64,7 +64,7 @@ of the log file and links to descriptions of some of the fields for each log typ +----------------------------+---------------------------------------+---------------------------------+ | modbus.log | Modbus protocol data | :bro:type:`Modbus::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| modbus_register_change.log | | | +| modbus_register_change.log | Tracks changes to holding registers | :bro:type:`Modbus::MemmapInfo` | +----------------------------+---------------------------------------+---------------------------------+ | notice.log | Bro notices | :bro:type:`Notice::Info` | +----------------------------+---------------------------------------+---------------------------------+ From 16c70a5179979c1489f5f0e28ce5f014b4c5a81e Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Thu, 25 Sep 2014 10:22:46 -0500 Subject: [PATCH 17/71] Broke down logs into grouped sections based on use & origin --- doc/script-reference/log-files.rst | 105 ++++++++++++++++++----------- 1 file changed, 65 insertions(+), 40 deletions(-) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 5c2c761a07..0098bce4c0 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -7,15 +7,15 @@ and the events generated in a series of relevant log files. These files can later be reviewed for monitoring, auditing and troubleshooting purposes. Listed below are the log files generated by Bro, including a brief description -of the log file and links to descriptions of some of the fields for each log type. +of the log file and links to descriptions of some of the fields for each log +type. + +Bro Diagnostics +--------------- +----------------------------+---------------------------------------+---------------------------------+ | Log File | Description | Field Descriptions | +============================+=======================================+=================================+ -| app_stats.log | Info about web apps in use on network | :bro:type:`AppStats::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| barnyard2.log | Alerts received from Barnyard2 | :bro:type:`Barnyard2::Info` | -+----------------------------+---------------------------------------+---------------------------------+ | capture_loss.log | Packet loss rate | :bro:type:`CaptureLoss::Info` | +----------------------------+---------------------------------------+---------------------------------+ | cluster.log | Cluster messages | :bro:type:`Cluster::Info` | @@ -23,6 +23,55 @@ of the log file and links to descriptions of some of the fields for each log typ | communication.log | Connections to remote Bro or Broccoli | :bro:type:`Communication::Info` | | | instances | | +----------------------------+---------------------------------------+---------------------------------+ +| intel.log | Details about the intelligence | :bro:type:`Intel::Info` | +| | framework | | ++----------------------------+---------------------------------------+---------------------------------+ +| loaded_scripts.log | Shows all scripts loaded by Bro | :bro:type:`LoadedScripts::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| notice.log | Bro notices | :bro:type:`Notice::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| notice_alarm.log | The alarm stream | :bro:enum:`Notice::ACTION_ALARM`| ++----------------------------+---------------------------------------+---------------------------------+ +| packetfilter.log | Status of packet filters | :bro:type:`PacketFilter::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| reporter.log | Records error messages, location, | :bro:type:`Reporter::Info` | +| | and severity | | ++----------------------------+---------------------------------------+---------------------------------+ +| stats.log | Shows log memory/packet/lag | :bro:type:`Stats::Info` | +| | statistics | | ++----------------------------+---------------------------------------+---------------------------------+ +| unified2.log | Interprets Snort's unified output | :bro:type:`Unified2::Info` | +| | format | | ++----------------------------+---------------------------------------+---------------------------------+ + +Known_* Logs +------------ + ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| known_certs.log | SSL certificates used | :bro:type:`Known::CertsInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_devices.log | MAC addresses of devices on the | :bro:type:`Known::DevicesInfo` | +| | network | | ++----------------------------+---------------------------------------+---------------------------------+ +| known_hosts.log | Daily record of completed TCP | :bro:type:`Known::HostsInfo` | +| | handshakes | | ++----------------------------+---------------------------------------+---------------------------------+ +| known_modbus.log | Modbus masters and workers | :bro:type:`Known::ModbusInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_services.log | Tracks services and protocols used | :bro:type:`Known::ServicesInfo` | +| | during a session | | ++----------------------------+---------------------------------------+---------------------------------+ + +Network Activity +---------------- + ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| barnyard2.log | Alerts received from Barnyard2 | :bro:type:`Barnyard2::Info` | ++----------------------------+---------------------------------------+---------------------------------+ | conn.log  | Connection info | :bro:type:`Conn::Info` | +----------------------------+---------------------------------------+---------------------------------+ | dhcp.log  | DHCP leases | :bro:type:`DHCP::Info` | @@ -42,41 +91,14 @@ of the log file and links to descriptions of some of the fields for each log typ +----------------------------+---------------------------------------+---------------------------------+ | http.log | HTTP requests and replies | :bro:type:`HTTP::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| intel.log | Details about the intelligence | :bro:type:`Intel::Info` | -| | framework | | -+----------------------------+---------------------------------------+---------------------------------+ | irc.log | IRC commands and responses | :bro:type:`IRC::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| known_certs.log | SSL certificates used | :bro:type:`Known::CertsInfo` | -+----------------------------+---------------------------------------+---------------------------------+ -| known_devices.log | MAC addresses of devices on the | :bro:type:`Known::DevicesInfo` | -| | network | | -+----------------------------+---------------------------------------+---------------------------------+ -| known_hosts.log | Daily record of completed TCP | :bro:type:`Known::HostsInfo` | -| | handshakes | | -+----------------------------+---------------------------------------+---------------------------------+ -| known_modbus.log | Modbus masters and workers | :bro:type:`Known::ModbusInfo` | -+----------------------------+---------------------------------------+---------------------------------+ -| known_services.log | Tracks services and protocols used | :bro:type:`Known::ServicesInfo` | -| | during a session | | -+----------------------------+---------------------------------------+---------------------------------+ -| loaded_scripts.log | Shows all scripts loaded by Bro | :bro:type:`LoadedScripts::Info` | -+----------------------------+---------------------------------------+---------------------------------+ | modbus.log | Modbus protocol data | :bro:type:`Modbus::Info` | +----------------------------+---------------------------------------+---------------------------------+ | modbus_register_change.log | Tracks changes to holding registers | :bro:type:`Modbus::MemmapInfo` | +----------------------------+---------------------------------------+---------------------------------+ -| notice.log | Bro notices | :bro:type:`Notice::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| notice_alarm.log | The alarm stream | :bro:type:`Notice::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| packetfilter.log | Status of packet filters | :bro:type:`PacketFilter::Info` | -+----------------------------+---------------------------------------+---------------------------------+ | radius.log  | RADIUS authentication attempts | :bro:type:`RADIUS::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| reporter.log | Records error messages, location, | :bro:type:`Reporter::Info` | -| | and severity | | -+----------------------------+---------------------------------------+---------------------------------+ | signatures.log | Tracks signatures used on TCP | :bro:type:`Signatures::Info` | | | connections | | +----------------------------+---------------------------------------+---------------------------------+ @@ -86,15 +108,10 @@ of the log file and links to descriptions of some of the fields for each log typ +----------------------------+---------------------------------------+---------------------------------+ | socks.log | SOCKS proxy requests | :bro:type:`SOCKS::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| software.log | Software being used on the network | :bro:type:`Software::Info` | -+----------------------------+---------------------------------------+---------------------------------+ | ssh.log  | SSH connections | :bro:type:`SSH::Info` | +----------------------------+---------------------------------------+---------------------------------+ | ssl.log  | SSL/TLS handshake info | :bro:type:`SSL::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| stats.log | Shows log memory/packet/lag | :bro:type:`Stats::Info` | -| | statistics | | -+----------------------------+---------------------------------------+---------------------------------+ | syslog.log  | Syslog messages and data | :bro:type:`Syslog::Info` | +----------------------------+---------------------------------------+---------------------------------+ | traceroute.log | Address and protocol data of a given | :bro:type:`Traceroute::Info` | @@ -102,11 +119,19 @@ of the log file and links to descriptions of some of the fields for each log typ +----------------------------+---------------------------------------+---------------------------------+ | tunnel.log | Tunnel data | :bro:type:`Tunnel::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| unified2.log | Interprets Snort's unified output | :bro:type:`Unified2::Info` | -| | format | | -+----------------------------+---------------------------------------+---------------------------------+ | weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` | | | activity | | +----------------------------+---------------------------------------+---------------------------------+ | x509.log | Tracks X.509 certificates | :bro:type:`X509::Info` | +----------------------------+---------------------------------------+---------------------------------+ + +Software Asset Tracking +----------------------- + ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| app_stats.log | Info about web apps in use on network | :bro:type:`AppStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| software.log | Software being used on the network | :bro:type:`Software::Info` | ++----------------------------+---------------------------------------+---------------------------------+ From 1817f960c6767c10738458ee61948afc06549758 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 25 Sep 2014 12:47:10 -0700 Subject: [PATCH 18/71] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index b1031e97e1..30c156d879 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit b1031e97e1cb57df0e0405a6b5c6d0eb49c32c62 +Subproject commit 30c156d879f6303f15ebf8e59989d8a42d882bdf diff --git a/aux/bro-aux b/aux/bro-aux index 92dea34b43..bdb83e43ce 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 92dea34b43f4109b97dc2cccda49dbb58c72f77a +Subproject commit bdb83e43ce29250b32033e96c3054c486cbee1ef diff --git a/aux/broccoli b/aux/broccoli index 64134bc778..07cfcc76fb 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 64134bc778b46307180192cff48f0d1f08a874e8 +Subproject commit 07cfcc76fb08365b545bd3f412c3f6e6c92824e9 diff --git a/aux/broctl b/aux/broctl index db3f7e375b..2606a95c9d 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit db3f7e375b785ee3ef9795bc4917d396871785ff +Subproject commit 2606a95c9dcbc83bd863c2981ce7189e5d58697b diff --git a/cmake b/cmake index 0b22aeb9f3..f2e8ba6b90 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 0b22aeb9f30b1edad54c225ef3e431c68750480b +Subproject commit f2e8ba6b90b3a2da9f1f77c55d0e718c25376bbb From 9d563f2f4d714a9063c608b992d5091c7224bf9d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 25 Sep 2014 12:47:55 -0700 Subject: [PATCH 19/71] Updating submodule(s). [nomail] --- aux/plugins | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/plugins b/aux/plugins index 783d47c854..ad600b5bdc 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 783d47c854c97dda6cff9b9eecb8709fe1ee749b +Subproject commit ad600b5bdcd56a2723e323c0f2c8e1708956ca4f From ea1803881dd8d96b11266a9f3f9f7590251caf70 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 25 Sep 2014 12:49:00 -0700 Subject: [PATCH 20/71] Updating submodule(s). [nomail] --- aux/bro-aux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/bro-aux b/aux/bro-aux index bdb83e43ce..30e6ff1a07 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit bdb83e43ce29250b32033e96c3054c486cbee1ef +Subproject commit 30e6ff1a0735f56c015cf979bc1feda38ff00dbf From cce09b75de5b354a9a86630d5cd66a94bfea3469 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 25 Sep 2014 17:53:27 -0700 Subject: [PATCH 21/71] Changing prefix for packet sources/dumper from ':' to '%'. Addresses BIT-1249. --- CHANGES | 14 ++++++++++++++ VERSION | 2 +- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- src/iosource/Component.cc | 4 ++-- src/iosource/Manager.cc | 2 +- testing/btest/plugins/pktdumper.bro | 2 +- testing/btest/plugins/pktsrc.bro | 2 +- 11 files changed, 25 insertions(+), 11 deletions(-) diff --git a/CHANGES b/CHANGES index 85de307c2a..b59145a138 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,18 @@ +2.3-196 | 2014-09-25 17:53:27 -0700 + + * Changing prefix for packet sources/dumper from ':' to '%'. + Addresses BIT-1249. (Robin Sommer) + + * Remove timeouts from remote communication loop. The select() now + blocks until there's work to do instead of relying on a small + timeout value which can cause unproductive use of cpu cycles. (Jon + Siwek) + + * Improve error message when failing to activate a plugin. Also fix + a unit test helper script that checks plugin availability. (Jon + Siwek) + 2.3-183 | 2014-09-24 10:08:04 -0500 * Add a "node" field to Intel::Seen struture and intel.log to diff --git a/VERSION b/VERSION index 5e605d2618..9301ded595 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-183 +2.3-196 diff --git a/aux/binpac b/aux/binpac index 30c156d879..3a4684801a 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 30c156d879f6303f15ebf8e59989d8a42d882bdf +Subproject commit 3a4684801aafa0558383199e9abd711650b53af9 diff --git a/aux/bro-aux b/aux/bro-aux index 30e6ff1a07..95afe42e74 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 30e6ff1a0735f56c015cf979bc1feda38ff00dbf +Subproject commit 95afe42e7474113a16cb2cb09ebdf8b552c59744 diff --git a/aux/broccoli b/aux/broccoli index 07cfcc76fb..33d0ed4a54 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 07cfcc76fb08365b545bd3f412c3f6e6c92824e9 +Subproject commit 33d0ed4a54a6ecf08a0b5fe18831aa413b437066 diff --git a/aux/broctl b/aux/broctl index 2606a95c9d..2f808bc854 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2606a95c9dcbc83bd863c2981ce7189e5d58697b +Subproject commit 2f808bc8541378b1a4953cca02c58c43945d154f diff --git a/cmake b/cmake index f2e8ba6b90..03de0cc467 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit f2e8ba6b90b3a2da9f1f77c55d0e718c25376bbb +Subproject commit 03de0cc467d2334dcb851eddd843d59fef217909 diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc index a285cd8552..1e4f029400 100644 --- a/src/iosource/Component.cc +++ b/src/iosource/Component.cc @@ -24,7 +24,7 @@ Component::~Component() PktSrcComponent::PktSrcComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) : iosource::Component(plugin::component::PKTSRC, arg_name) { - tokenize_string(arg_prefix, ":", &prefixes); + tokenize_string(arg_prefix, "%", &prefixes); type = arg_type; factory = arg_factory; } @@ -110,7 +110,7 @@ void PktSrcComponent::DoDescribe(ODesc* d) const PktDumperComponent::PktDumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory) : plugin::Component(plugin::component::PKTDUMPER, name) { - tokenize_string(arg_prefix, ":", &prefixes); + tokenize_string(arg_prefix, "%", &prefixes); factory = arg_factory; } diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 41118bdbfe..26a48c2911 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -201,7 +201,7 @@ static std::pair split_prefix(std::string path) // PktSrc to use. If not, choose default. std::string prefix; - std::string::size_type i = path.find(":"); + std::string::size_type i = path.find("%"); if ( i != std::string::npos ) { prefix = path.substr(0, i); diff --git a/testing/btest/plugins/pktdumper.bro b/testing/btest/plugins/pktdumper.bro index 61540897d8..64eda98cdd 100644 --- a/testing/btest/plugins/pktdumper.bro +++ b/testing/btest/plugins/pktdumper.bro @@ -3,6 +3,6 @@ # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace -w foo:XXX %INPUT FilteredTraceDetection::enable=F >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace -w foo/XXX %INPUT FilteredTraceDetection::enable=F >>output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/plugins/pktsrc.bro b/testing/btest/plugins/pktsrc.bro index 39d2fa9aff..87f23d9f75 100644 --- a/testing/btest/plugins/pktsrc.bro +++ b/testing/btest/plugins/pktsrc.bro @@ -3,6 +3,6 @@ # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo:XXX %INPUT FilteredTraceDetection::enable=F >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo/XXX %INPUT FilteredTraceDetection::enable=F >>output # @TEST-EXEC: btest-diff conn.log From 57d0346789e9a15f0ca8a6e6761cc4994ad73cb6 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 26 Sep 2014 10:59:40 -0500 Subject: [PATCH 22/71] Make unexpected pipe errors fatal as precaution. Addresses BIT-1260. --- src/Flare.cc | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/src/Flare.cc b/src/Flare.cc index 960e66cbf4..dcb5fa2c1f 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "Flare.h" +#include "Reporter.h" #include #include #include @@ -12,6 +13,13 @@ Flare::Flare() { } +static void bad_pipe_op(const char* which) + { + char buf[256]; + strerror_r(errno, buf, sizeof(buf)); + reporter->FatalErrorWithCore("unexpected pipe %s failure: %s", which, buf); + } + void Flare::Fire() { char tmp; @@ -24,11 +32,20 @@ void Flare::Fire() // Success -- wrote a byte to pipe. break; - if ( n < 0 && errno == EAGAIN ) - // Success -- pipe is full and just need at least one byte in it. - break; + if ( n < 0 ) + { + if ( errno == EAGAIN ) + // Success: pipe is full and just need at least one byte in it. + break; - // Loop because either the byte wasn't written or got EINTR error. + if ( errno == EINTR ) + // Interrupted: try again. + continue; + + bad_pipe_op("write"); + } + + // No error, but didn't write a byte: try again. } } @@ -37,7 +54,21 @@ void Flare::Extinguish() char tmp[256]; for ( ; ; ) - if ( read(pipe.ReadFD(), &tmp, sizeof(tmp)) == -1 && errno == EAGAIN ) - // Pipe is now drained. + { + int n = read(pipe.ReadFD(), &tmp, sizeof(tmp)); + + if ( n >= 0 ) + // Pipe may not be empty yet: try again. + continue; + + if ( errno == EAGAIN ) + // Success: pipe is now empty. break; + + if ( errno == EINTR ) + // Interrupted: try again. + continue; + + bad_pipe_op("read"); + } } From f933899b172647c58779ed914612ba182612b659 Mon Sep 17 00:00:00 2001 From: Hui Lin Date: Fri, 26 Sep 2014 14:47:51 -0500 Subject: [PATCH 23/71] adding a function in dnp3-analyzer.pac to translate the time stamp format --- src/analyzer/protocol/dnp3/dnp3-analyzer.pac | 44 ++++++++++++++----- src/analyzer/protocol/dnp3/events.bif | 24 +++++----- .../scripts/base/protocols/dnp3/events.bro | 24 +++++----- 3 files changed, 56 insertions(+), 36 deletions(-) diff --git a/src/analyzer/protocol/dnp3/dnp3-analyzer.pac b/src/analyzer/protocol/dnp3/dnp3-analyzer.pac index 2ae783c82e..bccf1376e4 100644 --- a/src/analyzer/protocol/dnp3/dnp3-analyzer.pac +++ b/src/analyzer/protocol/dnp3/dnp3-analyzer.pac @@ -4,6 +4,26 @@ connection DNP3_Conn(bro_analyzer: BroAnalyzer) { downflow = DNP3_Flow(false); }; +%header{ + uint64 bytestring_to_time(const_bytestring time48); + %} + +%code{ + uint64 bytestring_to_time(const_bytestring time48) + { + /* in DNP3, time stamp is represented by 6 bytes to epoch in millisecond */ + /* 6 bytes are stored following big endian format */ + uint64 epochTime = 0; + + for ( uint i = 0 ; i < 6; i++) + { + epochTime = time48[5-i] + epochTime * 256; + } + + return epochTime; + } + %} + flow DNP3_Flow(is_orig: bool) { flowunit = DNP3_PDU(is_orig) withcontext (connection, this); @@ -222,7 +242,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_counter_32wFlagTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, count_value, bytestring_to_val(time48)); + is_orig(), flag, count_value, bytestring_to_time(time48)); } return true; @@ -236,7 +256,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_counter_16wFlagTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, count_value, bytestring_to_val(time48)); + is_orig(), flag, count_value, bytestring_to_time(time48)); } return true; @@ -390,7 +410,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_32wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_val(time48)); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -404,7 +424,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_16wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_val(time48)); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -502,7 +522,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_32wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value, bytestring_to_val(time48)); + is_orig(), flag, value, bytestring_to_time(time48)); } return true; @@ -516,7 +536,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_16wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value, bytestring_to_val(time48)); + is_orig(), flag, value, bytestring_to_time(time48)); } return true; @@ -558,7 +578,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_SPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value, bytestring_to_val(time48)); + is_orig(), flag, value, bytestring_to_time(time48)); } return true; @@ -572,7 +592,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_DPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value_low, value_high, bytestring_to_val(time48)); + is_orig(), flag, value_low, value_high, bytestring_to_time(time48)); } return true; @@ -614,7 +634,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_32wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_val(time48)); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -628,7 +648,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_16wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_val(time48)); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -670,7 +690,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_SPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_val(time48)); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -684,7 +704,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_DPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_val(time48)); + is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_time(time48)); } return true; diff --git a/src/analyzer/protocol/dnp3/events.bif b/src/analyzer/protocol/dnp3/events.bif index 80f9504a9e..18fc42d6f7 100644 --- a/src/analyzer/protocol/dnp3/events.bif +++ b/src/analyzer/protocol/dnp3/events.bif @@ -130,11 +130,11 @@ event dnp3_frozen_counter_16wFlag%(c: connection, is_orig: bool, flag:count, cou ## Generated for DNP3 objects with the group number 21 and variation number 5 ## frozen counter 32 bit with flag and time -event dnp3_frozen_counter_32wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: string%); +event dnp3_frozen_counter_32wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: count%); ## Generated for DNP3 objects with the group number 21 and variation number 6 ## frozen counter 16 bit with flag and time -event dnp3_frozen_counter_16wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: string%); +event dnp3_frozen_counter_16wFlagTime%(c: connection, is_orig: bool, flag:count, count_value: count, time48: count%); ## Generated for DNP3 objects with the group number 21 and variation number 9 ## frozen counter 32 bit without flag @@ -178,11 +178,11 @@ event dnp3_frozen_analog_input_16wFlag%(c: connection, is_orig: bool, flag: coun ## Generated for DNP3 objects with the group number 31 and variation number 3 ## frozen analog input 32 bit with time-of-freeze -event dnp3_frozen_analog_input_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%); +event dnp3_frozen_analog_input_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%); ## Generated for DNP3 objects with the group number 31 and variation number 4 ## frozen analog input 16 bit with time-of-freeze -event dnp3_frozen_analog_input_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%); +event dnp3_frozen_analog_input_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%); ## Generated for DNP3 objects with the group number 31 and variation number 5 ## frozen analog input 32 bit without flag @@ -210,11 +210,11 @@ event dnp3_analog_input_event_16woTime%(c: connection, is_orig: bool, flag: coun ## Generated for DNP3 objects with the group number 32 and variation number 3 ## analog input event 32 bit with time -event dnp3_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%); +event dnp3_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: count%); ## Generated for DNP3 objects with the group number 32 and variation number 4 ## analog input event 16 bit with time -event dnp3_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%); +event dnp3_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, value: count, time48: count%); ## Generated for DNP3 objects with the group number 32 and variation number 5 ## analog input event single-precision float point without time @@ -226,11 +226,11 @@ event dnp3_analog_input_event_DPwoTime%(c: connection, is_orig: bool, flag: coun ## Generated for DNP3 objects with the group number 32 and variation number 7 ## analog input event single-precision float point with time -event dnp3_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, value: count, time48: string%); +event dnp3_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, value: count, time48: count%); ## Generated for DNP3 objects with the group number 32 and variation number 8 ## analog input event double-precisiion float point with time -event dnp3_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: string%); +event dnp3_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: count%); ## Generated for DNP3 objects with the group number 33 and variation number 1 ## frozen analog input event 32 bit without time @@ -242,11 +242,11 @@ event dnp3_frozen_analog_input_event_16woTime%(c: connection, is_orig: bool, fla ## Generated for DNP3 objects with the group number 33 and variation number 3 ## frozen analog input event 32 bit with time -event dnp3_frozen_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%); +event dnp3_frozen_analog_input_event_32wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%); ## Generated for DNP3 objects with the group number 33 and variation number 4 ## frozen analog input event 16 bit with time -event dnp3_frozen_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%); +event dnp3_frozen_analog_input_event_16wTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%); ## Generated for DNP3 objects with the group number 33 and variation number 5 ## frozen analog input event single-precision float point without time @@ -258,11 +258,11 @@ event dnp3_frozen_analog_input_event_DPwoTime%(c: connection, is_orig: bool, fla ## Generated for DNP3 objects with the group number 33 and variation number 7 ## frozen analog input event single-precision float point with time -event dnp3_frozen_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string%); +event dnp3_frozen_analog_input_event_SPwTime%(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count%); ## Generated for DNP3 objects with the group number 34 and variation number 8 ## frozen analog input event double-precision float point with time -event dnp3_frozen_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: string%); +event dnp3_frozen_analog_input_event_DPwTime%(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: count%); ## g70 event dnp3_file_transport%(c: connection, is_orig: bool, file_handle: count, block_num: count, file_data: string%); diff --git a/testing/btest/scripts/base/protocols/dnp3/events.bro b/testing/btest/scripts/base/protocols/dnp3/events.bro index aff5191d7f..9c968bdaac 100644 --- a/testing/btest/scripts/base/protocols/dnp3/events.bro +++ b/testing/btest/scripts/base/protocols/dnp3/events.bro @@ -82,12 +82,12 @@ event dnp3_frozen_counter_16wFlag(c: connection, is_orig: bool, flag:count, coun print "dnp3_frozen_counter_16wFlag", is_orig, flag; } -event dnp3_frozen_counter_32wFlagTime(c: connection, is_orig: bool, flag:count, count_value: count, time48: string) +event dnp3_frozen_counter_32wFlagTime(c: connection, is_orig: bool, flag:count, count_value: count, time48: count) { print "dnp3_frozen_counter_32wFlagTime", is_orig, flag; } -event dnp3_frozen_counter_16wFlagTime(c: connection, is_orig: bool, flag:count, count_value: count, time48: string) +event dnp3_frozen_counter_16wFlagTime(c: connection, is_orig: bool, flag:count, count_value: count, time48: count) { print "dnp3_frozen_counter_16wFlagTime", is_orig, flag; } @@ -142,12 +142,12 @@ event dnp3_frozen_analog_input_16wFlag(c: connection, is_orig: bool, flag: count print "dnp3_frozen_analog_input_16wFlag", is_orig, flag, frozen_value; } -event dnp3_frozen_analog_input_32wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string) +event dnp3_frozen_analog_input_32wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count) { print "dnp3_frozen_analog_input_32wTime", is_orig, flag, frozen_value, time48; } -event dnp3_frozen_analog_input_16wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string) +event dnp3_frozen_analog_input_16wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count) { print "dnp3_frozen_analog_input_16wTime", is_orig, flag, frozen_value, time48; } @@ -182,12 +182,12 @@ event dnp3_analog_input_event_16woTime(c: connection, is_orig: bool, flag: count print "dnp3_analog_input_event_16woTime", is_orig, flag, value; } -event dnp3_analog_input_event_32wTime(c: connection, is_orig: bool, flag: count, value: count, time48: string) +event dnp3_analog_input_event_32wTime(c: connection, is_orig: bool, flag: count, value: count, time48: count) { print "dnp3_analog_input_event_32wTime", is_orig, flag, value, time48; } -event dnp3_analog_input_16wTime(c: connection, is_orig: bool, flag: count, value: count, time48: string) +event dnp3_analog_input_16wTime(c: connection, is_orig: bool, flag: count, value: count, time48: count) { print "dnp3_analog_input_event_16wTime", is_orig, flag, value, time48; } @@ -202,12 +202,12 @@ event dnp3_analog_inputDP_woTime(c: connection, is_orig: bool, flag: count, valu print "dnp3_analog_input_event_DPwoTime", is_orig, flag, value_low, value_high; } -event dnp3_analog_inputSP_wTime(c: connection, is_orig: bool, flag: count, value: count, time48: string) +event dnp3_analog_inputSP_wTime(c: connection, is_orig: bool, flag: count, value: count, time48: count) { print "dnp3_analog_input_event_SPwTime", is_orig, flag, value, time48; } -event dnp3_analog_inputDP_wTime(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: string) +event dnp3_analog_inputDP_wTime(c: connection, is_orig: bool, flag: count, value_low: count, value_high: count, time48: count) { print "dnp3_analog_input_event_DPwTime", is_orig, flag, value_low, value_high, time48; } @@ -222,12 +222,12 @@ event dnp3_frozen_analog_input_event_16woTime(c: connection, is_orig: bool, flag print "dnp3_frozen_analog_input_event_16woTime", is_orig, flag, frozen_value; } -event dnp3_frozen_analog_input_event_32wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string) +event dnp3_frozen_analog_input_event_32wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count) { print "dnp3_frozen_analog_input_event_32wTime", is_orig, flag, frozen_value, time48; } -event dnp3_frozen_analog_input_event_16wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string) +event dnp3_frozen_analog_input_event_16wTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count) { print "dnp3_frozen_analog_input_event_16wTime", is_orig, flag, frozen_value, time48; } @@ -242,12 +242,12 @@ event dnp3_frozen_analog_input_event_DPwoTime(c: connection, is_orig: bool, flag print "dnp3_frozen_analog_input_event_DPwoTime", is_orig, flag, frozen_value_low, frozen_value_high; } -event dnp3_frozen_analog_input_event_SPwTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: string) +event dnp3_frozen_analog_input_event_SPwTime(c: connection, is_orig: bool, flag: count, frozen_value: count, time48: count) { print "dnp3_frozen_analog_inputeventSP_wTime", is_orig, flag, frozen_value, time48; } -event dnp3_frozen_analog_input_event_DPwTime(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: string) +event dnp3_frozen_analog_input_event_DPwTime(c: connection, is_orig: bool, flag: count, frozen_value_low: count, frozen_value_high: count, time48: count) { print "dnp3_frozen_analog_inputeventDP_wTime", is_orig, flag, frozen_value_low, frozen_value_high, time48; } From 6dc4863d8150a4e8f48c58dd4284ca5b8ffd531e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Fri, 26 Sep 2014 22:06:56 -0500 Subject: [PATCH 24/71] Add a test that detects changes in the list of all Bro log files --- .../btest/Baseline/coverage.find-bro-logs/out | 42 +++++++++++++++++ testing/btest/coverage/find-bro-logs.test | 45 +++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 testing/btest/Baseline/coverage.find-bro-logs/out create mode 100644 testing/btest/coverage/find-bro-logs.test diff --git a/testing/btest/Baseline/coverage.find-bro-logs/out b/testing/btest/Baseline/coverage.find-bro-logs/out new file mode 100644 index 0000000000..090a93d655 --- /dev/null +++ b/testing/btest/Baseline/coverage.find-bro-logs/out @@ -0,0 +1,42 @@ +app_stats +barnyard2 +capture_loss +cluster +communication +conn +dhcp +dnp3 +dns +dpd +files +ftp +http +intel +irc +known_certs +known_devices +known_hosts +known_modbus +known_services +loaded_scripts +modbus +modbus_register_change +notice +notice_alarm +packet_filter +radius +reporter +signatures +smtp +snmp +socks +software +ssh +ssl +stats +syslog +traceroute +tunnel +unified2 +weird +x509 diff --git a/testing/btest/coverage/find-bro-logs.test b/testing/btest/coverage/find-bro-logs.test new file mode 100644 index 0000000000..ffde5b8225 --- /dev/null +++ b/testing/btest/coverage/find-bro-logs.test @@ -0,0 +1,45 @@ +# This test is intended to help keep Bro's reference documentation up-to-date. +# If this test fails, then it indicates that the set of all the log filenames +# that Bro could potentially create (with the scripts included with Bro) has +# changed. In that case, the reference documentation listing all Bro log files +# should be checked and updated if necessary. + +# @TEST-EXEC: bash %INPUT +# @TEST-EXEC: btest-diff out + +BROSCRIPTS=${DIST}/scripts + +# For a given Bro script, look for a call to "create_stream". If found, +# extract the log ID (adding the module name if necessary), and print the +# log ID and script filename. +cat << '_EOF_' > find_logid.awk +/module[ ]+[A-Za-z0-9_]/ { + mod = $2 + if ( substr(mod, length(mod), 1) == ";" ) { + mod = substr(mod, 1, length(mod)-1) + } +} + +/Log::create_stream/ { + if ( substr($1, 1, 1) != "#" ) { + x = index($1, "(") + logid = substr($1, x+1, length($1)-x-1) + if ( logid == "LOG" ) { + printf "%s::", mod + } + printf "%s", logid + printf " %s\n", FILENAME + } +} +_EOF_ + +find ${BROSCRIPTS} -type f -exec awk -f find_logid.awk {} \; > out.logid + +# For each log ID, have Bro convert it to the corresponding log filename +# using the default mechanism for generating a log filename (we must load +# all Bro scripts so that all log IDs are defined). +awk '{print $1}' out.logid | while read logid; do + bro ${BROSCRIPTS}/test-all-policy.bro -e "print Log::default_path_func(${logid}, \"\", 0);" >> out.tmp +done + +grep -v WARNING out.tmp | sort -u > out From 470d86855822b362470f1b7eaaaf934b26a43d7e Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Sun, 28 Sep 2014 14:29:12 +0200 Subject: [PATCH 25/71] new ssl extension type from iana and a few other ssl const changes. --- scripts/base/protocols/ssl/consts.bro | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index a19aaecbe5..54952988f0 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -30,6 +30,7 @@ export { const HELLO_REQUEST = 0; const CLIENT_HELLO = 1; const SERVER_HELLO = 2; + const HELLO_VERIFY_REQUEST = 3; # RFC 6347 const SESSION_TICKET = 4; # RFC 5077 const CERTIFICATE = 11; const SERVER_KEY_EXCHANGE = 12; @@ -40,6 +41,7 @@ export { const FINISHED = 20; const CERTIFICATE_URL = 21; # RFC 3546 const CERTIFICATE_STATUS = 22; # RFC 3546 + const SUPPLEMENTAL_DATA = 23; # RFC 4680 ## Mapping between numeric codes and human readable strings for alert ## levels. @@ -112,7 +114,8 @@ export { [19] = "client_certificate_type", [20] = "server_certificate_type", [21] = "padding", # temporary till 2015-03-12 - [22] = "encrypt_then_mac", # temporary till 2015-06-05 + [22] = "encrypt_then_mac", + [23] = "extended_master_secret", # temporary till 2015-09-26 [35] = "SessionTicket TLS", [40] = "extended_random", [13172] = "next_protocol_negotiation", From c7354c6fa071de91149aa836e2d76197be2d9f8c Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 29 Sep 2014 10:42:01 -0500 Subject: [PATCH 26/71] Fix possible seg fault in TCP reassembler. --- CHANGES | 4 ++++ VERSION | 2 +- src/analyzer/protocol/tcp/TCP_Reassembler.cc | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index b59145a138..bf49f94c70 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-197 | 2014-09-29 10:42:01 -0500 + + * Fix possible seg fault in TCP reassembler. (Jon Siwek) + 2.3-196 | 2014-09-25 17:53:27 -0700 * Changing prefix for packet sources/dumper from ':' to '%'. diff --git a/VERSION b/VERSION index 9301ded595..b5cf6187a5 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-196 +2.3-197 diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 921f3a3204..0f7699011e 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -249,7 +249,9 @@ void TCP_Reassembler::Undelivered(uint64 up_to_seq) Gap(gap_at_seq, gap_len); last_reassem_seq += gap_len; BlockInserted(b); - b = b->next; + // Inserting a block may cause trimming of what's buffered, + // so have to assume 'b' is invalid, hence re-assign to start. + b = blocks; } if ( up_to_seq > last_reassem_seq ) From 999f846abecfcbe48cd4691c5f63a727306907d4 Mon Sep 17 00:00:00 2001 From: Jeannette Dopheide Date: Mon, 29 Sep 2014 10:50:46 -0500 Subject: [PATCH 27/71] Added missing log files prof, stderr, stdout --- doc/script-reference/log-files.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 0098bce4c0..4a539ea7da 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -34,12 +34,20 @@ Bro Diagnostics +----------------------------+---------------------------------------+---------------------------------+ | packetfilter.log | Status of packet filters | :bro:type:`PacketFilter::Info` | +----------------------------+---------------------------------------+---------------------------------+ +| prof.log | Generates profiling statistics | :bro:id:`do_profiling` | ++----------------------------+---------------------------------------+---------------------------------+ | reporter.log | Records error messages, location, | :bro:type:`Reporter::Info` | | | and severity | | +----------------------------+---------------------------------------+---------------------------------+ | stats.log | Shows log memory/packet/lag | :bro:type:`Stats::Info` | | | statistics | | +----------------------------+---------------------------------------+---------------------------------+ +| stderr.log | Standard error log when running | n/a | +| | BroControl | | ++----------------------------+---------------------------------------+---------------------------------+ +| stdout.log | Standard output log when running | n/a | +| | BroControl | | ++----------------------------+---------------------------------------+---------------------------------+ | unified2.log | Interprets Snort's unified output | :bro:type:`Unified2::Info` | | | format | | +----------------------------+---------------------------------------+---------------------------------+ From d9889d489fb0846241b350bd6d374973b4bc53c0 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Mon, 29 Sep 2014 20:06:54 -0700 Subject: [PATCH 28/71] Fix to use length parameter in DNP3 time conversion correctly now. --- CHANGES | 5 +++++ VERSION | 2 +- src/analyzer/protocol/dnp3/dnp3-analyzer.pac | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index fc8f67b985..9da6a5d5d5 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-203 | 2014-09-29 20:06:54 -0700 + + * Fix to use length parameter in DNP3 time conversion correctly now. + (Robin Sommer) + 2.3-202 | 2014-09-29 17:05:18 -0700 * New SSL extension type from IANA and a few other SSL const diff --git a/VERSION b/VERSION index d109a62b77..3065f78463 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-202 +2.3-203 diff --git a/src/analyzer/protocol/dnp3/dnp3-analyzer.pac b/src/analyzer/protocol/dnp3/dnp3-analyzer.pac index 4700c6b7c2..2065237f45 100644 --- a/src/analyzer/protocol/dnp3/dnp3-analyzer.pac +++ b/src/analyzer/protocol/dnp3/dnp3-analyzer.pac @@ -16,7 +16,7 @@ connection DNP3_Conn(bro_analyzer: BroAnalyzer) { uint64 epochTime = 0; for ( unsigned int i = 0; i < length; i++ ) - epochTime = time48[5-i] + epochTime * 256; + epochTime = time48[length - i - 1] + epochTime * 256; return epochTime; } From c16384b914aa79053bcb06e678602ed25c294a2e Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Tue, 30 Sep 2014 00:45:28 -0500 Subject: [PATCH 29/71] Improved the log file reference documentation Reorganized the log file reference documentation, improved some of the descriptions, and corrected a typo in a log filename. Also removed non-ascii characters that somehow got in the text. --- doc/script-reference/log-files.rst | 227 ++++++++++++++--------------- 1 file changed, 108 insertions(+), 119 deletions(-) diff --git a/doc/script-reference/log-files.rst b/doc/script-reference/log-files.rst index 4a539ea7da..f341c35f40 100644 --- a/doc/script-reference/log-files.rst +++ b/doc/script-reference/log-files.rst @@ -2,14 +2,105 @@ Log Files ========= -As a monitoring tool, Bro records a detailed view of the traffic inspected -and the events generated in a series of relevant log files. These files can -later be reviewed for monitoring, auditing and troubleshooting purposes. - Listed below are the log files generated by Bro, including a brief description -of the log file and links to descriptions of some of the fields for each log +of the log file and links to descriptions of the fields for each log type. +Network Protocols +----------------- + ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| conn.log | TCP/UDP/ICMP connections | :bro:type:`Conn::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dhcp.log | DHCP leases | :bro:type:`DHCP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dnp3.log | DNP3 requests and replies | :bro:type:`DNP3::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dns.log | DNS activity | :bro:type:`DNS::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| ftp.log | FTP activity | :bro:type:`FTP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| http.log | HTTP requests and replies | :bro:type:`HTTP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| irc.log | IRC commands and responses | :bro:type:`IRC::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| modbus.log | Modbus commands and responses | :bro:type:`Modbus::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| radius.log | RADIUS authentication attempts | :bro:type:`RADIUS::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| smtp.log | SMTP transactions | :bro:type:`SMTP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| snmp.log | SNMP messages | :bro:type:`SNMP::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| socks.log | SOCKS proxy requests | :bro:type:`SOCKS::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| ssh.log | SSH connections | :bro:type:`SSH::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| ssl.log | SSL/TLS handshake info | :bro:type:`SSL::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| syslog.log | Syslog messages | :bro:type:`Syslog::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| tunnel.log | Tunneling protocol events | :bro:type:`Tunnel::Info` | ++----------------------------+---------------------------------------+---------------------------------+ + +Other Logs +---------- + ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| barnyard2.log | Alerts received from Barnyard2 | :bro:type:`Barnyard2::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| dpd.log | Dynamic protocol detection failures | :bro:type:`DPD::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| files.log | File analysis results | :bro:type:`Files::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| intel.log | Intelligence data matches | :bro:type:`Intel::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| modbus_register_change.log | Tracks changes to Modbus holding | :bro:type:`Modbus::MemmapInfo` | +| | registers | | ++----------------------------+---------------------------------------+---------------------------------+ +| notice.log | Bro notices | :bro:type:`Notice::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| notice_alarm.log | The alarm stream | :bro:enum:`Notice::ACTION_ALARM`| ++----------------------------+---------------------------------------+---------------------------------+ +| signatures.log | Signature matches | :bro:type:`Signatures::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| traceroute.log | Traceroute detection | :bro:type:`Traceroute::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| unified2.log | Interprets Snort's unified output | :bro:type:`Unified2::Info` | +| | format | | ++----------------------------+---------------------------------------+---------------------------------+ +| weird.log | Unexpected protocol-level activity | :bro:type:`Weird::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| x509.log | X.509 certificate info | :bro:type:`X509::Info` | ++----------------------------+---------------------------------------+---------------------------------+ + +Network Observations +-------------------- + ++----------------------------+---------------------------------------+---------------------------------+ +| Log File | Description | Field Descriptions | ++============================+=======================================+=================================+ +| app_stats.log | Web app usage statistics | :bro:type:`AppStats::Info` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_certs.log | SSL certificates | :bro:type:`Known::CertsInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_devices.log | MAC addresses of devices on the | :bro:type:`Known::DevicesInfo` | +| | network | | ++----------------------------+---------------------------------------+---------------------------------+ +| known_hosts.log | Hosts that have completed TCP | :bro:type:`Known::HostsInfo` | +| | handshakes | | ++----------------------------+---------------------------------------+---------------------------------+ +| known_modbus.log | Modbus masters and slaves | :bro:type:`Known::ModbusInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| known_services.log | Services running on hosts | :bro:type:`Known::ServicesInfo` | ++----------------------------+---------------------------------------+---------------------------------+ +| software.log | Software being used on the network | :bro:type:`Software::Info` | ++----------------------------+---------------------------------------+---------------------------------+ + Bro Diagnostics --------------- @@ -18,128 +109,26 @@ Bro Diagnostics +============================+=======================================+=================================+ | capture_loss.log | Packet loss rate | :bro:type:`CaptureLoss::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| cluster.log | Cluster messages | :bro:type:`Cluster::Info` | +| cluster.log | Bro cluster messages | :bro:type:`Cluster::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| communication.log | Connections to remote Bro or Broccoli | :bro:type:`Communication::Info` | -| | instances | | -+----------------------------+---------------------------------------+---------------------------------+ -| intel.log | Details about the intelligence | :bro:type:`Intel::Info` | -| | framework | | +| communication.log | Communication events between Bro or | :bro:type:`Communication::Info` | +| | Broccoli instances | | +----------------------------+---------------------------------------+---------------------------------+ | loaded_scripts.log | Shows all scripts loaded by Bro | :bro:type:`LoadedScripts::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| notice.log | Bro notices | :bro:type:`Notice::Info` | +| packet_filter.log | List packet filters that were applied | :bro:type:`PacketFilter::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| notice_alarm.log | The alarm stream | :bro:enum:`Notice::ACTION_ALARM`| +| prof.log | Profiling statistics (to create this | N/A | +| | log, load policy/misc/profiling.bro) | | +----------------------------+---------------------------------------+---------------------------------+ -| packetfilter.log | Status of packet filters | :bro:type:`PacketFilter::Info` | +| reporter.log | Internal error/warning/info messages | :bro:type:`Reporter::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| prof.log | Generates profiling statistics | :bro:id:`do_profiling` | +| stats.log | Memory/event/packet/lag statistics | :bro:type:`Stats::Info` | +----------------------------+---------------------------------------+---------------------------------+ -| reporter.log | Records error messages, location, | :bro:type:`Reporter::Info` | -| | and severity | | +| stderr.log | Captures standard error when Bro is | N/A | +| | started from BroControl | | +----------------------------+---------------------------------------+---------------------------------+ -| stats.log | Shows log memory/packet/lag | :bro:type:`Stats::Info` | -| | statistics | | -+----------------------------+---------------------------------------+---------------------------------+ -| stderr.log | Standard error log when running | n/a | -| | BroControl | | -+----------------------------+---------------------------------------+---------------------------------+ -| stdout.log | Standard output log when running | n/a | -| | BroControl | | -+----------------------------+---------------------------------------+---------------------------------+ -| unified2.log | Interprets Snort's unified output | :bro:type:`Unified2::Info` | -| | format | | +| stdout.log | Captures standard output when Bro is | N/A | +| | started from BroControl | | +----------------------------+---------------------------------------+---------------------------------+ -Known_* Logs ------------- - -+----------------------------+---------------------------------------+---------------------------------+ -| Log File | Description | Field Descriptions | -+============================+=======================================+=================================+ -| known_certs.log | SSL certificates used | :bro:type:`Known::CertsInfo` | -+----------------------------+---------------------------------------+---------------------------------+ -| known_devices.log | MAC addresses of devices on the | :bro:type:`Known::DevicesInfo` | -| | network | | -+----------------------------+---------------------------------------+---------------------------------+ -| known_hosts.log | Daily record of completed TCP | :bro:type:`Known::HostsInfo` | -| | handshakes | | -+----------------------------+---------------------------------------+---------------------------------+ -| known_modbus.log | Modbus masters and workers | :bro:type:`Known::ModbusInfo` | -+----------------------------+---------------------------------------+---------------------------------+ -| known_services.log | Tracks services and protocols used | :bro:type:`Known::ServicesInfo` | -| | during a session | | -+----------------------------+---------------------------------------+---------------------------------+ - -Network Activity ----------------- - -+----------------------------+---------------------------------------+---------------------------------+ -| Log File | Description | Field Descriptions | -+============================+=======================================+=================================+ -| barnyard2.log | Alerts received from Barnyard2 | :bro:type:`Barnyard2::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| conn.log  | Connection info | :bro:type:`Conn::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| dhcp.log  | DHCP leases | :bro:type:`DHCP::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| dnp3.log | Requests and replies using DNP3 | :bro:type:`DNP3::Info` | -| | protocol | | -+----------------------------+---------------------------------------+---------------------------------+ -| dns.log  | DNS activity | :bro:type:`DNS::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| dpd.log | Network activity on non-standard | :bro:type:`DPD::Info` | -| | ports | | -+----------------------------+---------------------------------------+---------------------------------+ -| files.log | Info about files transmitted over the | :bro:type:`Files::Info` | -| | network | | -+----------------------------+---------------------------------------+---------------------------------+ -| ftp.log | FTP activity | :bro:type:`FTP::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| http.log | HTTP requests and replies | :bro:type:`HTTP::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| irc.log | IRC commands and responses | :bro:type:`IRC::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| modbus.log | Modbus protocol data | :bro:type:`Modbus::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| modbus_register_change.log | Tracks changes to holding registers | :bro:type:`Modbus::MemmapInfo` | -+----------------------------+---------------------------------------+---------------------------------+ -| radius.log  | RADIUS authentication attempts | :bro:type:`RADIUS::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| signatures.log | Tracks signatures used on TCP | :bro:type:`Signatures::Info` | -| | connections | | -+----------------------------+---------------------------------------+---------------------------------+ -| smtp.log | SMTP traffic on a network | :bro:type:`SMTP::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| snmp.log  | SNMP traffic on a network | :bro:type:`SNMP::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| socks.log | SOCKS proxy requests | :bro:type:`SOCKS::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| ssh.log  | SSH connections | :bro:type:`SSH::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| ssl.log  | SSL/TLS handshake info | :bro:type:`SSL::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| syslog.log  | Syslog messages and data | :bro:type:`Syslog::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| traceroute.log | Address and protocol data of a given | :bro:type:`Traceroute::Info` | -| | traceroute | | -+----------------------------+---------------------------------------+---------------------------------+ -| tunnel.log | Tunnel data | :bro:type:`Tunnel::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| weird.log | Records unexpected protocol-level | :bro:type:`Weird::Info` | -| | activity | | -+----------------------------+---------------------------------------+---------------------------------+ -| x509.log | Tracks X.509 certificates | :bro:type:`X509::Info` | -+----------------------------+---------------------------------------+---------------------------------+ - -Software Asset Tracking ------------------------ - -+----------------------------+---------------------------------------+---------------------------------+ -| Log File | Description | Field Descriptions | -+============================+=======================================+=================================+ -| app_stats.log | Info about web apps in use on network | :bro:type:`AppStats::Info` | -+----------------------------+---------------------------------------+---------------------------------+ -| software.log | Software being used on the network | :bro:type:`Software::Info` | -+----------------------------+---------------------------------------+---------------------------------+ From 9cd85be308b6a95780d23d01c589467172ec4ef9 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 2 Oct 2014 11:33:37 -0500 Subject: [PATCH 30/71] Fix regression causing the main loop to spin more frequently. Addresses BIT-1266. --- src/iosource/FD_Set.h | 17 +++++++++++++++++ src/iosource/Manager.cc | 3 +++ 2 files changed, 20 insertions(+) diff --git a/src/iosource/FD_Set.h b/src/iosource/FD_Set.h index 402d28b8cb..61e3e7a59b 100644 --- a/src/iosource/FD_Set.h +++ b/src/iosource/FD_Set.h @@ -77,6 +77,23 @@ public: return false; } + /** + * @return whether any file descriptors have been added to the set. + */ + bool Empty() const + { + return fds.empty(); + } + + /** + * @return the greatest file descriptor of all that have been added to the + * set, or -1 if the set is empty. + */ + int Max() const + { + return max; + } + private: int max; std::set fds; diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 26a48c2911..5a087bc4ef 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -117,6 +117,9 @@ IOSource* Manager::FindSoonest(double* ts) src->Clear(); src->src->GetFds(&src->fd_read, &src->fd_write, &src->fd_except); + if ( src->fd_read.Empty() ) src->fd_read.Insert(0); + if ( src->fd_write.Empty() ) src->fd_write.Insert(0); + if ( src->fd_except.Empty() ) src->fd_except.Insert(0); src->SetFds(&fd_read, &fd_write, &fd_except, &maxx); } From 31b7e984d1cc67b63a3385972ece1595bd38aa13 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 2 Oct 2014 12:16:33 -0500 Subject: [PATCH 31/71] Fix packet sources being treated as idle when a packet is available. Addresses BIT-1266. --- src/iosource/PktSrc.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index 6ee95a48a1..eaf85bbfa4 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -434,6 +434,7 @@ bool PktSrc::ExtractNextPacketInternal() if ( ! first_timestamp ) first_timestamp = current_packet.ts; + SetIdle(false); have_packet = true; return 1; } From 1555eb65d4fafe11968f778fb47baec9c22fbeab Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 2 Oct 2014 16:39:17 -0700 Subject: [PATCH 32/71] Updating plugin docs. The remaining components are now supported as well. --- CHANGES | 10 ++++++++++ VERSION | 2 +- doc/devel/plugins.rst | 14 +++++++------- 3 files changed, 18 insertions(+), 8 deletions(-) diff --git a/CHANGES b/CHANGES index 9da6a5d5d5..1709ec8aae 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,14 @@ +2.3-207 | 2014-10-02 16:39:17 -0700 + + * Updating plugin docs. (Robin Sommer) + + * Fix packet sources being treated as idle when a packet is + available. Addresses BIT-1266. (Jon Siwek) + + * Fix regression causing the main loop to spin more frequently. + Addresses BIT-1266. (Jon Siwek) + 2.3-203 | 2014-09-29 20:06:54 -0700 * Fix to use length parameter in DNP3 time conversion correctly now. diff --git a/VERSION b/VERSION index 3065f78463..cc0c518ed4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-203 +2.3-207 diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index 76f5c75a68..a66023cece 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -17,11 +17,11 @@ functionality to Bro: - File analyzers. - - Packet sources and packet dumpers. TODO: Not yet. + - Packet sources and packet dumpers. - - Logging framework backends. TODO: Not yet. + - Logging framework backends. - - Input framework readers. TODO: Not yet. + - Input framework readers. A plugin's functionality is available to the user just as if Bro had the corresponding code built-in. Indeed, internally many of Bro's @@ -315,22 +315,22 @@ TODO. Logging Writer -------------- -Not yet available as plugins. +TODO. Input Reader ------------ -Not yet available as plugins. +TODO. Packet Sources -------------- -Not yet available as plugins. +TODO. Packet Dumpers -------------- -Not yet available as plugins. +TODO. Hooks ===== From b3ff4151208b4be3b07ec2d0206da06b566d1ad4 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 3 Oct 2014 09:38:52 -0500 Subject: [PATCH 33/71] Fix uninitialized router_list argument in dhcp_offer/dhcp_ack. BIT-1268 #close --- CHANGES | 5 +++++ VERSION | 2 +- src/analyzer/protocol/dhcp/dhcp-analyzer.pac | 8 ++++++++ 3 files changed, 14 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 1709ec8aae..6320d72b76 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-208 | 2014-10-03 09:38:52 -0500 + + * BIT-1268: Fix uninitialized router_list argument in + dhcp_offer/dhcp_ack. (Jon Siwek) + 2.3-207 | 2014-10-02 16:39:17 -0700 * Updating plugin docs. (Robin Sommer) diff --git a/VERSION b/VERSION index cc0c518ed4..4fd3661566 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-207 +2.3-208 diff --git a/src/analyzer/protocol/dhcp/dhcp-analyzer.pac b/src/analyzer/protocol/dhcp/dhcp-analyzer.pac index 336c8dc760..a967940ca6 100644 --- a/src/analyzer/protocol/dhcp/dhcp-analyzer.pac +++ b/src/analyzer/protocol/dhcp/dhcp-analyzer.pac @@ -188,6 +188,9 @@ flow DHCP_Flow(is_orig: bool) { switch ( type ) { case DHCPOFFER: + if ( ! router_list ) + router_list = new TableVal(dhcp_router_list); + BifEvent::generate_dhcp_offer(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), dhcp_msg_val_->Ref(), new AddrVal(subnet_mask), @@ -195,6 +198,9 @@ flow DHCP_Flow(is_orig: bool) { break; case DHCPACK: + if ( ! router_list ) + router_list = new TableVal(dhcp_router_list); + BifEvent::generate_dhcp_ack(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), dhcp_msg_val_->Ref(), new AddrVal(subnet_mask), @@ -202,12 +208,14 @@ flow DHCP_Flow(is_orig: bool) { break; case DHCPNAK: + Unref(router_list); BifEvent::generate_dhcp_nak(connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), dhcp_msg_val_->Ref(), host_name); break; default: + Unref(router_list); Unref(host_name); break; } From 80656d5294cb0aebeaf014bf1b9b4fc0b9cdf986 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 6 Oct 2014 11:13:13 -0400 Subject: [PATCH 34/71] Improves shockwave flash file signatures. - This moves the signatures out of the libmagic imported signatures and into our own general.sig. - Expand the detection to LZMA compressed flash files. --- scripts/base/frameworks/files/magic/general.sig | 5 +++++ scripts/base/frameworks/files/magic/libmagic.sig | 13 ------------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index 20276f69ac..a11e4a05e4 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -9,3 +9,8 @@ signature file-tar { file-magic /([[:print:]\x00]){100}(([[:digit:]\x00\x20]){8}){3}/ file-mime "application/x-tar", 150 } + +signature file-swf { + file-magic /(F|C|Z)WS/ + file-mime "application/x-shockwave-flash", 60 +} \ No newline at end of file diff --git a/scripts/base/frameworks/files/magic/libmagic.sig b/scripts/base/frameworks/files/magic/libmagic.sig index 55486d411e..a4604959c3 100644 --- a/scripts/base/frameworks/files/magic/libmagic.sig +++ b/scripts/base/frameworks/files/magic/libmagic.sig @@ -2769,19 +2769,6 @@ signature file-magic-auto408 { file-magic /(.{512})(\xec\xa5\xc1)/ } -# >0 string,=FWS (len=3), ["Macromedia Flash data,"], swap_endian=0 -# >>3 byte&,x, ["version %d"], swap_endian=0 -signature file-magic-auto409 { - file-mime "application/x-shockwave-flash", 1 - file-magic /(FWS)(.{1})/ -} - -# >0 string,=CWS (len=3), ["Macromedia Flash data (compressed),"], swap_endian=0 -signature file-magic-auto410 { - file-mime "application/x-shockwave-flash", 60 - file-magic /(CWS)/ -} - # >0 regex/20,=^\.[A-Za-z0-9][A-Za-z0-9][ \t] (len=29), ["troff or preprocessor input text"], swap_endian=0 signature file-magic-auto411 { file-mime "text/troff", 59 From f24adc1a95e9e1e9e01624d996ab2ca8f67777dd Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Mon, 6 Oct 2014 13:27:21 -0500 Subject: [PATCH 35/71] Minor improvements to script language reference docs --- doc/script-reference/directives.rst | 4 +-- doc/script-reference/operators.rst | 44 ++++++++++++++++++----------- 2 files changed, 30 insertions(+), 18 deletions(-) diff --git a/doc/script-reference/directives.rst b/doc/script-reference/directives.rst index e513e93911..06122fec8b 100644 --- a/doc/script-reference/directives.rst +++ b/doc/script-reference/directives.rst @@ -16,7 +16,7 @@ executed. Directives are evaluated before script execution begins. Example:: - print "Directory:", @DIR + print "Directory:", @DIR; .. bro:keyword:: @FILENAME @@ -25,7 +25,7 @@ executed. Directives are evaluated before script execution begins. Example:: - print "File:", @FILENAME + print "File:", @FILENAME; .. bro:keyword:: @load diff --git a/doc/script-reference/operators.rst b/doc/script-reference/operators.rst index 7fa52cf4b2..07e02bf186 100644 --- a/doc/script-reference/operators.rst +++ b/doc/script-reference/operators.rst @@ -50,31 +50,37 @@ Arithmetic operators +------------------------------+-------------+-------------------------------+ | Name | Syntax | Notes | +==============================+=============+===============================+ -| Addition | *a* + *b* | If operands are strings, then | -| | | this performs string | -| | | concatenation. | +| Addition | *a* + *b* | For :bro:type:`string` | +| | | operands, this performs | +| | | string concatenation. | +------------------------------+-------------+-------------------------------+ | Subtraction | *a* - *b* | | +------------------------------+-------------+-------------------------------+ | Multiplication | *a* \* *b* | | +------------------------------+-------------+-------------------------------+ -| Division | *a* / *b* | | +| Division | *a* / *b* | For :bro:type:`int` or | +| | | :bro:type:`count` operands, | +| | | the fractional part of the | +| | | result is dropped. | +------------------------------+-------------+-------------------------------+ | Modulo | *a* % *b* | Operand types cannot be | -| | | double. | +| | | "double". | +------------------------------+-------------+-------------------------------+ | Unary plus | \+ *a* | | +------------------------------+-------------+-------------------------------+ | Unary minus | \- *a* | | +------------------------------+-------------+-------------------------------+ | Pre-increment | ++ *a* | Operand type cannot be | -| | | double. | +| | | "double". | +------------------------------+-------------+-------------------------------+ | Pre-decrement | ``--`` *a* | Operand type cannot be | -| | | double. | +| | | "double". | +------------------------------+-------------+-------------------------------+ -| Absolute value | \| *a* \| | If operand is string, set, | -| | | table, or vector, this | +| Absolute value | \| *a* \| | If operand is | +| | | :bro:type:`string`, | +| | | :bro:type:`set`, | +| | | :bro:type:`table`, or | +| | | :bro:type:`vector`, this | | | | evaluates to number | | | | of elements. | +------------------------------+-------------+-------------------------------+ @@ -108,7 +114,8 @@ field name must be in the declaration of the record type. +==============================+=============+===============================+ | Field access | *a* $ *b* | | +------------------------------+-------------+-------------------------------+ -| Field value existence test | *a* ?$ *b* | Evaluates to type "bool". | +| Field value existence test | *a* ?$ *b* | Evaluates to type | +| | | :bro:type:`bool`. | | | | True if the specified field | | | | has been assigned a value, or | | | | false if not. | @@ -122,7 +129,7 @@ Other operators | Name | Syntax | Notes | +================================+===================+========================+ | Membership test | *a* in *b* |Evaluates to type | -| | |"bool". Do not | +| | |:bro:type:`bool`. Do not| | | |confuse this use of "in"| | | |with that used in a | | | |:bro:keyword:`for` | @@ -135,7 +142,8 @@ Other operators | | |"!(a in b)". | +--------------------------------+-------------------+------------------------+ | Table or vector element access | *a* [ *b* ] |This operator can also | -| | |be used with a set, but | +| | |be used with a | +| | |:bro:type:`set`, but | | | |only with the | | | |:bro:keyword:`add` or | | | |:bro:keyword:`delete` | @@ -148,8 +156,11 @@ Other operators | Create a deep copy | copy ( *a* ) |This is relevant only | | | |for data types that are | | | |assigned by reference, | -| | |such as "vector", "set",| -| | |"table", and "record". | +| | |such as | +| | |:bro:type:`vector`, | +| | |:bro:type:`set`, | +| | |:bro:type:`table`, | +| | |and :bro:type:`record`. | +--------------------------------+-------------------+------------------------+ | Module namespace access | *a* \:\: *b* |The first operand is the| | | |module name, and the | @@ -162,8 +173,9 @@ Other operators | | |module. | +--------------------------------+-------------------+------------------------+ | Conditional | *a* ? *b* : *c* |The first operand must | -| | |evaluate to a "bool" | -| | |type. If true, then the| +| | |evaluate to type | +| | |:bro:type:`bool`. | +| | |If true, then the | | | |second expression is | | | |evaluated and is the | | | |result of the entire | From 446578ea9737570975214045d52bc645017f1669 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 7 Oct 2014 15:13:35 -0700 Subject: [PATCH 36/71] Updating plugin documentation. Extending debugging section a bit, and claryyhing why some content is missing. Also linking into new development section at top-level. --- CHANGES | 4 +++ VERSION | 2 +- doc/devel/plugins.rst | 65 ++++++++++++++++++++++++------------------- doc/index.rst | 8 +++++- 4 files changed, 49 insertions(+), 30 deletions(-) diff --git a/CHANGES b/CHANGES index 3eada23d18..81f85e53bd 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-225 | 2014-10-07 15:13:35 -0700 + + * Updating plugin documentation. (Robin Sommer) + 2.3-224 | 2014-10-07 14:32:17 -0700 * Improved the log file reference documentation. (Jeannette Dopheide diff --git a/VERSION b/VERSION index 8a8cf53724..fe794e513d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-224 +2.3-225 diff --git a/doc/devel/plugins.rst b/doc/devel/plugins.rst index a66023cece..c703345891 100644 --- a/doc/devel/plugins.rst +++ b/doc/devel/plugins.rst @@ -265,23 +265,25 @@ plugins to unconditionally activate, even in bare mode. activated plugins. Note that plugins compiled statically into Bro are always activated, and hence show up as such even in bare mode. -Plugin Component -================ +Plugin Components +================= -The following gives additional information about providing individual -types of functionality via plugins. Note that a single plugin can -provide more than one type. For example, a plugin could provide -multiple protocol analyzers at once; or both a logging backend and -input reader at the same time. +The following subsections detail providing individual types of +functionality via plugins. Note that a single plugin can provide more +than one component type. For example, a plugin could provide multiple +protocol analyzers at once; or both a logging backend and input reader +at the same time. -We now walk briefly through the specifics of providing a specific type -of functionality (a *component*) through a plugin. We'll focus on -their interfaces to the plugin system, rather than specifics on -writing the corresponding logic (usually the best way to get going on -that is to start with an existing plugin providing a corresponding -component and adapt that). We'll also point out how the CMake -infrastructure put in place by the ``init-plugin`` helper script ties -the various pieces together. +.. todo:: + + These subsections are mostly missing right now, as much of their + content isn't actually plugin-specific, but concerns generally + writing such functionality for Bro. The best way to get started + right now is to look at existing code implementing similar + functionality, either as a plugin or inside Bro proper. Also, for + each component type there's a unit test in + ``testing/btest/plugins`` creating a basic plugin skeleton with a + corresponding component. Bro Scripts ----------- @@ -412,25 +414,32 @@ Run the test-suite:: Debugging Plugins ================= -Plugins can use Bro's standard debug logger by using the -``PLUGIN_DBG_LOG(, )`` macro (defined in -``DebugLogger.h``), where ```` is the ``Plugin`` instance and -```` are printf-style arguments, just as with Bro's standard -debuggging macros. +If your plugin isn't loading as expected, Bro's debugging facilities +can help to illuminate what's going on. To enable, recompile Bro +with debugging support (``./configure --enable-debug``), and +afterwards rebuild your plugin as well. If you then run Bro with ``-B +plugins``, it will produce a file ``debug.log`` that records details +about the process for searching, loading, and activating plugins. -At runtime, one then activates a plugin's debugging output with ``-B -plugin-``, where ```` is the name of the plugin as -returned by its ``Configure()`` method, yet with the -namespace-separator ``::`` replaced with a simple dash. Example: If -the plugin is called ``Bro::Demo``, use ``-B plugin-Bro-Demo``. As -usual, the debugging output will be recorded to ``debug.log`` if Bro's -compiled in debug mode. +To generate your own debugging output from inside your plugin, you can +add a custom debug stream by using the ``PLUGIN_DBG_LOG(, +)`` macro (defined in ``DebugLogger.h``), where ```` is +the ``Plugin`` instance and ```` are printf-style arguments, +just as with Bro's standard debugging macros (grep for ``DBG_LOG`` in +Bro's ``src/`` to see examples). At runtime, you can then activate +your plugin's debugging output with ``-B plugin-``, where +```` is the name of the plugin as returned by its +``Configure()`` method, yet with the namespace-separator ``::`` +replaced with a simple dash. Example: If the plugin is called +``Bro::Demo``, use ``-B plugin-Bro-Demo``. As usual, the debugging +output will be recorded to ``debug.log`` if Bro's compiled in debug +mode. Documenting Plugins =================== -..todo:: +.. todo:: Integrate all this with Broxygen. diff --git a/doc/index.rst b/doc/index.rst index 6161ee1ff8..22fb8cbe1a 100644 --- a/doc/index.rst +++ b/doc/index.rst @@ -45,7 +45,13 @@ Reference Section script-reference/index.rst components/index.rst -.. +Development +=========== + +.. toctree:: + :maxdepth: 2 + + devel/plugins.rst * :ref:`General Index ` * :ref:`search` From 56a2a1a1e54bdd503f13db77b73923feedce7c1d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 7 Oct 2014 15:26:30 -0700 Subject: [PATCH 37/71] Fix for allowing a packet source plugin to provide multiple prefixes with a colon. --- src/iosource/Component.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/iosource/Component.cc b/src/iosource/Component.cc index 1e4f029400..a285cd8552 100644 --- a/src/iosource/Component.cc +++ b/src/iosource/Component.cc @@ -24,7 +24,7 @@ Component::~Component() PktSrcComponent::PktSrcComponent(const std::string& arg_name, const std::string& arg_prefix, InputType arg_type, factory_callback arg_factory) : iosource::Component(plugin::component::PKTSRC, arg_name) { - tokenize_string(arg_prefix, "%", &prefixes); + tokenize_string(arg_prefix, ":", &prefixes); type = arg_type; factory = arg_factory; } @@ -110,7 +110,7 @@ void PktSrcComponent::DoDescribe(ODesc* d) const PktDumperComponent::PktDumperComponent(const std::string& name, const std::string& arg_prefix, factory_callback arg_factory) : plugin::Component(plugin::component::PKTDUMPER, name) { - tokenize_string(arg_prefix, "%", &prefixes); + tokenize_string(arg_prefix, ":", &prefixes); factory = arg_factory; } From 38beb6632e21718048a67230a90a86e0d5aa1955 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 7 Oct 2014 15:27:16 -0700 Subject: [PATCH 38/71] Switching the prefix separator for packet source/dumper plugins once more, now to "::". Addresses BIT-1267. --- src/iosource/Manager.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/iosource/Manager.cc b/src/iosource/Manager.cc index 5a087bc4ef..f71807dcbe 100644 --- a/src/iosource/Manager.cc +++ b/src/iosource/Manager.cc @@ -204,11 +204,11 @@ static std::pair split_prefix(std::string path) // PktSrc to use. If not, choose default. std::string prefix; - std::string::size_type i = path.find("%"); + std::string::size_type i = path.find("::"); if ( i != std::string::npos ) { prefix = path.substr(0, i); - path = path.substr(++i, std::string::npos); + path = path.substr(i + 2, std::string::npos); } else From 91c218d44a48e5c1260b9421895d78085b5656a4 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 7 Oct 2014 15:28:15 -0700 Subject: [PATCH 39/71] Include plugin unit tests into the top-level btest configuration. Turns out they weren't part of it yet. Comes with some baseline updates. --- CHANGES | 10 ++ VERSION | 2 +- testing/btest/Baseline/plugins.hooks/output | 132 +++++++++--------- .../btest/Baseline/plugins.pktdumper/output | 2 +- testing/btest/Baseline/plugins.writer/output | 4 +- testing/btest/btest.cfg | 2 +- testing/btest/plugins/pktdumper.bro | 2 +- testing/btest/plugins/pktsrc.bro | 2 +- 8 files changed, 83 insertions(+), 73 deletions(-) diff --git a/CHANGES b/CHANGES index 81f85e53bd..e49f37975c 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,14 @@ +2.3-228 | 2014-10-07 15:32:37 -0700 + + * Include plugin unit tests into the top-level btest configuration. (Robin Sommer) + + * Switching the prefix separator for packet source/dumper plugins + once more, now to "::". Addresses BIT-1267. (Robin Sommer) + + * Fix for allowing a packet source/dumper plugin to support multiple + prefixes with a colon. (Robin Sommer) + 2.3-225 | 2014-10-07 15:13:35 -0700 * Updating plugin documentation. (Robin Sommer) diff --git a/VERSION b/VERSION index fe794e513d..ea0e505073 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-225 +2.3-228 diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 83341f3075..5deb40ca77 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -182,7 +182,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> -0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Conn::LOG)) -> @@ -273,8 +273,8 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> -0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> -0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> @@ -705,7 +705,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Conn::LOG)) @@ -796,8 +796,8 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, (Unified2::LOG, [columns=, ev=Unified2::log_unified2])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) -0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) @@ -1228,7 +1228,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Unified2::LOG, [columns=, ev=Unified2::log_unified2]) 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1319,8 +1319,8 @@ 0.000000 | HookCallFunction Log::create_stream(Unified2::LOG, [columns=, ev=Unified2::log_unified2]) 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509]) -0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1409853900.737227, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1412721129.083128, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) @@ -1532,10 +1532,20 @@ 1362692527.008509 MetaHookPre UpdateNetworkTime(1362692527.008509) 1362692527.008509 | HookUpdateNetworkTime 1362692527.008509 1362692527.008509 | HookDrainEvents +1362692527.009512 MetaHookPost CallFunction(Files::__add_analyzers_for_mime_type, (FakNcS1Jfe01uljb3, text/plain, [chunk_event=, stream_event=, extract_filename=, extract_limit=0])) -> +1362692527.009512 MetaHookPost CallFunction(Files::add_analyzers_for_mime_type, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) -> +1362692527.009512 MetaHookPost CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=])) -> +1362692527.009512 MetaHookPost CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -> 1362692527.009512 MetaHookPost CallFunction(HTTP::code_in_range, (200, 100, 199)) -> +1362692527.009512 MetaHookPost CallFunction(HTTP::get_file_handle, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> 1362692527.009512 MetaHookPost CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) -> 1362692527.009512 MetaHookPost CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) -> 1362692527.009512 MetaHookPost CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) -> +1362692527.009512 MetaHookPost CallFunction(cat, (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> +1362692527.009512 MetaHookPost CallFunction(file_new, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -> +1362692527.009512 MetaHookPost CallFunction(file_over_new_connection, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> +1362692527.009512 MetaHookPost CallFunction(fmt, (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> +1362692527.009512 MetaHookPost CallFunction(get_file_handle, (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> 1362692527.009512 MetaHookPost CallFunction(http_begin_entity, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> 1362692527.009512 MetaHookPost CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, ACCEPT-RANGES, bytes)) -> 1362692527.009512 MetaHookPost CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, CONNECTION, Keep-Alive)) -> @@ -1547,7 +1557,13 @@ 1362692527.009512 MetaHookPost CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/2.4.3 (Fedora))) -> 1362692527.009512 MetaHookPost CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain; charset=UTF-8)) -> 1362692527.009512 MetaHookPost CallFunction(http_reply, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], 1.1, 200, OK)) -> +1362692527.009512 MetaHookPost CallFunction(id_string, ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> +1362692527.009512 MetaHookPost CallFunction(set_file_handle, (Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80)) -> +1362692527.009512 MetaHookPost CallFunction(split_all, (HTTP, <...>/)) -> 1362692527.009512 MetaHookPost DrainEvents() -> +1362692527.009512 MetaHookPost QueueEvent(file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -> false +1362692527.009512 MetaHookPost QueueEvent(file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false +1362692527.009512 MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false 1362692527.009512 MetaHookPost QueueEvent(http_begin_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false 1362692527.009512 MetaHookPost QueueEvent(http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, ACCEPT-RANGES, bytes)) -> false 1362692527.009512 MetaHookPost QueueEvent(http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, CONNECTION, Keep-Alive)) -> false @@ -1560,10 +1576,20 @@ 1362692527.009512 MetaHookPost QueueEvent(http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain; charset=UTF-8)) -> false 1362692527.009512 MetaHookPost QueueEvent(http_reply([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], 1.1, 200, OK)) -> false 1362692527.009512 MetaHookPost UpdateNetworkTime(1362692527.009512) -> +1362692527.009512 MetaHookPre CallFunction(Files::__add_analyzers_for_mime_type, (FakNcS1Jfe01uljb3, text/plain, [chunk_event=, stream_event=, extract_filename=, extract_limit=0])) +1362692527.009512 MetaHookPre CallFunction(Files::add_analyzers_for_mime_type, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) +1362692527.009512 MetaHookPre CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=])) +1362692527.009512 MetaHookPre CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) 1362692527.009512 MetaHookPre CallFunction(HTTP::code_in_range, (200, 100, 199)) +1362692527.009512 MetaHookPre CallFunction(HTTP::get_file_handle, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009512 MetaHookPre CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) 1362692527.009512 MetaHookPre CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) 1362692527.009512 MetaHookPre CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) +1362692527.009512 MetaHookPre CallFunction(cat, (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) +1362692527.009512 MetaHookPre CallFunction(file_new, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) +1362692527.009512 MetaHookPre CallFunction(file_over_new_connection, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) +1362692527.009512 MetaHookPre CallFunction(fmt, (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) +1362692527.009512 MetaHookPre CallFunction(get_file_handle, (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009512 MetaHookPre CallFunction(http_begin_entity, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009512 MetaHookPre CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, ACCEPT-RANGES, bytes)) 1362692527.009512 MetaHookPre CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, CONNECTION, Keep-Alive)) @@ -1575,7 +1601,13 @@ 1362692527.009512 MetaHookPre CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/2.4.3 (Fedora))) 1362692527.009512 MetaHookPre CallFunction(http_header, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain; charset=UTF-8)) 1362692527.009512 MetaHookPre CallFunction(http_reply, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], 1.1, 200, OK)) +1362692527.009512 MetaHookPre CallFunction(id_string, ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) +1362692527.009512 MetaHookPre CallFunction(set_file_handle, (Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80)) +1362692527.009512 MetaHookPre CallFunction(split_all, (HTTP, <...>/)) 1362692527.009512 MetaHookPre DrainEvents() +1362692527.009512 MetaHookPre QueueEvent(file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) +1362692527.009512 MetaHookPre QueueEvent(file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) +1362692527.009512 MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009512 MetaHookPre QueueEvent(http_begin_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009512 MetaHookPre QueueEvent(http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, ACCEPT-RANGES, bytes)) 1362692527.009512 MetaHookPre QueueEvent(http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, CONNECTION, Keep-Alive)) @@ -1589,10 +1621,20 @@ 1362692527.009512 MetaHookPre QueueEvent(http_reply([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], 1.1, 200, OK)) 1362692527.009512 MetaHookPre UpdateNetworkTime(1362692527.009512) 1362692527.009512 | HookUpdateNetworkTime 1362692527.009512 +1362692527.009512 | HookCallFunction Files::__add_analyzers_for_mime_type(FakNcS1Jfe01uljb3, text/plain, [chunk_event=, stream_event=, extract_filename=, extract_limit=0]) +1362692527.009512 | HookCallFunction Files::add_analyzers_for_mime_type([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain) +1362692527.009512 | HookCallFunction Files::set_info([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=]) +1362692527.009512 | HookCallFunction Files::set_info([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=]) 1362692527.009512 | HookCallFunction HTTP::code_in_range(200, 100, 199) +1362692527.009512 | HookCallFunction HTTP::get_file_handle([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009512 | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F) 1362692527.009512 | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F) 1362692527.009512 | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F) +1362692527.009512 | HookCallFunction cat(Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) +1362692527.009512 | HookCallFunction file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=]) +1362692527.009512 | HookCallFunction file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) +1362692527.009512 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp) +1362692527.009512 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009512 | HookCallFunction http_begin_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009512 | HookCallFunction http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, ACCEPT-RANGES, bytes) 1362692527.009512 | HookCallFunction http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, CONNECTION, Keep-Alive) @@ -1604,7 +1646,13 @@ 1362692527.009512 | HookCallFunction http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/2.4.3 (Fedora)) 1362692527.009512 | HookCallFunction http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain; charset=UTF-8) 1362692527.009512 | HookCallFunction http_reply([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], 1.1, 200, OK) +1362692527.009512 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp]) +1362692527.009512 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80) +1362692527.009512 | HookCallFunction split_all(HTTP, <...>/) 1362692527.009512 | HookDrainEvents +1362692527.009512 | HookQueueEvent file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=]) +1362692527.009512 | HookQueueEvent file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) +1362692527.009512 | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009512 | HookQueueEvent http_begin_entity([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009512 | HookQueueEvent http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, ACCEPT-RANGES, bytes) 1362692527.009512 | HookQueueEvent http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, CONNECTION, Keep-Alive) @@ -1616,60 +1664,12 @@ 1362692527.009512 | HookQueueEvent http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/2.4.3 (Fedora)) 1362692527.009512 | HookQueueEvent http_header([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain; charset=UTF-8) 1362692527.009512 | HookQueueEvent http_reply([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=, status_msg=, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=, orig_mime_depth=1, resp_mime_depth=0]}, current_request=1, current_response=0], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], 1.1, 200, OK) -1362692527.009721 MetaHookPost CallFunction(Files::__add_analyzers_for_mime_type, (FakNcS1Jfe01uljb3, text/plain, [chunk_event=, stream_event=, extract_filename=, extract_limit=0])) -> -1362692527.009721 MetaHookPost CallFunction(Files::add_analyzers_for_mime_type, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) -> -1362692527.009721 MetaHookPost CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=])) -> -1362692527.009721 MetaHookPost CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -> -1362692527.009721 MetaHookPost CallFunction(HTTP::get_file_handle, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> -1362692527.009721 MetaHookPost CallFunction(cat, (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> -1362692527.009721 MetaHookPost CallFunction(file_new, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -> -1362692527.009721 MetaHookPost CallFunction(file_over_new_connection, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> -1362692527.009721 MetaHookPost CallFunction(fmt, (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -> -1362692527.009721 MetaHookPost CallFunction(get_file_handle, (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> -1362692527.009721 MetaHookPost CallFunction(id_string, ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -> -1362692527.009721 MetaHookPost CallFunction(set_file_handle, (Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80)) -> -1362692527.009721 MetaHookPost CallFunction(split_all, (HTTP, <...>/)) -> 1362692527.009721 MetaHookPost DrainEvents() -> -1362692527.009721 MetaHookPost QueueEvent(file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -> false -1362692527.009721 MetaHookPost QueueEvent(file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false -1362692527.009721 MetaHookPost QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> false 1362692527.009721 MetaHookPost UpdateNetworkTime(1362692527.009721) -> -1362692527.009721 MetaHookPre CallFunction(Files::__add_analyzers_for_mime_type, (FakNcS1Jfe01uljb3, text/plain, [chunk_event=, stream_event=, extract_filename=, extract_limit=0])) -1362692527.009721 MetaHookPre CallFunction(Files::add_analyzers_for_mime_type, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain)) -1362692527.009721 MetaHookPre CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=])) -1362692527.009721 MetaHookPre CallFunction(Files::set_info, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -1362692527.009721 MetaHookPre CallFunction(HTTP::get_file_handle, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -1362692527.009721 MetaHookPre CallFunction(cat, (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -1362692527.009721 MetaHookPre CallFunction(file_new, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -1362692527.009721 MetaHookPre CallFunction(file_over_new_connection, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -1362692527.009721 MetaHookPre CallFunction(fmt, (%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp)) -1362692527.009721 MetaHookPre CallFunction(get_file_handle, (Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -1362692527.009721 MetaHookPre CallFunction(id_string, ([orig_h=141.142.228.5, orig_p=59856<...>/tcp])) -1362692527.009721 MetaHookPre CallFunction(set_file_handle, (Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80)) -1362692527.009721 MetaHookPre CallFunction(split_all, (HTTP, <...>/)) 1362692527.009721 MetaHookPre DrainEvents() -1362692527.009721 MetaHookPre QueueEvent(file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=])) -1362692527.009721 MetaHookPre QueueEvent(file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -1362692527.009721 MetaHookPre QueueEvent(get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009721 MetaHookPre UpdateNetworkTime(1362692527.009721) 1362692527.009721 | HookUpdateNetworkTime 1362692527.009721 -1362692527.009721 | HookCallFunction Files::__add_analyzers_for_mime_type(FakNcS1Jfe01uljb3, text/plain, [chunk_event=, stream_event=, extract_filename=, extract_limit=0]) -1362692527.009721 | HookCallFunction Files::add_analyzers_for_mime_type([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain) -1362692527.009721 | HookCallFunction Files::set_info([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=]) -1362692527.009721 | HookCallFunction Files::set_info([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=]) -1362692527.009721 | HookCallFunction HTTP::get_file_handle([id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) -1362692527.009721 | HookCallFunction cat(Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) -1362692527.009721 | HookCallFunction file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=]) -1362692527.009721 | HookCallFunction file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) -1362692527.009721 | HookCallFunction fmt(%s:%d > %s:%d, 141.142.228.5, 59856<...>/tcp) -1362692527.009721 | HookCallFunction get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) -1362692527.009721 | HookCallFunction id_string([orig_h=141.142.228.5, orig_p=59856<...>/tcp]) -1362692527.009721 | HookCallFunction set_file_handle(Analyzer::ANALYZER_HTTP1362692526.869344F11141.142.228.5:59856 > 192.150.187.43:80) -1362692527.009721 | HookCallFunction split_all(HTTP, <...>/) 1362692527.009721 | HookDrainEvents -1362692527.009721 | HookQueueEvent file_new([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain]], info=, u2_events=]) -1362692527.009721 | HookQueueEvent file_over_new_connection([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) -1362692527.009721 | HookQueueEvent get_file_handle(Analyzer::ANALYZER_HTTP, [id=[orig_h=141.142.228.5, orig_p=59856<...>/1.14 (darwin12.2.0), request_body_len=0, response_body_len=0, status_code=200, status_msg=OK, info_code=, info_msg=, filename=, tags={}, username=, password=, capture_password=F, proxied=, range_request=F, orig_fuids=, orig_mime_types=, resp_fuids=, resp_mime_types=, current_entity=[filename=], orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009765 MetaHookPost DrainEvents() -> 1362692527.009765 MetaHookPost UpdateNetworkTime(1362692527.009765) -> 1362692527.009765 MetaHookPre DrainEvents() @@ -1680,11 +1680,11 @@ 1362692527.009775 MetaHookPost CallFunction(HTTP::code_in_range, (200, 100, 199)) -> 1362692527.009775 MetaHookPost CallFunction(HTTP::get_file_handle, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) -> 1362692527.009775 MetaHookPost CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) -> -1362692527.009775 MetaHookPost CallFunction(Log::__write, (Files::LOG, [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> +1362692527.009775 MetaHookPost CallFunction(Log::__write, (Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> 1362692527.009775 MetaHookPost CallFunction(Log::__write, (HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -> -1362692527.009775 MetaHookPost CallFunction(Log::default_path_func, (Files::LOG, , [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> +1362692527.009775 MetaHookPost CallFunction(Log::default_path_func, (Files::LOG, , [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> 1362692527.009775 MetaHookPost CallFunction(Log::default_path_func, (HTTP::LOG, , [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -> -1362692527.009775 MetaHookPost CallFunction(Log::write, (Files::LOG, [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> +1362692527.009775 MetaHookPost CallFunction(Log::write, (Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) -> 1362692527.009775 MetaHookPost CallFunction(Log::write, (HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -> 1362692527.009775 MetaHookPost CallFunction(cat, (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) -> 1362692527.009775 MetaHookPost CallFunction(file_state_remove, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=])) -> @@ -1712,11 +1712,11 @@ 1362692527.009775 MetaHookPre CallFunction(HTTP::code_in_range, (200, 100, 199)) 1362692527.009775 MetaHookPre CallFunction(HTTP::get_file_handle, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F)) 1362692527.009775 MetaHookPre CallFunction(HTTP::set_state, ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F)) -1362692527.009775 MetaHookPre CallFunction(Log::__write, (Files::LOG, [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) +1362692527.009775 MetaHookPre CallFunction(Log::__write, (Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) 1362692527.009775 MetaHookPre CallFunction(Log::__write, (HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -1362692527.009775 MetaHookPre CallFunction(Log::default_path_func, (Files::LOG, , [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) +1362692527.009775 MetaHookPre CallFunction(Log::default_path_func, (Files::LOG, , [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) 1362692527.009775 MetaHookPre CallFunction(Log::default_path_func, (HTTP::LOG, , [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) -1362692527.009775 MetaHookPre CallFunction(Log::write, (Files::LOG, [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) +1362692527.009775 MetaHookPre CallFunction(Log::write, (Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=])) 1362692527.009775 MetaHookPre CallFunction(Log::write, (HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1])) 1362692527.009775 MetaHookPre CallFunction(cat, (Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80)) 1362692527.009775 MetaHookPre CallFunction(file_state_remove, ([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=])) @@ -1745,11 +1745,11 @@ 1362692527.009775 | HookCallFunction HTTP::code_in_range(200, 100, 199) 1362692527.009775 | HookCallFunction HTTP::get_file_handle([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F) 1362692527.009775 | HookCallFunction HTTP::set_state([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]}, current_request=1, current_response=1], irc=, modbus=, radius=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], F, F) -1362692527.009775 | HookCallFunction Log::__write(Files::LOG, [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) +1362692527.009775 | HookCallFunction Log::__write(Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) 1362692527.009775 | HookCallFunction Log::__write(HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]) -1362692527.009775 | HookCallFunction Log::default_path_func(Files::LOG, , [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) +1362692527.009775 | HookCallFunction Log::default_path_func(Files::LOG, , [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) 1362692527.009775 | HookCallFunction Log::default_path_func(HTTP::LOG, , [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]) -1362692527.009775 | HookCallFunction Log::write(Files::LOG, [ts=1362692527.009721, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=53.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) +1362692527.009775 | HookCallFunction Log::write(Files::LOG, [ts=1362692527.009512, fuid=FakNcS1Jfe01uljb3, tx_hosts={192.150.187.43}, rx_hosts={141.142.228.5}, conn_uids={CXWv6p3arKYeMETxOg}, source=HTTP, depth=0, analyzers={}, mime_type=text/plain, filename=, duration=262.0 usecs, local_orig=, is_orig=F, seen_bytes=4705, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=]) 1362692527.009775 | HookCallFunction Log::write(HTTP::LOG, [ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1]) 1362692527.009775 | HookCallFunction cat(Analyzer::ANALYZER_HTTP, 1362692526.869344, F, 1, 1, 141.142.228.5:59856 > 192.150.187.43:80) 1362692527.009775 | HookCallFunction file_state_remove([id=FakNcS1Jfe01uljb3, parent_id=, source=HTTP, is_orig=F, conns={[[orig_h=141.142.228.5, orig_p=59856<...>/plain, filename=, duration=0 secs, local_orig=, is_orig=F, seen_bytes=0, total_bytes=4705, missing_bytes=0, overflow_bytes=0, timedout=F, parent_fuid=, md5=, sha1=, sha256=, x509=, extracted=], u2_events=]) diff --git a/testing/btest/Baseline/plugins.pktdumper/output b/testing/btest/Baseline/plugins.pktdumper/output index 05ffec25cb..42b51e8051 100644 --- a/testing/btest/Baseline/plugins.pktdumper/output +++ b/testing/btest/Baseline/plugins.pktdumper/output @@ -1,5 +1,5 @@ Demo::Foo - A Foo packet dumper (dynamic, version 1.0) - [Packet Dumper] FooPktDumper (dumper prefix: foo) + [Packet Dumper] FooPktDumper (dumper prefix: "foo") === Dumping to XXX: 1373858797.646968 len 94 diff --git a/testing/btest/Baseline/plugins.writer/output b/testing/btest/Baseline/plugins.writer/output index 49c130d6e2..0882718f03 100644 --- a/testing/btest/Baseline/plugins.writer/output +++ b/testing/btest/Baseline/plugins.writer/output @@ -9,7 +9,7 @@ Demo::Foo - A Foo test logging writer (dynamic, version 1.0) [conn] 1340213162.160367|CRJuHdVW0XPVINV8a|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| [conn] 1340213226.561757|CPbrpk1qSsw6ESzHV4|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| [conn] 1340213290.981995|C6pKV8GSxOnSLghOa|10.0.0.55|53994|60.190.189.214|8124|tcp|-|-|-|-|SH|-|0|F|1|52|0|0| -[files] 1340213020.732581|FBtZ7y1ppK8iIeY622|60.190.189.214|10.0.0.55|CjhGID4nQcgTWjvg4c|HTTP|0||image/gif|-|0.000000|-|F|1368|1368|0|0|F|-|-|-|-|- +[files] 1340213020.732547|FBtZ7y1ppK8iIeY622|60.190.189.214|10.0.0.55|CjhGID4nQcgTWjvg4c|HTTP|0||image/gif|-|0.000034|-|F|1368|1368|0|0|F|-|-|-|-|- [http] 1340213019.013158|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|1|GET|www.osnews.com|/images/printer2.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213019.013426|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|2|GET|www.osnews.com|/img2/shorturl.jpg|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213019.580162|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|3|GET|www.osnews.com|/images/icons/9.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- @@ -17,6 +17,6 @@ Demo::Foo - A Foo test logging writer (dynamic, version 1.0) [http] 1340213020.732963|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|GET|www.osnews.com|/images/icons/17.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213021.300269|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|6|GET|www.osnews.com|/images/left.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- [http] 1340213021.861584|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|7|GET|www.osnews.com|/images/icons/32.gif|http://www.osnews.com/|Mozilla/5.0 (Macintosh; Intel Mac OS X 10.7; rv:10.0.2) Gecko/20100101 Firefox/10.0.2|0|0|304|Not Modified|-|-|-||-|-|-|-|-|-|- -[packet_filter] 1409859343.786281|bro|ip or not ip|T|T +[packet_filter] 1412721099.419280|bro|ip or not ip|T|T [socks] 1340213015.276495|CjhGID4nQcgTWjvg4c|10.0.0.55|53994|60.190.189.214|8124|5|-|succeeded|-|www.osnews.com|80|192.168.0.31|-|2688 [tunnel] 1340213015.276495|-|10.0.0.55|0|60.190.189.214|8124|Tunnel::SOCKS|Tunnel::DISCOVER diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 430f4ba47c..43f29d40a1 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,5 +1,5 @@ [btest] -TestDirs = doc bifs language core scripts istate coverage signatures +TestDirs = doc bifs language core scripts istate coverage signatures plugins TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp diff --git a/testing/btest/plugins/pktdumper.bro b/testing/btest/plugins/pktdumper.bro index 64eda98cdd..29b69acadd 100644 --- a/testing/btest/plugins/pktdumper.bro +++ b/testing/btest/plugins/pktdumper.bro @@ -3,6 +3,6 @@ # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace -w foo/XXX %INPUT FilteredTraceDetection::enable=F >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r $TRACES/port4242.trace -w foo::XXX %INPUT FilteredTraceDetection::enable=F >>output # @TEST-EXEC: btest-diff output diff --git a/testing/btest/plugins/pktsrc.bro b/testing/btest/plugins/pktsrc.bro index 87f23d9f75..349e361664 100644 --- a/testing/btest/plugins/pktsrc.bro +++ b/testing/btest/plugins/pktsrc.bro @@ -3,6 +3,6 @@ # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output -# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo/XXX %INPUT FilteredTraceDetection::enable=F >>output +# @TEST-EXEC: BRO_PLUGIN_PATH=`pwd` bro -r foo::XXX %INPUT FilteredTraceDetection::enable=F >>output # @TEST-EXEC: btest-diff conn.log From 81933d25a85c413340947a598809f3ddfa0432cf Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Tue, 7 Oct 2014 20:18:11 -0700 Subject: [PATCH 40/71] Fix for test portability. --- CHANGES | 4 ++++ VERSION | 2 +- testing/btest/plugins/api-version-mismatch.sh | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e49f37975c..528c244503 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-229 | 2014-10-07 20:18:11 -0700 + + * Fix for test portability. (Robin Sommer) + 2.3-228 | 2014-10-07 15:32:37 -0700 * Include plugin unit tests into the top-level btest configuration. (Robin Sommer) diff --git a/VERSION b/VERSION index ea0e505073..67bae97864 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-228 +2.3-229 diff --git a/testing/btest/plugins/api-version-mismatch.sh b/testing/btest/plugins/api-version-mismatch.sh index f8d88b4fc4..c584e2c61e 100644 --- a/testing/btest/plugins/api-version-mismatch.sh +++ b/testing/btest/plugins/api-version-mismatch.sh @@ -2,6 +2,6 @@ # @TEST-EXEC: bash %INPUT # @TEST-EXEC: ./configure --bro-dist=${DIST} && make # @TEST-EXEC-FAIL: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output 2>&1 -# @TEST-EXEC: btest-diff output +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output ( echo '#define BRO_PLUGIN_API_VERSION 42'; cat src/Plugin.cc; ) >src/Plugin.cc.tmp && mv src/Plugin.cc.tmp src/Plugin.cc From f4f5cfd3216e58132d6be75adba1d6c398fb3e9c Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Wed, 8 Oct 2014 08:15:17 -0700 Subject: [PATCH 41/71] Further baseline normalization for plugin test portability. --- CHANGES | 5 +++++ VERSION | 2 +- testing/btest/Baseline/plugins.api-version-mismatch/output | 2 +- testing/btest/plugins/api-version-mismatch.sh | 3 ++- 4 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGES b/CHANGES index 528c244503..6fa563f468 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-230 | 2014-10-08 08:15:17 -0700 + + * Further baseline normalization for plugin test portability. (Robin + Sommer) + 2.3-229 | 2014-10-07 20:18:11 -0700 * Fix for test portability. (Robin Sommer) diff --git a/VERSION b/VERSION index 67bae97864..076001d654 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-229 +2.3-230 diff --git a/testing/btest/Baseline/plugins.api-version-mismatch/output b/testing/btest/Baseline/plugins.api-version-mismatch/output index 806623cd02..1e4dae5e65 100644 --- a/testing/btest/Baseline/plugins.api-version-mismatch/output +++ b/testing/btest/Baseline/plugins.api-version-mismatch/output @@ -1 +1 @@ -fatal error in /home/robin/bro/master/scripts/base/init-bare.bro, line 1: plugin's API version does not match Bro (expected 2, got 42 in /home/robin/bro/master/testing/btest/.tmp/plugins.api-version-mismatch//lib/Demo-Foo.linux-x86_64.so) +fatal error in /home/robin/bro/master/scripts/base/init-bare.bro, line 1: plugin's API version does not match Bro (expected 2, got 42 in /home/robin/bro/master/testing/btest/.tmp/plugins.api-version-mismatch//lib/XXX) diff --git a/testing/btest/plugins/api-version-mismatch.sh b/testing/btest/plugins/api-version-mismatch.sh index c584e2c61e..cfb4269946 100644 --- a/testing/btest/plugins/api-version-mismatch.sh +++ b/testing/btest/plugins/api-version-mismatch.sh @@ -1,7 +1,8 @@ # @TEST-EXEC: ${DIST}/aux/bro-aux/plugin-support/init-plugin Demo Foo # @TEST-EXEC: bash %INPUT # @TEST-EXEC: ./configure --bro-dist=${DIST} && make -# @TEST-EXEC-FAIL: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >>output 2>&1 +# @TEST-EXEC-FAIL: BRO_PLUGIN_PATH=`pwd` bro -NN Demo::Foo >tmp 2>&1 +# @TEST-EXEC: cat tmp | sed 's/Demo-Foo[-a-zA-Z0-9_.]*/XXX/' >>output # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff output ( echo '#define BRO_PLUGIN_API_VERSION 42'; cat src/Plugin.cc; ) >src/Plugin.cc.tmp && mv src/Plugin.cc.tmp src/Plugin.cc From 072dad6508c1ff984cbe4fe8ba4e23c3a1fdb640 Mon Sep 17 00:00:00 2001 From: Daniel Thayer Date: Wed, 8 Oct 2014 10:42:35 -0500 Subject: [PATCH 42/71] Add error checks and messages to a test script --- testing/btest/coverage/find-bro-logs.test | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/testing/btest/coverage/find-bro-logs.test b/testing/btest/coverage/find-bro-logs.test index ffde5b8225..b4ab3aad2d 100644 --- a/testing/btest/coverage/find-bro-logs.test +++ b/testing/btest/coverage/find-bro-logs.test @@ -9,6 +9,11 @@ BROSCRIPTS=${DIST}/scripts +if [ ! -d "${BROSCRIPTS}" ]; then + echo "Directory not found: ${BROSCRIPTS}" 1>&2 + exit 1 +fi + # For a given Bro script, look for a call to "create_stream". If found, # extract the log ID (adding the module name if necessary), and print the # log ID and script filename. @@ -35,6 +40,11 @@ _EOF_ find ${BROSCRIPTS} -type f -exec awk -f find_logid.awk {} \; > out.logid +if [ ! -s out.logid ]; then + echo "Did not find Bro scripts in directory: ${BROSCRIPTS}" 1>&2 + exit 1 +fi + # For each log ID, have Bro convert it to the corresponding log filename # using the default mechanism for generating a log filename (we must load # all Bro scripts so that all log IDs are defined). From 7ef1409b40298006042e6a86fe6027a073a4ff6f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 9 Oct 2014 16:00:27 -0500 Subject: [PATCH 43/71] Change find-bro-logs unit test to follow symlinks. --- CHANGES | 6 ++++++ VERSION | 2 +- testing/btest/coverage/find-bro-logs.test | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 6fa563f468..fcf1dc038b 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,10 @@ +2.3-233 | 2014-10-09 16:00:27 -0500 + + * Change find-bro-logs unit test to follow symlinks. (Jon Siwek) + + * Add error checks and messages to a test script (Daniel Thayer) + 2.3-230 | 2014-10-08 08:15:17 -0700 * Further baseline normalization for plugin test portability. (Robin diff --git a/VERSION b/VERSION index 076001d654..03eeeb0bd9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-230 +2.3-233 diff --git a/testing/btest/coverage/find-bro-logs.test b/testing/btest/coverage/find-bro-logs.test index b4ab3aad2d..ce63078698 100644 --- a/testing/btest/coverage/find-bro-logs.test +++ b/testing/btest/coverage/find-bro-logs.test @@ -38,7 +38,7 @@ cat << '_EOF_' > find_logid.awk } _EOF_ -find ${BROSCRIPTS} -type f -exec awk -f find_logid.awk {} \; > out.logid +find -L ${BROSCRIPTS} -type f -exec awk -f find_logid.awk {} \; > out.logid if [ ! -s out.logid ]; then echo "Did not find Bro scripts in directory: ${BROSCRIPTS}" 1>&2 From 191e5da74d8f07aa266678e77de308f1d59dd448 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 14 Oct 2014 14:42:09 -0500 Subject: [PATCH 44/71] Documentation fixes. BIT-1272 #close --- CHANGES | 4 ++++ VERSION | 2 +- doc/scripting/data_struct_vector_declaration.bro | 2 +- doc/scripting/data_type_pattern_01.bro | 2 +- .../btest-doc.sphinx.data_struct_vector_declaration#1 | 2 +- .../btest-doc.sphinx.data_type_pattern#1 | 2 +- 6 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGES b/CHANGES index fcf1dc038b..e3f132b3d9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-234 | 2014-10-14 14:42:09 -0500 + + * Documentation fixes. (Steve Smoot) + 2.3-233 | 2014-10-09 16:00:27 -0500 * Change find-bro-logs unit test to follow symlinks. (Jon Siwek) diff --git a/VERSION b/VERSION index 03eeeb0bd9..5433a332a3 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-233 +2.3-234 diff --git a/doc/scripting/data_struct_vector_declaration.bro b/doc/scripting/data_struct_vector_declaration.bro index d64754b97b..6d684d09b1 100644 --- a/doc/scripting/data_struct_vector_declaration.bro +++ b/doc/scripting/data_struct_vector_declaration.bro @@ -10,6 +10,6 @@ event bro_init() print fmt("contents of v1: %s", v1); print fmt("length of v1: %d", |v1|); - print fmt("contents of v1: %s", v2); + print fmt("contents of v2: %s", v2); print fmt("length of v2: %d", |v2|); } diff --git a/doc/scripting/data_type_pattern_01.bro b/doc/scripting/data_type_pattern_01.bro index 08378dd124..e57650a589 100644 --- a/doc/scripting/data_type_pattern_01.bro +++ b/doc/scripting/data_type_pattern_01.bro @@ -1,6 +1,6 @@ event bro_init() { - local test_string = "The quick brown fox jumped over the lazy dog."; + local test_string = "The quick brown fox jumps over the lazy dog."; local test_pattern = /quick|lazy/; if ( test_pattern in test_string ) diff --git a/testing/btest/Baseline/doc.sphinx.data_struct_vector_declaration/btest-doc.sphinx.data_struct_vector_declaration#1 b/testing/btest/Baseline/doc.sphinx.data_struct_vector_declaration/btest-doc.sphinx.data_struct_vector_declaration#1 index d6e63d72ba..e8bb16ee00 100644 --- a/testing/btest/Baseline/doc.sphinx.data_struct_vector_declaration/btest-doc.sphinx.data_struct_vector_declaration#1 +++ b/testing/btest/Baseline/doc.sphinx.data_struct_vector_declaration/btest-doc.sphinx.data_struct_vector_declaration#1 @@ -7,6 +7,6 @@ # bro data_struct_vector_declaration.bro contents of v1: [1, 2, 3, 4] length of v1: 4 - contents of v1: [1, 2, 3, 4] + contents of v2: [1, 2, 3, 4] length of v2: 4 diff --git a/testing/btest/Baseline/doc.sphinx.data_type_pattern/btest-doc.sphinx.data_type_pattern#1 b/testing/btest/Baseline/doc.sphinx.data_type_pattern/btest-doc.sphinx.data_type_pattern#1 index 99281b205e..a05d4cdabc 100644 --- a/testing/btest/Baseline/doc.sphinx.data_type_pattern/btest-doc.sphinx.data_type_pattern#1 +++ b/testing/btest/Baseline/doc.sphinx.data_type_pattern/btest-doc.sphinx.data_type_pattern#1 @@ -6,6 +6,6 @@ # bro data_type_pattern_01.bro The - brown fox jumped over the + brown fox jumps over the dog. From ccc88beeee0f1bcbf14a8ac27f11fa0c40f775c8 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Wed, 15 Oct 2014 10:20:47 -0500 Subject: [PATCH 45/71] Add error message for bad enum declaration syntax. BIT-1273 #close --- CHANGES | 5 +++++ VERSION | 2 +- src/parse.y | 6 +++++- 3 files changed, 11 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index e3f132b3d9..2756ccbae6 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-235 | 2014-10-15 10:20:47 -0500 + + * BIT-1273: Add error message for bad enum declaration syntax. + (Jon Siwek) + 2.3-234 | 2014-10-14 14:42:09 -0500 * Documentation fixes. (Steve Smoot) diff --git a/VERSION b/VERSION index 5433a332a3..dbd5144240 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-234 +2.3-235 diff --git a/src/parse.y b/src/parse.y index 0289184055..83760dbbf0 100644 --- a/src/parse.y +++ b/src/parse.y @@ -127,7 +127,11 @@ static void parser_new_enum (void) { /* Starting a new enum definition. */ assert(cur_enum_type == NULL); - cur_enum_type = new EnumType(cur_decl_type_id->Name()); + + if ( cur_decl_type_id ) + cur_enum_type = new EnumType(cur_decl_type_id->Name()); + else + reporter->FatalError("incorrect syntax for enum type declaration"); } static void parser_redef_enum (ID *id) From ab62a375ac8a686b78a5fcdfeb54606651adfc08 Mon Sep 17 00:00:00 2001 From: Seth Hall Date: Mon, 20 Oct 2014 10:10:21 -0400 Subject: [PATCH 46/71] Fix some Coverity warnings about the DNP3 analyzer. --- src/analyzer/protocol/dnp3/dnp3-analyzer.pac | 32 ++++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/analyzer/protocol/dnp3/dnp3-analyzer.pac b/src/analyzer/protocol/dnp3/dnp3-analyzer.pac index 2065237f45..dca4a8a5be 100644 --- a/src/analyzer/protocol/dnp3/dnp3-analyzer.pac +++ b/src/analyzer/protocol/dnp3/dnp3-analyzer.pac @@ -5,18 +5,18 @@ connection DNP3_Conn(bro_analyzer: BroAnalyzer) { }; %header{ - uint64 bytestring_to_time(const_bytestring time48, size_t length); + uint64 bytestring_to_time(const_bytestring time48); %} %code{ - uint64 bytestring_to_time(const_bytestring time48, size_t length) + uint64 bytestring_to_time(const_bytestring time48) { /* In DNP3, a timestamp is represented by 6 bytes since epoch in milliseconds. The 6 bytes are stored in big endian format. */ uint64 epochTime = 0; - for ( unsigned int i = 0; i < length; i++ ) - epochTime = time48[length - i - 1] + epochTime * 256; + for ( uint i = time48.length()-1; i == 0; i-- ) + epochTime = time48[i] + epochTime * 256; return epochTime; } @@ -240,7 +240,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_counter_32wFlagTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, count_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, count_value, bytestring_to_time(time48)); } return true; @@ -254,7 +254,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_counter_16wFlagTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, count_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, count_value, bytestring_to_time(time48)); } return true; @@ -408,7 +408,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_32wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -422,7 +422,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_16wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -520,7 +520,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_32wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, value, bytestring_to_time(time48)); } return true; @@ -534,7 +534,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_16wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, value, bytestring_to_time(time48)); } return true; @@ -576,7 +576,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_SPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, value, bytestring_to_time(time48)); } return true; @@ -590,7 +590,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_analog_input_event_DPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, value_low, value_high, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, value_low, value_high, bytestring_to_time(time48)); } return true; @@ -632,7 +632,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_32wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -646,7 +646,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_16wTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -688,7 +688,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_SPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, frozen_value, bytestring_to_time(time48)); } return true; @@ -702,7 +702,7 @@ flow DNP3_Flow(is_orig: bool) { BifEvent::generate_dnp3_frozen_analog_input_event_DPwTime( connection()->bro_analyzer(), connection()->bro_analyzer()->Conn(), - is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_time(time48, sizeof(time48))); + is_orig(), flag, frozen_value_low, frozen_value_high, bytestring_to_time(time48)); } return true; From 4c305d6b92c706d6a14c9c6c34e0766e7162a172 Mon Sep 17 00:00:00 2001 From: Christian Struck Date: Mon, 20 Oct 2014 15:59:58 -0700 Subject: [PATCH 47/71] [FIX] Add files to result table even if the files are empty --- scripts/base/utils/active-http.bro | 4 +++- scripts/base/utils/exec.bro | 10 ++++++++++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/base/utils/active-http.bro b/scripts/base/utils/active-http.bro index 9f62e7bbaa..1bc390d6d6 100644 --- a/scripts/base/utils/active-http.bro +++ b/scripts/base/utils/active-http.bro @@ -65,12 +65,14 @@ function request2curl(r: Request, bodyfile: string, headersfile: string): string cmd = fmt("%s -m %.0f", cmd, r$max_time); if ( r?$client_data ) - cmd = fmt("%s -d -", cmd); + cmd = fmt("%s -d @-", cmd); if ( r?$addl_curl_args ) cmd = fmt("%s %s", cmd, r$addl_curl_args); cmd = fmt("%s \"%s\"", cmd, str_shell_escape(r$url)); + # hack so the bodyfile will exsist even if curl did not write one. + cmd = fmt("%s && touch %s", cmd, str_shell_escape(bodyfile)); return cmd; } diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro index 12f5a0087b..dd992a63cc 100644 --- a/scripts/base/utils/exec.bro +++ b/scripts/base/utils/exec.bro @@ -106,6 +106,16 @@ event Input::end_of_data(name: string, source:string) local track_file = parts[2]; + # If the file is empty, add it to the result$files table + # this is needed because it is expected that the file was read + # even if it was empty + local result = results[name]; + if ( ! result?$files ) + result$files = table(); + + if ( track_file !in result$files ) + result$files[track_file] = vector(source); + Input::remove(name); if ( name !in pending_files ) From 624aa3cac1579722508225f114f34130a33ce7d7 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 21 Oct 2014 11:38:02 -0700 Subject: [PATCH 48/71] Update baseline of new SSL policy script for changes --- .../ssl-digicert.log | 10 +++++----- .../ssl-twimg.log | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log index bb0a25ac0c..fbf18207cd 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-digicert.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-09-04-19-17-18 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string string -1404148886.994021 CXWv6p3arKYeMETxOg 192.168.4.149 51293 72.21.91.29 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 - - - T FhwjYM0FkbvVCvMf2,Fajs2d2lipsadwoK1h (empty) CN=www.digicert.com,O=DigiCert\, Inc.,L=Lehi,ST=Utah,C=US,postalCode=84043,street=2600 West Executive Parkway,street=Suite 500,serialNumber=5299537-0142,1.3.6.1.4.1.311.60.2.1.2=#130455746168,1.3.6.1.4.1.311.60.2.1.3=#13025553,businessCategory=Private Organization CN=DigiCert SHA2 Extended Validation Server CA,OU=www.digicert.com,O=DigiCert Inc,C=US - - good -#close 2014-09-04-19-17-18 +#open 2014-10-21-18-37-44 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1404148886.994021 CXWv6p3arKYeMETxOg 192.168.4.149 51293 72.21.91.29 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 - F - - T FhwjYM0FkbvVCvMf2,Fajs2d2lipsadwoK1h (empty) CN=www.digicert.com,O=DigiCert\, Inc.,L=Lehi,ST=Utah,C=US,postalCode=84043,street=2600 West Executive Parkway,street=Suite 500,serialNumber=5299537-0142,1.3.6.1.4.1.311.60.2.1.2=#130455746168,1.3.6.1.4.1.311.60.2.1.3=#13025553,businessCategory=Private Organization CN=DigiCert SHA2 Extended Validation Server CA,OU=www.digicert.com,O=DigiCert Inc,C=US - - good +#close 2014-10-21-18-37-44 diff --git a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log index 4806744a5c..16c2e35a39 100644 --- a/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log +++ b/testing/btest/Baseline/scripts.policy.protocols.ssl.validate-ocsp/ssl-twimg.log @@ -3,8 +3,8 @@ #empty_field (empty) #unset_field - #path ssl -#open 2014-09-04-19-17-14 -#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name session_id last_alert established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status -#types time string addr port addr port string string string string string string bool vector[string] vector[string] string string string string string -1409786981.016881 CXWv6p3arKYeMETxOg 192.168.4.149 53106 93.184.216.146 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 - - - T FtaZVlJfywdNmVFr1,FoILekwkdtTuZtlVa (empty) CN=si0.twimg.com,O=Twitter\, Inc.,L=San Francisco,ST=California,C=US CN=DigiCert High Assurance CA-3,OU=www.digicert.com,O=DigiCert Inc,C=US - - good -#close 2014-09-04-19-17-14 +#open 2014-10-21-18-37-33 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p version cipher curve server_name resumed last_alert next_protocol established cert_chain_fuids client_cert_chain_fuids subject issuer client_subject client_issuer ocsp_status +#types time string addr port addr port string string string string bool string string bool vector[string] vector[string] string string string string string +1409786981.016881 CXWv6p3arKYeMETxOg 192.168.4.149 53106 93.184.216.146 443 TLSv10 TLS_ECDHE_RSA_WITH_RC4_128_SHA secp256r1 - F - - T FtaZVlJfywdNmVFr1,FoILekwkdtTuZtlVa (empty) CN=si0.twimg.com,O=Twitter\, Inc.,L=San Francisco,ST=California,C=US CN=DigiCert High Assurance CA-3,OU=www.digicert.com,O=DigiCert Inc,C=US - - good +#close 2014-10-21-18-37-33 From 04746c7ffc382cd4a870ed279d78544bc1536b1a Mon Sep 17 00:00:00 2001 From: Christian Struck Date: Wed, 22 Oct 2014 11:57:03 -0700 Subject: [PATCH 49/71] [FIX] exec should write an empty string when file is empty instead of the filename --- scripts/base/utils/exec.bro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/base/utils/exec.bro b/scripts/base/utils/exec.bro index dd992a63cc..6111608eca 100644 --- a/scripts/base/utils/exec.bro +++ b/scripts/base/utils/exec.bro @@ -114,7 +114,7 @@ event Input::end_of_data(name: string, source:string) result$files = table(); if ( track_file !in result$files ) - result$files[track_file] = vector(source); + result$files[track_file] = vector(); Input::remove(name); From 0a597720434fa893623d5c97e081ac5002b896c6 Mon Sep 17 00:00:00 2001 From: Christian Struck Date: Wed, 22 Oct 2014 16:02:19 -0700 Subject: [PATCH 50/71] [ADD] added baseline for the new exec test and added a test to check for the empty files fix. --- .../btest/Baseline/scripts.base.utils.exec/bro..stdout | 6 +++++- testing/btest/scripts/base/utils/exec.test | 10 +++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout b/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout index 3cfdaafb4c..043e1d4baa 100644 --- a/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout +++ b/testing/btest/Baseline/scripts.base.utils.exec/bro..stdout @@ -1,6 +1,10 @@ +test2, [exit_code=1, signal_exit=F, stdout=[here's something on stdout, some more stdout, last stdout], stderr=[and some stderr, more stderr, last stderr], files=] test1, [exit_code=0, signal_exit=F, stdout=[done, exit, stop], stderr=, files={ [out1] = [insert text here, and here], [out2] = [insert more text here, and there] }] -test2, [exit_code=1, signal_exit=F, stdout=[here's something on stdout, some more stdout, last stdout], stderr=[and some stderr, more stderr, last stderr], files=] test4, [exit_code=0, signal_exit=F, stdout=[hibye], stderr=, files=] +test5, [exit_code=0, signal_exit=F, stdout=, stderr=, files={ +[out4] = [test], +[out3] = [] +}] diff --git a/testing/btest/scripts/base/utils/exec.test b/testing/btest/scripts/base/utils/exec.test index 33ba10f97a..389527bcfc 100644 --- a/testing/btest/scripts/base/utils/exec.test +++ b/testing/btest/scripts/base/utils/exec.test @@ -14,7 +14,7 @@ function check_exit_condition() { c += 1; - if ( c == 3 ) + if ( c == 4 ) terminate(); } @@ -35,6 +35,8 @@ event bro_init() # Not sure of a portable way to test signals yet. #test_cmd("test3", [$cmd="bash ../suicide.sh"]); test_cmd("test4", [$cmd="bash ../stdin.sh", $stdin="hibye"]); + test_cmd("test5", [$cmd="bash ../empty_file.sh", + $read_files=set("out3", "out4")]); } @TEST-END-FILE @@ -73,3 +75,9 @@ echo "nope" read -r line echo "$line" @TEST-END-FILE + +@TEST-START-FILE empty_file.sh +#! /usr/bin/env bash +touch out3 +echo "test" > out4 +@TEST-END-FILE From d17b3746cf400c430ff2bbd733aacb9c1b5c431b Mon Sep 17 00:00:00 2001 From: Christian Struck Date: Wed, 22 Oct 2014 16:04:04 -0700 Subject: [PATCH 51/71] [ADD] added baseline for the new active-http test and added a test to check for the content-length 0 fix. --- .../bro..stdout | 7 ++++- .../btest/scripts/base/utils/active-http.test | 30 ++++++++++++++----- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout b/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout index 0284eb19b3..eac07da9a8 100644 --- a/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout +++ b/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout @@ -1,4 +1,9 @@ -[code=200, msg=OK^M, body=It works!, headers={ +test1, [code=200, msg=OK^M, body=It works!, headers={ +[Server] = 1.0, +[Content-type] = text/plain, +[Date] = July 22, 2013 +}] +test2, [code=200, msg=OK^M, body=, headers={ [Server] = 1.0, [Content-type] = text/plain, [Date] = July 22, 2013 diff --git a/testing/btest/scripts/base/utils/active-http.test b/testing/btest/scripts/base/utils/active-http.test index 442d5b9e06..b910ea1c7e 100644 --- a/testing/btest/scripts/base/utils/active-http.test +++ b/testing/btest/scripts/base/utils/active-http.test @@ -1,7 +1,7 @@ # @TEST-REQUIRES: which python # @TEST-REQUIRES: which curl # -# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 1 --addr=127.0.0.1 +# @TEST-EXEC: btest-bg-run httpd python $SCRIPTS/httpd.py --max 2 --addr=127.0.0.1 # @TEST-EXEC: sleep 3 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait 15 @@ -11,18 +11,32 @@ @load base/frameworks/communication # let network-time run. otherwise there are no heartbeats... redef exit_only_after_terminate = T; -event bro_init() - { - local req = ActiveHTTP::Request($url="127.0.0.1:32123"); +global c: count = 0; - when ( local resp = ActiveHTTP::request(req) ) - { - print resp; +function check_exit_condition() + { + c += 1; + + if ( c == 2 ) terminate(); + } + +function test_request(label: string, req: ActiveHTTP::Request) + { + when ( local response = ActiveHTTP::request(req) ) + { + print label, response; + check_exit_condition(); } timeout 1min { print "HTTP request timeout"; - terminate(); + check_exit_condition(); } } + +event bro_init() + { + test_request("test1", [$url="127.0.0.1:32123"]); + test_request("test2", [$url="127.0.0.1:32123/empty", $method="POST"]); + } From de334905866806728a3730bb137f5fefb99f5db2 Mon Sep 17 00:00:00 2001 From: Christian Struck Date: Wed, 22 Oct 2014 16:05:06 -0700 Subject: [PATCH 52/71] [ADD] Added the feature to return 0 content to the python http test server and added functionality for post requests --- testing/scripts/httpd.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/testing/scripts/httpd.py b/testing/scripts/httpd.py index 0732614bc2..e00eb3a6bc 100755 --- a/testing/scripts/httpd.py +++ b/testing/scripts/httpd.py @@ -2,13 +2,26 @@ import BaseHTTPServer + class MyRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler): def do_GET(self): self.send_response(200) self.send_header("Content-type", "text/plain") self.end_headers() - self.wfile.write("It works!") + if "/empty" in self.path: + self.wfile.write("") + else: + self.wfile.write("It works!") + + def do_POST(self): + self.send_response(200) + self.send_header("Content-type", "text/plain") + self.end_headers() + if "/empty" in self.path: + self.wfile.write("") + else: + self.wfile.write("It works!") def version_string(self): return "1.0" From 3e508d316a521607c7600cb6d945e810021766e3 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Thu, 23 Oct 2014 14:20:59 -0700 Subject: [PATCH 53/71] Plugins: Change order in which plugins' scripts are loaded at startup. We now load the top-level __init__.bro before the internal bif.bro so that the former can define types used by the latter. --- CHANGES | 5 +++++ VERSION | 2 +- src/plugin/Manager.cc | 4 ++-- .../Baseline/plugins.bifs-and-scripts-install/output | 2 +- testing/btest/Baseline/plugins.bifs-and-scripts/output | 10 ++++------ testing/btest/plugins/bifs-and-scripts-install.sh | 2 +- testing/btest/plugins/bifs-and-scripts.sh | 2 +- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/CHANGES b/CHANGES index 7049142282..40b9f84d04 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-248 | 2014-10-23 14:20:59 -0700 + + * Change order in which a plugin's scripts are loaded at startup. + (Robin Sommer) + 2.3-247 | 2014-10-21 13:42:38 -0700 * Updates to the SSL analyzer. (Johanna Amann) diff --git a/VERSION b/VERSION index 2b33e98521..762f2d22e9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-247 +2.3-248 diff --git a/src/plugin/Manager.cc b/src/plugin/Manager.cc index ab0b85676b..2ca34d94f3 100644 --- a/src/plugin/Manager.cc +++ b/src/plugin/Manager.cc @@ -172,7 +172,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ // Load {bif,scripts}/__load__.bro automatically. - string init = dir + "lib/bif/__load__.bro"; + string init = dir + "scripts/__load__.bro"; if ( is_file(init) ) { @@ -180,7 +180,7 @@ bool Manager::ActivateDynamicPluginInternal(const std::string& name, bool ok_if_ scripts_to_load.push_back(init); } - init = dir + "scripts/__load__.bro"; + init = dir + "lib/bif/__load__.bro"; if ( is_file(init) ) { diff --git a/testing/btest/Baseline/plugins.bifs-and-scripts-install/output b/testing/btest/Baseline/plugins.bifs-and-scripts-install/output index f03cfddc81..62e53550a1 100644 --- a/testing/btest/Baseline/plugins.bifs-and-scripts-install/output +++ b/testing/btest/Baseline/plugins.bifs-and-scripts-install/output @@ -3,5 +3,5 @@ Demo::Foo - (dynamic, version 1.0) [Event] plugin_event plugin: automatically loaded at startup -calling bif, Hello from the plugin! plugin: manually loaded +calling bif, Hello from the plugin! diff --git a/testing/btest/Baseline/plugins.bifs-and-scripts/output b/testing/btest/Baseline/plugins.bifs-and-scripts/output index 47dd6ed430..89a783241d 100644 --- a/testing/btest/Baseline/plugins.bifs-and-scripts/output +++ b/testing/btest/Baseline/plugins.bifs-and-scripts/output @@ -4,21 +4,19 @@ Demo::Foo - (dynamic, version 1.0) === plugin: automatically loaded at startup -calling bif, Hello from the plugin! === plugin: automatically loaded at startup -calling bif, Hello from the plugin! plugin: manually loaded +calling bif, Hello from the plugin! =-= =-= === plugin: automatically loaded at startup +=== +plugin: automatically loaded at startup +plugin: manually loaded calling bif, Hello from the plugin! === plugin: automatically loaded at startup -calling bif, Hello from the plugin! plugin: manually loaded -=== -plugin: automatically loaded at startup calling bif, Hello from the plugin! -plugin: manually loaded diff --git a/testing/btest/plugins/bifs-and-scripts-install.sh b/testing/btest/plugins/bifs-and-scripts-install.sh index 158f5fc01f..627eb0f2c5 100644 --- a/testing/btest/plugins/bifs-and-scripts-install.sh +++ b/testing/btest/plugins/bifs-and-scripts-install.sh @@ -21,6 +21,7 @@ cat >scripts/demo/foo/manually.bro <scripts/demo/foo/base/at-startup.bro <scripts/demo/foo/manually.bro <scripts/demo/foo/base/at-startup.bro < Date: Fri, 24 Oct 2014 13:39:44 -0700 Subject: [PATCH 54/71] Fixing unstable test. --- CHANGES | 4 ++++ VERSION | 2 +- .../scripts.base.utils.active-http/bro..stdout | 10 ---------- testing/btest/scripts/base/utils/active-http.test | 3 ++- 4 files changed, 7 insertions(+), 12 deletions(-) delete mode 100644 testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout diff --git a/CHANGES b/CHANGES index 7554c9fe88..c363f3c09a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-255 | 2014-10-24 13:39:44 -0700 + + * Fixing unstable active-http test. (Robin Sommer) + 2.3-254 | 2014-10-24 11:40:51 -0700 * Fix active-http.bro to deal reliably with empty server responses, diff --git a/VERSION b/VERSION index 6bd8c152a6..b6bf87e070 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-254 +2.3-255 diff --git a/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout b/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout deleted file mode 100644 index eac07da9a8..0000000000 --- a/testing/btest/Baseline/scripts.base.utils.active-http/bro..stdout +++ /dev/null @@ -1,10 +0,0 @@ -test1, [code=200, msg=OK^M, body=It works!, headers={ -[Server] = 1.0, -[Content-type] = text/plain, -[Date] = July 22, 2013 -}] -test2, [code=200, msg=OK^M, body=, headers={ -[Server] = 1.0, -[Content-type] = text/plain, -[Date] = July 22, 2013 -}] diff --git a/testing/btest/scripts/base/utils/active-http.test b/testing/btest/scripts/base/utils/active-http.test index b910ea1c7e..dbd3fbe141 100644 --- a/testing/btest/scripts/base/utils/active-http.test +++ b/testing/btest/scripts/base/utils/active-http.test @@ -5,7 +5,8 @@ # @TEST-EXEC: sleep 3 # @TEST-EXEC: btest-bg-run bro bro -b %INPUT # @TEST-EXEC: btest-bg-wait 15 -# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: cat bro/.stdout | sort >output +# @TEST-EXEC: btest-diff output @load base/utils/active-http @load base/frameworks/communication # let network-time run. otherwise there are no heartbeats... From 087a9f975d23ef141ad331ca7dc39264402617f6 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 24 Oct 2014 15:33:45 -0700 Subject: [PATCH 55/71] Adding missing baseline. --- CHANGES | 4 ++++ VERSION | 2 +- .../Baseline/scripts.base.utils.active-http/output | 10 ++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/scripts.base.utils.active-http/output diff --git a/CHANGES b/CHANGES index c363f3c09a..25af26df25 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-256 | 2014-10-24 15:33:45 -0700 + + * Adding missing test baseline. (Robin Sommer) + 2.3-255 | 2014-10-24 13:39:44 -0700 * Fixing unstable active-http test. (Robin Sommer) diff --git a/VERSION b/VERSION index b6bf87e070..1c1249d09f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-255 +2.3-256 diff --git a/testing/btest/Baseline/scripts.base.utils.active-http/output b/testing/btest/Baseline/scripts.base.utils.active-http/output new file mode 100644 index 0000000000..43b13ff29a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.utils.active-http/output @@ -0,0 +1,10 @@ +[Content-type] = text/plain, +[Content-type] = text/plain, +[Date] = July 22, 2013 +[Date] = July 22, 2013 +[Server] = 1.0, +[Server] = 1.0, +test1, [code=200, msg=OK^M, body=It works!, headers={ +test2, [code=200, msg=OK^M, body=, headers={ +}] +}] From 65ab987eb609dd9da40a1f896cc70e32cbc66501 Mon Sep 17 00:00:00 2001 From: Vicente Jimenez Aguilar Date: Sat, 25 Oct 2014 11:52:17 +0200 Subject: [PATCH 56/71] Wrong port in scripting documentation HTTP is port 80 not 53 --- doc/scripting/index.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index 559e131b8f..d18f87ac38 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -260,7 +260,7 @@ originating host is referenced by ``c$id$orig_h`` which if given a narrative relates to ``orig_h`` which is a member of ``id`` which is a member of the data structure referred to as ``c`` that was passed into the event handler. Given that the responder port -``c$id$resp_p`` is ``53/tcp``, it's likely that Bro's base HTTP scripts +``c$id$resp_p`` is ``80/tcp``, it's likely that Bro's base HTTP scripts can further populate the connection record. Let's load the ``base/protocols/http`` scripts and check the output of our script. From a26c674dfd71eb1a4fb2e667ffe5acda6559a95e Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 27 Oct 2014 10:04:04 -0500 Subject: [PATCH 57/71] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 7 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index 25af26df25..60f6c515e2 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-259 | 2014-10-27 10:04:04 -0500 + + * Documentation fixes. (Vicente Jimenez Aguilar and Stefano Azzalini) + 2.3-256 | 2014-10-24 15:33:45 -0700 * Adding missing test baseline. (Robin Sommer) diff --git a/VERSION b/VERSION index 1c1249d09f..2ca780950f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-256 +2.3-259 diff --git a/aux/binpac b/aux/binpac index 3a4684801a..c8e017b4b1 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 3a4684801aafa0558383199e9abd711650b53af9 +Subproject commit c8e017b4b1893cf254fc2bc8eedd86b852a2e654 diff --git a/aux/bro-aux b/aux/bro-aux index 95afe42e74..977654dc51 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 95afe42e7474113a16cb2cb09ebdf8b552c59744 +Subproject commit 977654dc51ab08a2afde32241f108cdb4a581d8f diff --git a/aux/broccoli b/aux/broccoli index 33d0ed4a54..acb8fbe8e7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 33d0ed4a54a6ecf08a0b5fe18831aa413b437066 +Subproject commit acb8fbe8e7bc6ace5135fb73dca8e29432cdc1ca diff --git a/aux/broctl b/aux/broctl index 2f808bc854..2b13bfcc94 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2f808bc8541378b1a4953cca02c58c43945d154f +Subproject commit 2b13bfcc941018c76f74b81a6e74e5e4e723c747 diff --git a/cmake b/cmake index 03de0cc467..1316c07f70 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 03de0cc467d2334dcb851eddd843d59fef217909 +Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f From e60ceea87cc8b6a58a845e9dde7682b47bac3c84 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 27 Oct 2014 12:54:17 -0500 Subject: [PATCH 58/71] Fix errors/warnings when compiling with -std=c++11 These are compatibility changes only. --- src/Brofiler.cc | 2 +- src/DNS_Mgr.cc | 4 ++-- src/H3.h | 2 +- src/Reassem.cc | 2 +- src/RemoteSerializer.cc | 10 +++++----- src/SerializationFormat.cc | 6 +++--- src/Sessions.cc | 2 +- src/Stats.cc | 2 +- src/analyzer/Analyzer.cc | 4 ++-- src/analyzer/protocol/http/HTTP.cc | 4 ++-- src/analyzer/protocol/smtp/SMTP.cc | 7 ++++--- src/analyzer/protocol/snmp/snmp-analyzer.pac | 2 +- src/analyzer/protocol/tcp/TCP.cc | 4 ++-- src/analyzer/protocol/tcp/TCP_Reassembler.cc | 10 +++++----- src/cq.c | 4 ++-- 15 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/Brofiler.cc b/src/Brofiler.cc index 777be52217..e7d8c8fdeb 100644 --- a/src/Brofiler.cc +++ b/src/Brofiler.cc @@ -96,7 +96,7 @@ bool Brofiler::WriteStats() map, uint64 >::const_iterator it; for ( it = usage_map.begin(); it != usage_map.end(); ++it ) { - fprintf(f, "%"PRIu64"%c%s%c%s\n", it->second, delim, + fprintf(f, "%" PRIu64"%c%s%c%s\n", it->second, delim, it->first.first.c_str(), delim, it->first.second.c_str()); } diff --git a/src/DNS_Mgr.cc b/src/DNS_Mgr.cc index 2c049ba803..11fd258d09 100644 --- a/src/DNS_Mgr.cc +++ b/src/DNS_Mgr.cc @@ -214,7 +214,7 @@ DNS_Mapping::DNS_Mapping(FILE* f) char req_buf[512+1], name_buf[512+1]; int is_req_host; - if ( sscanf(buf, "%lf %d %512s %d %512s %d %d %"PRIu32, &creation_time, + if ( sscanf(buf, "%lf %d %512s %d %512s %d %d %" PRIu32, &creation_time, &is_req_host, req_buf, &failed, name_buf, &map_type, &num_addrs, &req_ttl) != 8 ) return; @@ -360,7 +360,7 @@ void DNS_Mapping::Clear() void DNS_Mapping::Save(FILE* f) const { - fprintf(f, "%.0f %d %s %d %s %d %d %"PRIu32"\n", creation_time, req_host != 0, + fprintf(f, "%.0f %d %s %d %s %d %d %" PRIu32"\n", creation_time, req_host != 0, req_host ? req_host : req_addr.AsString().c_str(), failed, (names && names[0]) ? names[0] : "*", map_type, num_addrs, req_ttl); diff --git a/src/H3.h b/src/H3.h index 321fda924b..3b4b9ee539 100644 --- a/src/H3.h +++ b/src/H3.h @@ -110,7 +110,7 @@ public: T result = 0; // loop optmized with Duff's Device - register unsigned n = (size + 7) / 8; + unsigned n = (size + 7) / 8; switch ( size % 8 ) { case 0: do { result ^= byte_lookup[offset++][*p++]; case 7: result ^= byte_lookup[offset++][*p++]; diff --git a/src/Reassem.cc b/src/Reassem.cc index 27fb26561f..1ad0cb2717 100644 --- a/src/Reassem.cc +++ b/src/Reassem.cc @@ -182,7 +182,7 @@ DataBlock* Reassembler::AddAndCheck(DataBlock* b, uint64 seq, uint64 upper, { if ( DEBUG_reassem ) { - DEBUG_MSG("%.6f Reassembler::AddAndCheck seq=%"PRIu64", upper=%"PRIu64"\n", + DEBUG_MSG("%.6f Reassembler::AddAndCheck seq=%" PRIu64", upper=%" PRIu64"\n", network_time, seq, upper); } diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index b475c4a8cc..9756e0b0ae 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -707,7 +707,7 @@ RemoteSerializer::PeerID RemoteSerializer::Connect(const IPAddr& ip, const size_t BUFSIZE = 1024; char* data = new char[BUFSIZE]; snprintf(data, BUFSIZE, - "%"PRI_PTR_COMPAT_UINT",%s,%s,%"PRIu16",%"PRIu32",%d", p->id, + "%" PRI_PTR_COMPAT_UINT",%s,%s,%" PRIu16",%" PRIu32",%d", p->id, ip.AsString().c_str(), zone_id.c_str(), port, uint32(retry), use_ssl); @@ -1267,7 +1267,7 @@ bool RemoteSerializer::Listen(const IPAddr& ip, uint16 port, bool expect_ssl, const size_t BUFSIZE = 1024; char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, "%s,%"PRIu16",%d,%d,%s,%"PRIu32, + snprintf(data, BUFSIZE, "%s,%" PRIu16",%d,%d,%s,%" PRIu32, ip.AsString().c_str(), port, expect_ssl, ipv6, zone_id.c_str(), (uint32) retry); @@ -4075,7 +4075,7 @@ bool SocketComm::Connect(Peer* peer) const size_t BUFSIZE = 1024; char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, "%s,%"PRIu32, peer->ip.AsString().c_str(), + snprintf(data, BUFSIZE, "%s,%" PRIu32, peer->ip.AsString().c_str(), peer->port); if ( ! SendToParent(MSG_CONNECTED, peer, data) ) @@ -4190,7 +4190,7 @@ bool SocketComm::Listen() setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &on, sizeof(on)) < 0 ) Error(fmt("can't set IPV6_V6ONLY, %s", strerror(errno))); - if ( bind(fd, res->ai_addr, res->ai_addrlen) < 0 ) + if ( ::bind(fd, res->ai_addr, res->ai_addrlen) < 0 ) { Error(fmt("can't bind to %s:%s, %s", l_addr_str.c_str(), port_str, strerror(errno))); @@ -4287,7 +4287,7 @@ bool SocketComm::AcceptConnection(int fd) const size_t BUFSIZE = 1024; char* data = new char[BUFSIZE]; - snprintf(data, BUFSIZE, "%s,%"PRIu32, peer->ip.AsString().c_str(), + snprintf(data, BUFSIZE, "%s,%" PRIu32, peer->ip.AsString().c_str(), peer->port); if ( ! SendToParent(MSG_CONNECTED, peer, data) ) diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index 6a133d64e4..58935fe175 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -541,19 +541,19 @@ bool XMLSerializationFormat::Write(uint16 v, const char* tag) bool XMLSerializationFormat::Write(uint32 v, const char* tag) { - const char* tmp = fmt("%"PRIu32, v); + const char* tmp = fmt("%" PRIu32, v); return WriteElem(tag, "uint32", tmp, strlen(tmp)); } bool XMLSerializationFormat::Write(uint64 v, const char* tag) { - const char* tmp = fmt("%"PRIu64, v); + const char* tmp = fmt("%" PRIu64, v); return WriteElem(tag, "uint64", tmp, strlen(tmp)); } bool XMLSerializationFormat::Write(int64 v, const char* tag) { - const char* tmp = fmt("%"PRId64, v); + const char* tmp = fmt("%" PRId64, v); return WriteElem(tag, "int64", tmp, strlen(tmp)); } diff --git a/src/Sessions.cc b/src/Sessions.cc index 43e55dd95a..ffc2baf944 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -544,7 +544,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr, if ( proto_typ != 0x0800 && proto_typ != 0x86dd ) { // Not IPv4/IPv6 payload. - Weird(fmt("unknown_gre_protocol_%"PRIu16, proto_typ), ip_hdr, + Weird(fmt("unknown_gre_protocol_%" PRIu16, proto_typ), ip_hdr, encapsulation); return; } diff --git a/src/Stats.cc b/src/Stats.cc index 6cf9a622e1..01ca0a41d3 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -160,7 +160,7 @@ void ProfileLogger::Log() file->Write(fmt("%.06f Connections expired due to inactivity: %d\n", network_time, killed_by_inactivity)); - file->Write(fmt("%.06f Total reassembler data: %"PRIu64"K\n", network_time, + file->Write(fmt("%.06f Total reassembler data: %" PRIu64"K\n", network_time, Reassembler::TotalMemoryAllocation() / 1024)); // Signature engine. diff --git a/src/analyzer/Analyzer.cc b/src/analyzer/Analyzer.cc index fb5602f96e..b4048af467 100644 --- a/src/analyzer/Analyzer.cc +++ b/src/analyzer/Analyzer.cc @@ -598,7 +598,7 @@ SupportAnalyzer* Analyzer::FirstSupportAnalyzer(bool orig) void Analyzer::DeliverPacket(int len, const u_char* data, bool is_orig, uint64 seq, const IP_Hdr* ip, int caplen) { - DBG_LOG(DBG_ANALYZER, "%s DeliverPacket(%d, %s, %"PRIu64", %p, %d) [%s%s]", + DBG_LOG(DBG_ANALYZER, "%s DeliverPacket(%d, %s, %" PRIu64", %p, %d) [%s%s]", fmt_analyzer(this).c_str(), len, is_orig ? "T" : "F", seq, ip, caplen, fmt_bytes((const char*) data, min(40, len)), len > 40 ? "..." : ""); } @@ -612,7 +612,7 @@ void Analyzer::DeliverStream(int len, const u_char* data, bool is_orig) void Analyzer::Undelivered(uint64 seq, int len, bool is_orig) { - DBG_LOG(DBG_ANALYZER, "%s Undelivered(%"PRIu64", %d, %s)", + DBG_LOG(DBG_ANALYZER, "%s Undelivered(%" PRIu64", %d, %s)", fmt_analyzer(this).c_str(), seq, len, is_orig ? "T" : "F"); } diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index e63c8280c9..924c958e43 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -416,7 +416,7 @@ void HTTP_Entity::SubmitHeader(mime::MIME_Header* h) int64_t len = l - f + 1; if ( DEBUG_http ) - DEBUG_MSG("Content-Range length = %"PRId64"\n", len); + DEBUG_MSG("Content-Range length = %" PRId64"\n", len); if ( len > 0 ) { @@ -1060,7 +1060,7 @@ void HTTP_Analyzer::Undelivered(uint64 seq, int len, bool is_orig) { if ( msg ) msg->SubmitEvent(mime::MIME_EVENT_CONTENT_GAP, - fmt("seq=%"PRIu64", len=%d", seq, len)); + fmt("seq=%" PRIu64", len=%d", seq, len)); } // Check if the content gap falls completely within a message body diff --git a/src/analyzer/protocol/smtp/SMTP.cc b/src/analyzer/protocol/smtp/SMTP.cc index 61ed1a4949..a835672378 100644 --- a/src/analyzer/protocol/smtp/SMTP.cc +++ b/src/analyzer/protocol/smtp/SMTP.cc @@ -21,7 +21,9 @@ static const char* smtp_cmd_word[] = { #include "SMTP_cmd.def" }; -#define SMTP_CMD_WORD(code) ((code >= 0) ? smtp_cmd_word[code] : "(UNKNOWN)") +static const char* unknown_cmd = "(UNKNOWN)"; + +#define SMTP_CMD_WORD(code) ((code >= 0) ? smtp_cmd_word[code] : unknown_cmd) SMTP_Analyzer::SMTP_Analyzer(Connection* conn) @@ -83,7 +85,7 @@ void SMTP_Analyzer::Undelivered(uint64 seq, int len, bool is_orig) if ( len <= 0 ) return; - const char* buf = fmt("seq = %"PRIu64", len = %d", seq, len); + const char* buf = fmt("seq = %" PRIu64", len = %d", seq, len); int buf_len = strlen(buf); Unexpected(is_orig, "content gap", buf_len, buf); @@ -422,7 +424,6 @@ void SMTP_Analyzer::NewReply(const int reply_code) if ( state == SMTP_AFTER_GAP && reply_code > 0 ) { state = SMTP_GAP_RECOVERY; - const char* unknown_cmd = SMTP_CMD_WORD(-1); RequestEvent(strlen(unknown_cmd), unknown_cmd, 0, ""); /* if ( line_after_gap ) diff --git a/src/analyzer/protocol/snmp/snmp-analyzer.pac b/src/analyzer/protocol/snmp/snmp-analyzer.pac index cc190e6ebe..feb4474feb 100644 --- a/src/analyzer/protocol/snmp/snmp-analyzer.pac +++ b/src/analyzer/protocol/snmp/snmp-analyzer.pac @@ -84,7 +84,7 @@ StringVal* asn1_oid_to_val(const ASN1Encoding* oid) if ( i > 0 ) { rval += "."; - snprintf(tmp, sizeof(tmp), "%"PRIu64, subidentifier_values[i]); + snprintf(tmp, sizeof(tmp), "%" PRIu64, subidentifier_values[i]); rval += tmp; } else diff --git a/src/analyzer/protocol/tcp/TCP.cc b/src/analyzer/protocol/tcp/TCP.cc index f9fb0fb2b7..88def89689 100644 --- a/src/analyzer/protocol/tcp/TCP.cc +++ b/src/analyzer/protocol/tcp/TCP.cc @@ -1901,7 +1901,7 @@ void TCP_ApplicationAnalyzer::DeliverPacket(int len, const u_char* data, const IP_Hdr* ip, int caplen) { Analyzer::DeliverPacket(len, data, is_orig, seq, ip, caplen); - DBG_LOG(DBG_ANALYZER, "TCP_ApplicationAnalyzer ignoring DeliverPacket(%d, %s, %"PRIu64", %p, %d) [%s%s]", + DBG_LOG(DBG_ANALYZER, "TCP_ApplicationAnalyzer ignoring DeliverPacket(%d, %s, %" PRIu64", %p, %d) [%s%s]", len, is_orig ? "T" : "F", seq, ip, caplen, fmt_bytes((const char*) data, min(40, len)), len > 40 ? "..." : ""); } @@ -2053,7 +2053,7 @@ int TCPStats_Endpoint::DataSent(double /* t */, uint64 seq, int len, int caplen, num_rxmit_bytes += len; } - DEBUG_MSG("%.6f rexmit %"PRIu64" + %d <= %"PRIu64" data_in_flight = %d\n", + DEBUG_MSG("%.6f rexmit %" PRIu64" + %d <= %" PRIu64" data_in_flight = %d\n", network_time, seq, len, max_top_seq, data_in_flight); if ( tcp_rexmit ) diff --git a/src/analyzer/protocol/tcp/TCP_Reassembler.cc b/src/analyzer/protocol/tcp/TCP_Reassembler.cc index 0f7699011e..e00e32ef1b 100644 --- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc +++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc @@ -188,7 +188,7 @@ void TCP_Reassembler::Undelivered(uint64 up_to_seq) if ( DEBUG_tcp_contents ) { - DEBUG_MSG("%.6f Undelivered: IsOrig()=%d up_to_seq=%"PRIu64", last_reassm=%"PRIu64", " + DEBUG_MSG("%.6f Undelivered: IsOrig()=%d up_to_seq=%" PRIu64", last_reassm=%" PRIu64", " "endp: FIN_cnt=%d, RST_cnt=%d, " "peer: FIN_cnt=%d, RST_cnt=%d\n", network_time, IsOrig(), up_to_seq, last_reassem_seq, @@ -219,7 +219,7 @@ void TCP_Reassembler::Undelivered(uint64 up_to_seq) { if ( DEBUG_tcp_contents ) { - DEBUG_MSG("%.6f Undelivered: IsOrig()=%d, seq=%"PRIu64", len=%"PRIu64", " + DEBUG_MSG("%.6f Undelivered: IsOrig()=%d, seq=%" PRIu64", len=%" PRIu64", " "skip_deliveries=%d\n", network_time, IsOrig(), last_reassem_seq, up_to_seq - last_reassem_seq, @@ -350,7 +350,7 @@ void TCP_Reassembler::RecordBlock(DataBlock* b, BroFile* f) void TCP_Reassembler::RecordGap(uint64 start_seq, uint64 upper_seq, BroFile* f) { - if ( f->Write(fmt("\n<>\n", upper_seq - start_seq)) ) + if ( f->Write(fmt("\n<>\n", upper_seq - start_seq)) ) return; reporter->Error("TCP_Reassembler contents gap write failed"); @@ -420,7 +420,7 @@ void TCP_Reassembler::BlockInserted(DataBlock* start_block) void TCP_Reassembler::Overlap(const u_char* b1, const u_char* b2, uint64 n) { if ( DEBUG_tcp_contents ) - DEBUG_MSG("%.6f TCP contents overlap: %"PRIu64" IsOrig()=%d\n", network_time, n, IsOrig()); + DEBUG_MSG("%.6f TCP contents overlap: %" PRIu64" IsOrig()=%d\n", network_time, n, IsOrig()); if ( rexmit_inconsistency && memcmp((const void*) b1, (const void*) b2, n) && @@ -465,7 +465,7 @@ int TCP_Reassembler::DataSent(double t, uint64 seq, int len, if ( DEBUG_tcp_contents ) { - DEBUG_MSG("%.6f DataSent: IsOrig()=%d seq=%"PRIu64" upper=%"PRIu64" ack=%"PRIu64"\n", + DEBUG_MSG("%.6f DataSent: IsOrig()=%d seq=%" PRIu64" upper=%" PRIu64" ack=%" PRIu64"\n", network_time, IsOrig(), seq, upper_seq, ack); } diff --git a/src/cq.c b/src/cq.c index c5405e526a..8005544400 100644 --- a/src/cq.c +++ b/src/cq.c @@ -357,7 +357,7 @@ cq_remove(register struct cq_handle *hp, register double pri, /* The priority must be positive and the cookie non-null */ if (pri <= 0.0 || cookie == NULL) - return (-0); + return (0); bp = hp->buckets + PRI2BUCKET(hp, pri); if (! BUCKETINUSE(bp)) @@ -370,7 +370,7 @@ cq_remove(register struct cq_handle *hp, register double pri, } if ( ! bp ) - return (-0); + return (0); /* Unlink entry */ if ( ! bp2 ) { From 832a2b7bab35e794ed8166452d76387cf433cc78 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 27 Oct 2014 13:03:46 -0500 Subject: [PATCH 59/71] Updating CHANGES and VERSION. --- CHANGES | 4 ++++ VERSION | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/CHANGES b/CHANGES index 60f6c515e2..b41bb8a200 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-260 | 2014-10-27 12:54:17 -0500 + + * Fix errors/warnings when compiling with -std=c++11 (Jon Siwek) + 2.3-259 | 2014-10-27 10:04:04 -0500 * Documentation fixes. (Vicente Jimenez Aguilar and Stefano Azzalini) diff --git a/VERSION b/VERSION index 2ca780950f..30b361afe2 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-259 +2.3-260 From ed73c83b61910b7c5916e46f8a37dfdb145487ef Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Tue, 28 Oct 2014 07:20:26 -0700 Subject: [PATCH 60/71] Fix checking of fwrite return values --- src/File.cc | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/src/File.cc b/src/File.cc index bf6a7e7f51..be46615581 100644 --- a/src/File.cc +++ b/src/File.cc @@ -708,10 +708,10 @@ void BroFile::InitEncrypt(const char* keyfile) secret_len = htonl(secret_len); - if ( ! (fwrite("BROENC1", 7, 1, f) && - fwrite(&secret_len, sizeof(secret_len), 1, f) && - fwrite(secret, ntohl(secret_len), 1, f) && - fwrite(iv, iv_len, 1, f)) ) + if ( fwrite("BROENC1", 7, 1, f) < 7 || + fwrite(&secret_len, sizeof(secret_len), 1, f) < sizeof(secret_len) || + fwrite(secret, ntohl(secret_len), 1, f) < ntohl(secret_len) || + fwrite(iv, iv_len, 1, f) < iv_len ) { reporter->Error("can't write header to log file %s: %s", name, strerror(errno)); @@ -736,7 +736,7 @@ void BroFile::FinishEncrypt() int outl; EVP_SealFinal(cipher_ctx, cipher_buffer, &outl); - if ( outl && ! fwrite(cipher_buffer, outl, 1, f) ) + if ( outl && fwrite(cipher_buffer, outl, 1, f) < outl ) { reporter->Error("write error for %s: %s", name, strerror(errno)); @@ -777,7 +777,7 @@ int BroFile::Write(const char* data, int len) return 0; } - if ( outl && ! fwrite(cipher_buffer, outl, 1, f) ) + if ( outl && fwrite(cipher_buffer, outl, 1, f) < outl ) { reporter->Error("write error for %s: %s", name, strerror(errno)); @@ -792,8 +792,7 @@ int BroFile::Write(const char* data, int len) return 1; } - len = fwrite(data, 1, len, f); - if ( len <= 0 ) + if ( fwrite(data, 1, len, f) < len ) return false; if ( rotate_size && current_size < rotate_size && current_size + len >= rotate_size ) From e5f75cde9340c203744d6808554ac64f6a289079 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Tue, 28 Oct 2014 14:21:16 -0500 Subject: [PATCH 61/71] BIT-1280: Fix checking vector indices via "in". $ cat test.bro local vec: vector of string = { "zero" }; vec[2] = "two"; print 0 in vec, 1 in vec, 2 in vec; $ bro -b test.bro T, F, T --- src/Expr.cc | 12 +++++++++--- .../Baseline/language.vector-in-operator/out | 11 +++++++++++ testing/btest/language/vector-in-operator.bro | 17 +++++++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 testing/btest/Baseline/language.vector-in-operator/out create mode 100644 testing/btest/language/vector-in-operator.bro diff --git a/src/Expr.cc b/src/Expr.cc index 4a29c11cb5..c7ea906865 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -636,7 +636,7 @@ Val* BinaryExpr::Eval(Frame* f) const return v_result; } - if ( is_vec1 || is_vec2 ) + if ( IsVector(Type()->Tag()) && (is_vec1 || is_vec2) ) { // fold vector against scalar VectorVal* vv = (is_vec1 ? v1 : v2)->AsVectorVal(); VectorVal* v_result = new VectorVal(Type()->AsVectorType()); @@ -4703,8 +4703,14 @@ Val* InExpr::Fold(Val* v1, Val* v2) const v2->Type()->Tag() == TYPE_SUBNET ) return new Val(v2->AsSubNetVal()->Contains(v1->AsAddr()), TYPE_BOOL); - TableVal* vt = v2->AsTableVal(); - if ( vt->Lookup(v1, false) ) + Val* res; + + if ( is_vector(v2) ) + res = v2->AsVectorVal()->Lookup(v1); + else + res = v2->AsTableVal()->Lookup(v1, false); + + if ( res ) return new Val(1, TYPE_BOOL); else return new Val(0, TYPE_BOOL); diff --git a/testing/btest/Baseline/language.vector-in-operator/out b/testing/btest/Baseline/language.vector-in-operator/out new file mode 100644 index 0000000000..5d4600a188 --- /dev/null +++ b/testing/btest/Baseline/language.vector-in-operator/out @@ -0,0 +1,11 @@ +[zero, one, , , , five, , seven] +vec[0] = zero.exe +vec[1] = one.exe +vec[2] = +vec[3] = +vec[4] = +vec[5] = five.exe +vec[6] = +vec[7] = seven.exe +vec[8] = +vec[9] = diff --git a/testing/btest/language/vector-in-operator.bro b/testing/btest/language/vector-in-operator.bro new file mode 100644 index 0000000000..5936145363 --- /dev/null +++ b/testing/btest/language/vector-in-operator.bro @@ -0,0 +1,17 @@ +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +local ten = "0123456789"; +local vec: vector of string = { "zero", "one" }; +local n = 0; +vec[5] = "five"; +vec[7] = "seven"; +print vec; +vec = vec + ".exe"; + +for ( c in ten ) + { + local is_set: bool = (n in vec); + print fmt("vec[%s] = %s", n, is_set ? vec[n] : ""); + ++n; + } From 1f7facda5b6a589d5d1046b078435821a1766468 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 30 Oct 2014 12:19:25 -0500 Subject: [PATCH 62/71] Fix segfault if when statement's RHS is unitialized. If it is ever assigned a value, the body of the when can be triggered as usual. Addresses BIT-1176. --- src/Trigger.cc | 2 +- .../language.when-unitialized-rhs/out | 38 +++++++++++++++++++ .../btest/language/when-unitialized-rhs.bro | 32 ++++++++++++++++ 3 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 testing/btest/Baseline/language.when-unitialized-rhs/out create mode 100644 testing/btest/language/when-unitialized-rhs.bro diff --git a/src/Trigger.cc b/src/Trigger.cc index ed5d0e18f6..c2ca9aeb6b 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -206,7 +206,7 @@ bool Trigger::Eval() return false; } - if ( v->IsZero() ) + if ( ! v || v->IsZero() ) { // Not true. Perhaps next time... DBG_LOG(DBG_NOTIFIERS, "%s: trigger condition is false", Name()); diff --git a/testing/btest/Baseline/language.when-unitialized-rhs/out b/testing/btest/Baseline/language.when-unitialized-rhs/out new file mode 100644 index 0000000000..620b384da2 --- /dev/null +++ b/testing/btest/Baseline/language.when-unitialized-rhs/out @@ -0,0 +1,38 @@ +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.when-unitialized-rhs/when-unitialized-rhs.bro, line 9: value used but not set (crashMe) +error in /Users/jsiwek/Projects/bro/bro/testing/btest/.tmp/language.when-unitialized-rhs/when-unitialized-rhs.bro, line 14: value used but not set (x) +1 +2 +3 +4 +5 +6 +7 +8 +9 +10 +2nd when stmt executing, 999 +1st when stmt executing, not anymore you don't +11 +12 +13 +14 +15 +16 +17 +18 +19 +20 +21 +22 +23 +24 +25 +26 +27 +28 +29 +30 +31 +32 +33 +34 diff --git a/testing/btest/language/when-unitialized-rhs.bro b/testing/btest/language/when-unitialized-rhs.bro new file mode 100644 index 0000000000..21b94c6e02 --- /dev/null +++ b/testing/btest/language/when-unitialized-rhs.bro @@ -0,0 +1,32 @@ +# @TEST-EXEC: bro -b -r $TRACES/wikipedia.trace %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +global crashMe: function(): string; +global x: int; + +event bro_init() + { + when( local result = crashMe() ) + { + print "1st when stmt executing", result; + } + + when( local other_result = x ) + { + print "2nd when stmt executing", other_result; + } + } + +global conn_count = 0; + +event new_connection(c: connection) + { + ++conn_count; + print conn_count; + + if ( conn_count == 10 ) + { + x = 999; + crashMe = function(): string { return "not anymore you don't"; }; + } + } From dec96234e3604302c612ddbee4a8c84ce24bfe44 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 30 Oct 2014 13:25:29 -0500 Subject: [PATCH 63/71] Fix some minor Coverity Scan complaints. --- CHANGES | 4 ++++ VERSION | 2 +- src/Flare.cc | 2 +- src/iosource/PktSrc.cc | 1 + src/iosource/PktSrc.h | 4 ++++ src/iosource/pcap/Source.cc | 3 +++ 6 files changed, 14 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index beadb32a63..7c7da83804 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-264 | 2014-10-30 13:25:57 -0500 + + * Fix some minor Coverity Scan complaints. (Jon Siwek) + 2.3-263 | 2014-10-28 15:09:10 -0500 * Fix checking of fwrite return values (Johanna Amann) diff --git a/VERSION b/VERSION index a24ec6faab..b62bade18f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-263 +2.3-264 diff --git a/src/Flare.cc b/src/Flare.cc index dcb5fa2c1f..5df6d663aa 100644 --- a/src/Flare.cc +++ b/src/Flare.cc @@ -22,7 +22,7 @@ static void bad_pipe_op(const char* which) void Flare::Fire() { - char tmp; + char tmp = 0; for ( ; ; ) { diff --git a/src/iosource/PktSrc.cc b/src/iosource/PktSrc.cc index eaf85bbfa4..527dadd393 100644 --- a/src/iosource/PktSrc.cc +++ b/src/iosource/PktSrc.cc @@ -506,6 +506,7 @@ bool PktSrc::ApplyBPFFilter(int index, const struct pcap_pkthdr *hdr, const u_ch { Error(fmt("BPF filter %d not compiled", index)); Close(); + return false; } if ( code->MatchesAnything() ) diff --git a/src/iosource/PktSrc.h b/src/iosource/PktSrc.h index 9c05115257..7137798129 100644 --- a/src/iosource/PktSrc.h +++ b/src/iosource/PktSrc.h @@ -266,7 +266,11 @@ protected: Properties() { + selectable_fd = -1; + link_type = -1; + hdr_size = -1; netmask = PCAP_NETMASK_UNKNOWN; + is_live = false; } }; diff --git a/src/iosource/pcap/Source.cc b/src/iosource/pcap/Source.cc index e96933aaa6..72b19b2f14 100644 --- a/src/iosource/pcap/Source.cc +++ b/src/iosource/pcap/Source.cc @@ -21,6 +21,9 @@ PcapSource::PcapSource(const std::string& path, bool is_live) { props.path = path; props.is_live = is_live; + pd = 0; + memset(¤t_hdr, 0, sizeof(current_hdr)); + memset(&last_hdr, 0, sizeof(last_hdr)); last_data = 0; } From 28770937b5f57cf73354beb850efcfc3a0b5b4af Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Thu, 30 Oct 2014 17:11:46 -0500 Subject: [PATCH 64/71] Add configure options to fine tune local state dirs used by BroControl. --logdir: logs produced at run time --spooldir: other data produced at run time --localstatedir: contains spool or log dirs if those options aren't set Addresses BIT-1166. --- aux/broctl | 2 +- configure | 16 ++++++++++++++++ pkg/make-deb-packages | 5 +++-- pkg/make-rpm-packages | 5 +++-- 4 files changed, 23 insertions(+), 5 deletions(-) diff --git a/aux/broctl b/aux/broctl index 2b13bfcc94..9461f1e1ad 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2b13bfcc941018c76f74b81a6e74e5e4e723c747 +Subproject commit 9461f1e1ad6f7d2e141af1f543ac1d9bc635770b diff --git a/configure b/configure index 5747586db8..2b1c568b26 100755 --- a/configure +++ b/configure @@ -24,6 +24,13 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --prefix=PREFIX installation directory [/usr/local/bro] --scriptdir=PATH root installation directory for Bro scripts [PREFIX/share/bro] + --localstatedir=PATH when using BroControl, path to store log files + and run-time data (within log/ and spool/ subdirs) + [PREFIX] + --spooldir=PATH when using BroControl, path to store run-time data + [PREFIX/spool] + --logdir=PATH when using BroControl, path to store log file + [PREFIX/logs] --conf-files-dir=PATH config files installation directory [PREFIX/etc] Optional Features: @@ -144,6 +151,15 @@ while [ $# -ne 0 ]; do append_cache_entry BRO_ETC_INSTALL_DIR PATH $optarg user_set_conffilesdir="true" ;; + --localstatedir=*) + append_cache_entry BRO_LOCAL_STATE_DIR PATH $optarg + ;; + --spooldir=*) + append_cache_entry BRO_SPOOL_DIR PATH $optarg + ;; + --logdir=*) + append_cache_entry BRO_LOG_DIR PATH $optarg + ;; --enable-debug) append_cache_entry ENABLE_DEBUG BOOL true ;; diff --git a/pkg/make-deb-packages b/pkg/make-deb-packages index 432de8336a..0a435a756f 100755 --- a/pkg/make-deb-packages +++ b/pkg/make-deb-packages @@ -16,6 +16,7 @@ the 'dpkg-dev' package, please install it first. } prefix=/opt/bro +localstatedir=/var/opt/bro # During the packaging process, `dpkg-shlibs` will fail if used on a library # that links to other internal/project libraries unless an RPATH is used or @@ -31,7 +32,7 @@ cd .. ( cd build && make package ) # Full Bro package -./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package +./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package ( cd build && make package ) # Broccoli @@ -42,6 +43,6 @@ cd ../.. # Broctl cd aux/broctl -./configure --prefix=${prefix} --binary-package +./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package ( cd build && make package && mv *.deb ../../../build/ ) cd ../.. diff --git a/pkg/make-rpm-packages b/pkg/make-rpm-packages index 9560cc80ff..43b962f417 100755 --- a/pkg/make-rpm-packages +++ b/pkg/make-rpm-packages @@ -15,6 +15,7 @@ the 'rpm-build' package, please install it first. } prefix=/opt/bro +localstatedir=/var/opt/bro cd .. @@ -24,7 +25,7 @@ cd .. ( cd build && make package ) # Full Bro package -./configure --prefix=${prefix} --pkg-name-prefix=Bro --binary-package +./configure --prefix=${prefix} --localstatedir=${localstatedir} --pkg-name-prefix=Bro --binary-package ( cd build && make package ) # Broccoli @@ -35,6 +36,6 @@ cd ../.. # Broctl cd aux/broctl -./configure --prefix=${prefix} --binary-package +./configure --prefix=${prefix} --localstatedir=${localstatedir} --binary-package ( cd build && make package && mv *.rpm ../../../build/ ) cd ../.. From 2a181a88c5806b8392c0fa004f22789a58cf0a67 Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Oct 2014 10:35:02 -0500 Subject: [PATCH 65/71] Allow arbitrary when statement timeout expressions BIT-1284 #close --- CHANGES | 5 +++++ VERSION | 2 +- src/Trigger.cc | 7 ++++--- testing/btest/language/when.bro | 16 ++++++++++++++-- 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CHANGES b/CHANGES index 522c40bc49..7c4d0a9798 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,9 @@ +2.3-267 | 2014-10-31 10:35:02 -0500 + + * BIT-1284: Allow arbitrary when statement timeout expressions + (Jon Siwek) + 2.3-266 | 2014-10-31 09:21:28 -0500 * BIT-1166: Add configure options to fine tune local state dirs used diff --git a/VERSION b/VERSION index e509a66a4e..aad560167b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-266 +2.3-267 diff --git a/src/Trigger.cc b/src/Trigger.cc index ed5d0e18f6..3e275ac6d9 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -131,18 +131,19 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts, arg_frame->SetDelayed(); } - Val* timeout = arg_timeout ? arg_timeout->ExprVal() : 0; + Val* timeout_val = arg_timeout ? arg_timeout->Eval(arg_frame) : 0; // Make sure we don't get deleted if somebody calls a method like // Timeout() while evaluating the trigger. Ref(this); - if ( ! Eval() && timeout ) + if ( ! Eval() && timeout_val ) { - timer = new TriggerTimer(timeout->AsInterval(), this); + timer = new TriggerTimer(timeout_val->AsInterval(), this); timer_mgr->Add(timer); } + Unref(timeout_val); Unref(this); } diff --git a/testing/btest/language/when.bro b/testing/btest/language/when.bro index 84c1f06cef..d996d1c026 100644 --- a/testing/btest/language/when.bro +++ b/testing/btest/language/when.bro @@ -8,13 +8,25 @@ event bro_init() { - local h1: addr = 127.0.0.1; + local h: addr = 127.0.0.1; - when ( local h1name = lookup_addr(h1) ) + when ( local hname = lookup_addr(h) ) { print "lookup successful"; terminate(); } + timeout 10sec + { + print "timeout (1)"; + } + + local to = 5sec; + # Just checking that timeouts can use arbitrary expressions... + when ( local hname2 = lookup_addr(h) ) {} + timeout to {} + when ( local hname3 = lookup_addr(h) ) {} + timeout to + 2sec {} + print "done"; } From 3b4e5eda5542d5fa947a41161cfff64867f494be Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Fri, 31 Oct 2014 12:12:22 -0500 Subject: [PATCH 66/71] BIT-1283: Fix crash when using &encrypt. --- CHANGES | 4 ++++ VERSION | 2 +- src/File.cc | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 7c4d0a9798..a6b5fa8b6a 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-268 | 2014-10-31 12:12:22 -0500 + + * BIT-1283: Fix crash when using &encrypt. (Jon Siwek) + 2.3-267 | 2014-10-31 10:35:02 -0500 * BIT-1284: Allow arbitrary when statement timeout expressions diff --git a/VERSION b/VERSION index aad560167b..a841cc8a65 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-267 +2.3-268 diff --git a/src/File.cc b/src/File.cc index 8d1c063a15..e62ca732cd 100644 --- a/src/File.cc +++ b/src/File.cc @@ -527,7 +527,7 @@ void BroFile::SetAttrs(Attributes* arg_attrs) if ( ef->AttrExpr() ) InitEncrypt(ef->AttrExpr()->ExprVal()->AsString()->CheckString()); else - InitEncrypt(log_encryption_key->AsString()->CheckString()); + InitEncrypt(opt_internal_string("log_encryption_key")->CheckString()); } if ( attrs->FindAttr(ATTR_RAW_OUTPUT) ) From 5ef6dd0e3c68d01561fbd6566653f8b0a139ad0d Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 31 Oct 2014 17:44:58 -0700 Subject: [PATCH 67/71] Adding call to new binpac::init() function. --- aux/binpac | 2 +- src/main.cc | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/aux/binpac b/aux/binpac index 3a4684801a..3c80db8b56 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 3a4684801aafa0558383199e9abd711650b53af9 +Subproject commit 3c80db8b5697c7b95e1d0d48ce01b625cf70c5a1 diff --git a/src/main.cc b/src/main.cc index 63949c5093..15aea3d3fe 100644 --- a/src/main.cc +++ b/src/main.cc @@ -775,6 +775,9 @@ int main(int argc, char** argv) // DEBUG_MSG("HMAC key: %s\n", md5_digest_print(shared_hmac_md5_key)); init_hash_function(); + // Must come after hash initialization. + binpac::init(); + ERR_load_crypto_strings(); OPENSSL_add_all_algorithms_conf(); SSL_library_init(); From 395f06d93caa9729018d8bd31d7049d63c772e1f Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 31 Oct 2014 17:45:25 -0700 Subject: [PATCH 68/71] Updating submodule(s). [nomail] --- CHANGES | 4 ++++ VERSION | 2 +- aux/binpac | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 47bf14c0de..3109d4670e 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,8 @@ +2.3-274 | 2014-10-31 17:45:25 -0700 + + * Adding call to new binpac::init() function. (Robin Sommer) + 2.3-272 | 2014-10-31 16:29:42 -0700 * Fix segfault if when statement's RHS is unitialized. Addresses diff --git a/VERSION b/VERSION index 88a48cd475..a11a6bac50 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-272 +2.3-274 diff --git a/aux/binpac b/aux/binpac index 3c80db8b56..3a4684801a 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 3c80db8b5697c7b95e1d0d48ce01b625cf70c5a1 +Subproject commit 3a4684801aafa0558383199e9abd711650b53af9 From e0d9adc9c9098ba1ead88adb98c52dc431d80688 Mon Sep 17 00:00:00 2001 From: Robin Sommer Date: Fri, 31 Oct 2014 17:49:02 -0700 Subject: [PATCH 69/71] Updating submodule(s). [nomail] --- aux/binpac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/aux/binpac b/aux/binpac index 3a4684801a..7f440d060e 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 3a4684801aafa0558383199e9abd711650b53af9 +Subproject commit 7f440d060e0df675c1aab3357ff7b93fcf1c2cae From 705989da39a89074849b8d1e4a2cc9588f8a3a28 Mon Sep 17 00:00:00 2001 From: Johanna Amann Date: Sat, 1 Nov 2014 19:37:27 -0700 Subject: [PATCH 70/71] add new curves from draft-ietf-tls-negotiated-ff-dhe --- scripts/base/protocols/ssl/consts.bro | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/scripts/base/protocols/ssl/consts.bro b/scripts/base/protocols/ssl/consts.bro index 54952988f0..278a2a37ae 100644 --- a/scripts/base/protocols/ssl/consts.bro +++ b/scripts/base/protocols/ssl/consts.bro @@ -158,6 +158,12 @@ export { [26] = "brainpoolP256r1", [27] = "brainpoolP384r1", [28] = "brainpoolP512r1", + # draft-ietf-tls-negotiated-ff-dhe-02 + [256] = "ffdhe2432", + [257] = "ffdhe3072", + [258] = "ffdhe4096", + [259] = "ffdhe6144", + [260] = "ffdhe8192", [0xFF01] = "arbitrary_explicit_prime_curves", [0xFF02] = "arbitrary_explicit_char2_curves" } &default=function(i: count):string { return fmt("unknown-%d", i); }; From 25a58f501bbd40dbf3c3e4288cbf5a1447751b1f Mon Sep 17 00:00:00 2001 From: Jon Siwek Date: Mon, 3 Nov 2014 10:19:48 -0600 Subject: [PATCH 71/71] Updating submodule(s). [nomail] --- aux/binpac | 2 +- aux/bro-aux | 2 +- aux/broccoli | 2 +- aux/broctl | 2 +- cmake | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aux/binpac b/aux/binpac index 7f440d060e..77a86591dc 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit 7f440d060e0df675c1aab3357ff7b93fcf1c2cae +Subproject commit 77a86591dcf89d7252d3676d3f1199d6c927d073 diff --git a/aux/bro-aux b/aux/bro-aux index 95afe42e74..977654dc51 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit 95afe42e7474113a16cb2cb09ebdf8b552c59744 +Subproject commit 977654dc51ab08a2afde32241f108cdb4a581d8f diff --git a/aux/broccoli b/aux/broccoli index 33d0ed4a54..acb8fbe8e7 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 33d0ed4a54a6ecf08a0b5fe18831aa413b437066 +Subproject commit acb8fbe8e7bc6ace5135fb73dca8e29432cdc1ca diff --git a/aux/broctl b/aux/broctl index 2f808bc854..39e865dec9 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 2f808bc8541378b1a4953cca02c58c43945d154f +Subproject commit 39e865dec9611b9b53b609cbc8df519cebae0a1e diff --git a/cmake b/cmake index 03de0cc467..1316c07f70 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 03de0cc467d2334dcb851eddd843d59fef217909 +Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f