diff --git a/CHANGES b/CHANGES index a28e7c5250..04a619eea9 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,89 @@ +2.5-106 | 2017-03-13 11:19:03 -0700 + + * print version string to stdout on --version, instead + of printing it to stderr. Output is not an error output. (Pete) + + * Fix compiler warning raised by llvm8. (Johanna Amann) + + * Fix coverity warning in Ascii reader. (Johanna Amann) + +2.5-101 | 2017-03-09 12:20:11 -0500 + + * Input's framework's ascii reader is now more resilient. + + By default, the ASCII reader does not fail on errors anymore. + If there is a problem parsing a line, a reporter warning is + written and parsing continues. If the file is missing or can't + be read, the input thread just tries again on the next heartbeat. + (Seth Hall, Johanna Amann) + +2.5-92 | 2017-03-03 10:44:14 -0800 + + * Move most threading to C++11 primitives (mostly). (Johanna Amann) + + * Fix a test that sometimes fails on FreeBSD. (Daniel Thayer) + + * Remove build time warnings. (Seth Hall) + +2.5-84 | 2017-02-27 15:08:55 -0500 + + * Change semantics of Broker's remote logging to match old communication + framework. (Robin Sommer) + + * Add and fix documentation for HookSetupAnalyzerTree (Johanna Amann) + + +2.5-76 | 2017-02-23 10:19:57 -0800 + + * Kerberos ciphertext had some additional ASN.1 content being lumped + in. (Vlad Grigorescu) + + * Updated Windows version detection to include Windows 10. (Fatema + Bannatwala, Keith Lehigh, Mike, Seth Hall). + +2.5-70 | 2017-02-20 00:20:02 -0500 + + * Rework the RADIUS base script. + + Fixes BIT-1769 which improves logging behavior when replies aren't + seen. Also added a `framed_addr` field to indicate if the radius + server is hinting at an address for the client and a `ttl` field to + show how quickly the server is responding. (Seth Hall) + +2.5-68 | 2017-02-18 13:59:05 -0500 + + * Refactored base krb scripts. (Seth Hall) + + * New script to log ticket hashes in krb log + (policy/protocols/krb/ticket-logging.bro). Also, add + ciphertext to ticket data structure. (John E. Rollinson) + +2.5-62 | 2017-02-15 15:56:38 -0800 + + * Fix case in which scripts were able to access unitialized variables + in certain cases. Addresses BIT-1785. (Jon Siwek) + +2.5-60 | 2017-02-15 15:19:20 -0800 + + * Implement ERSPAN support. + + There is a small caveat to this implementation. The ethernet + header that is carried over the tunnel is ignored. If a user + tries to do MAC address logging, it will only show the MAC + addresses for the outer tunnel and the inner MAC addresses + will be stripped and not available anywhere. (Seth Hall) + + * Tiny mime-type fix from Dan Caselden. (Seth Hall) + + * Update failing intel framework test. (Johanna Amann) + +2.5-55 | 2017-02-10 09:50:43 -0500 + + * Fixed intel expiration reset. Reinserting the same indicator did not reset + the expiration timer for the indicator in the underlying data store. + Addresses BIT-1790. (Jan Grashoefer) + 2.5-51 | 2017-02-06 10:15:56 -0500 * Fix memory leak in file analyzer. (Johanna Amann) diff --git a/NEWS b/NEWS index 9aa1e35c7c..7fbc7cfd4f 100644 --- a/NEWS +++ b/NEWS @@ -4,6 +4,19 @@ release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as BroControl and Broccoli, come with their own ``CHANGES``.) +Bro 2.6 +======= + +Changed Functionality +--------------------- + +- The input framework's Ascii reader is now more resilient. If an input + is marked to reread a file when it changes and the file didn't exist + during a check Bro would stop watching the file in previous versions. + The same could happen with bad data in a line of a file. These + situations do not cause Bro to stop watching input files anymore. The + old behavior is available through settings in the Ascii reader. + Bro 2.5 ======= diff --git a/VERSION b/VERSION index 14bed8d9bf..7e123dc220 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.5-51 +2.5-106 diff --git a/aux/bro-aux b/aux/bro-aux index b1e75f6a21..51bf79d3fc 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit b1e75f6a212250b1730a438f27fc778618b67ec3 +Subproject commit 51bf79d3fc78b5e86c554afe7c24c44b025aa67f diff --git a/aux/broctl b/aux/broctl index 73dbc79ac2..96583ab378 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit 73dbc79ac24cdfef07d8574a4da5d43056ba5fa5 +Subproject commit 96583ab378b1de32ac9804246e1b0e2845fc8b3e diff --git a/aux/btest b/aux/btest index 9d5c7bcac9..dceda16935 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 9d5c7bcac9b04710931bc8a42b545f0691561b2f +Subproject commit dceda169351ddd0c7fe7a5ae5496be1d7af2367b diff --git a/aux/plugins b/aux/plugins index 2322840bcd..c4b5df3aa8 160000 --- a/aux/plugins +++ b/aux/plugins @@ -1 +1 @@ -Subproject commit 2322840bcdbd618ae7bd24e22d874fb30ab89bbb +Subproject commit c4b5df3aa8e5c58a2dc5e5040c7da8369894f24d diff --git a/scripts/base/frameworks/files/magic/general.sig b/scripts/base/frameworks/files/magic/general.sig index d3bed97efa..23b1c1d074 100644 --- a/scripts/base/frameworks/files/magic/general.sig +++ b/scripts/base/frameworks/files/magic/general.sig @@ -116,7 +116,7 @@ signature file-reg-utf16 { # Microsoft Registry format (typically DESKTOP.DAT) signature file-regf { - file-mime "application vnd.ms-regf", 49 + file-mime "application/vnd.ms-regf", 49 file-magic /^\x72\x65\x67\x66/ } diff --git a/scripts/base/frameworks/input/readers/ascii.bro b/scripts/base/frameworks/input/readers/ascii.bro index 1b486ddba0..1d4072e118 100644 --- a/scripts/base/frameworks/input/readers/ascii.bro +++ b/scripts/base/frameworks/input/readers/ascii.bro @@ -18,4 +18,33 @@ export { ## String to use for an unset &optional field. const unset_field = Input::unset_field &redef; + + ## Fail on invalid lines. If set to false, the ascii + ## input reader will jump over invalid lines, reporting + ## warnings in reporter.log. If set to true, errors in + ## input lines will be handled as fatal errors for the + ## reader thread; reading will abort immediately and + ## an error will be logged to reporter.log. + ## Individual readers can use a different value using + ## the $config table. + ## fail_on_invalid_lines = T was the default behavior + ## until Bro 2.6. + const fail_on_invalid_lines = F &redef; + + ## Fail on file read problems. If set to true, the ascii + ## input reader will fail when encountering any problems + ## while reading a file different from invalid lines. + ## Examples of such problems are permission problems, or + ## missing files. + ## When set to false, these problems will be ignored. This + ## has an especially big effect for the REREAD mode, which will + ## seamlessly recover from read errors when a file is + ## only temporarily inaccessible. For MANUAL or STREAM files, + ## errors will most likely still be fatal since no automatic + ## re-reading of the file is attempted. + ## Individual readers can use a different value using + ## the $config table. + ## fail_on_file_problem = T was the default behavior + ## until Bro 2.6. + const fail_on_file_problem = F &redef; } diff --git a/scripts/base/frameworks/intel/main.bro b/scripts/base/frameworks/intel/main.bro index aa51af5ee0..30f9a6bf75 100644 --- a/scripts/base/frameworks/intel/main.bro +++ b/scripts/base/frameworks/intel/main.bro @@ -407,7 +407,11 @@ function insert(item: Item) if ( host !in data_store$host_data ) data_store$host_data[host] = table(); else + { is_new = F; + # Reset expiration timer. + data_store$host_data[host] = data_store$host_data[host]; + } meta_tbl = data_store$host_data[host]; } @@ -422,7 +426,11 @@ function insert(item: Item) if ( !check_subnet(net, data_store$subnet_data) ) data_store$subnet_data[net] = table(); else + { is_new = F; + # Reset expiration timer. + data_store$subnet_data[net] = data_store$subnet_data[net]; + } meta_tbl = data_store$subnet_data[net]; } @@ -436,7 +444,12 @@ function insert(item: Item) if ( [lower_indicator, item$indicator_type] !in data_store$string_data ) data_store$string_data[lower_indicator, item$indicator_type] = table(); else + { is_new = F; + # Reset expiration timer. + data_store$string_data[lower_indicator, item$indicator_type] = + data_store$string_data[lower_indicator, item$indicator_type]; + } meta_tbl = data_store$string_data[lower_indicator, item$indicator_type]; } diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index f3421fd557..d9691e63aa 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -3958,6 +3958,8 @@ export { service_name : string; ## Cipher the ticket was encrypted with cipher : count; + ## Cipher text of the ticket + ciphertext : string &optional; }; type KRB::Ticket_Vector: vector of KRB::Ticket; diff --git a/scripts/base/protocols/krb/files.bro b/scripts/base/protocols/krb/files.bro index 947c2db41a..43e782c696 100644 --- a/scripts/base/protocols/krb/files.bro +++ b/scripts/base/protocols/krb/files.bro @@ -78,34 +78,23 @@ event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priori if ( f$source != "KRB_TCP" && f$source != "KRB" ) return; - local info: Info; - - if ( ! c?$krb ) - { - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; - } - else - info = c$krb; + set_session(c); if ( is_orig ) { - info$client_cert = f$info; - info$client_cert_fuid = f$id; + c$krb$client_cert = f$info; + c$krb$client_cert_fuid = f$id; } else { - info$server_cert = f$info; - info$server_cert_fuid = f$id; + c$krb$server_cert = f$info; + c$krb$server_cert_fuid = f$id; } - - c$krb = info; } function fill_in_subjects(c: connection) { - if ( !c?$krb ) + if ( ! c?$krb ) return; if ( c$krb?$client_cert && c$krb$client_cert?$x509 && c$krb$client_cert$x509?$certificate ) diff --git a/scripts/base/protocols/krb/main.bro b/scripts/base/protocols/krb/main.bro index 13200a559e..02abced683 100644 --- a/scripts/base/protocols/krb/main.bro +++ b/scripts/base/protocols/krb/main.bro @@ -10,41 +10,41 @@ export { type Info: record { ## Timestamp for when the event happened. - ts: time &log; + ts: time &log; ## Unique ID for the connection. - uid: string &log; + uid: string &log; ## The connection's 4-tuple of endpoint addresses/ports. - id: conn_id &log; + id: conn_id &log; ## Request type - Authentication Service ("AS") or ## Ticket Granting Service ("TGS") - request_type: string &log &optional; + request_type: string &log &optional; ## Client - client: string &log &optional; + client: string &log &optional; ## Service - service: string &log; + service: string &log &optional; ## Request result - success: bool &log &optional; + success: bool &log &optional; ## Error code - error_code: count &optional; + error_code: count &optional; ## Error message - error_msg: string &log &optional; + error_msg: string &log &optional; ## Ticket valid from - from: time &log &optional; + from: time &log &optional; ## Ticket valid till - till: time &log &optional; + till: time &log &optional; ## Ticket encryption type - cipher: string &log &optional; + cipher: string &log &optional; ## Forwardable ticket requested - forwardable: bool &log &optional; + forwardable: bool &log &optional; ## Renewable ticket requested - renewable: bool &log &optional; + renewable: bool &log &optional; ## We've already logged this - logged: bool &default=F; + logged: bool &default=F; }; ## The server response error texts which are *not* logged. @@ -80,172 +80,140 @@ event bro_init() &priority=5 Log::create_stream(KRB::LOG, [$columns=Info, $ev=log_krb, $path="kerberos"]); } -event krb_error(c: connection, msg: Error_Msg) &priority=5 +function set_session(c: connection): bool { - local info: Info; - - if ( msg?$error_text && msg$error_text in ignored_errors ) + if ( ! c?$krb ) { - if ( c?$krb ) delete c$krb; - return; + c$krb = Info($ts = network_time(), + $uid = c$uid, + $id = c$id); } - - if ( c?$krb && c$krb$logged ) - return; - - if ( c?$krb ) - info = c$krb; - - if ( ! info?$ts ) - { - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; - } - - if ( ! info?$client && ( msg?$client_name || msg?$client_realm ) ) - info$client = fmt("%s%s", msg?$client_name ? msg$client_name + "/" : "", - msg?$client_realm ? msg$client_realm : ""); - - info$service = msg$service_name; - info$success = F; - - info$error_code = msg$error_code; - - if ( msg?$error_text ) info$error_msg = msg$error_text; - else if ( msg$error_code in error_msg ) info$error_msg = error_msg[msg$error_code]; - - c$krb = info; + + return c$krb$logged; } -event krb_error(c: connection, msg: Error_Msg) &priority=-5 +function do_log(c: connection) { - if ( c?$krb ) + if ( c?$krb && ! c$krb$logged ) { Log::write(KRB::LOG, c$krb); c$krb$logged = T; } } -event krb_as_request(c: connection, msg: KDC_Request) &priority=5 +event krb_error(c: connection, msg: Error_Msg) &priority=5 { - if ( c?$krb && c$krb$logged ) + if ( set_session(c) ) return; - local info: Info; - - if ( !c?$krb ) + if ( msg?$error_text && msg$error_text in ignored_errors ) { - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; + if ( c?$krb ) + delete c$krb; + + return; } - else - info = c$krb; - info$request_type = "AS"; - info$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", msg$service_realm); - info$service = msg$service_name; + if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) ) + c$krb$client = fmt("%s%s", msg?$client_name ? msg$client_name + "/" : "", + msg?$client_realm ? msg$client_realm : ""); - if ( msg?$from ) - info$from = msg$from; + c$krb$service = msg$service_name; + c$krb$success = F; + c$krb$error_code = msg$error_code; - info$till = msg$till; - - info$forwardable = msg$kdc_options$forwardable; - info$renewable = msg$kdc_options$renewable; - - c$krb = info; + if ( msg?$error_text ) + c$krb$error_msg = msg$error_text; + else if ( msg$error_code in error_msg ) + c$krb$error_msg = error_msg[msg$error_code]; } -event krb_tgs_request(c: connection, msg: KDC_Request) &priority=5 +event krb_error(c: connection, msg: Error_Msg) &priority=-5 { - if ( c?$krb && c$krb$logged ) + do_log(c); + } + +event krb_as_request(c: connection, msg: KDC_Request) &priority=5 + { + if ( set_session(c) ) return; - local info: Info; - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; - info$request_type = "TGS"; - info$service = msg$service_name; - if ( msg?$from ) info$from = msg$from; - info$till = msg$till; + c$krb$request_type = "AS"; + c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", msg$service_realm); + c$krb$service = msg$service_name; - info$forwardable = msg$kdc_options$forwardable; - info$renewable = msg$kdc_options$renewable; + if ( msg?$from ) + c$krb$from = msg$from; + c$krb$till = msg$till; - c$krb = info; + c$krb$forwardable = msg$kdc_options$forwardable; + c$krb$renewable = msg$kdc_options$renewable; } event krb_as_response(c: connection, msg: KDC_Response) &priority=5 { - local info: Info; - - if ( c?$krb && c$krb$logged ) + if ( set_session(c) ) return; - if ( c?$krb ) - info = c$krb; - - if ( ! info?$ts ) + if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) ) { - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; + c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", + msg?$client_realm ? msg$client_realm : ""); } - if ( ! info?$client && ( msg?$client_name || msg?$client_realm ) ) - info$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", msg?$client_realm ? msg$client_realm : ""); - - info$service = msg$ticket$service_name; - info$cipher = cipher_name[msg$ticket$cipher]; - info$success = T; - - c$krb = info; + c$krb$service = msg$ticket$service_name; + c$krb$cipher = cipher_name[msg$ticket$cipher]; + c$krb$success = T; } event krb_as_response(c: connection, msg: KDC_Response) &priority=-5 { - Log::write(KRB::LOG, c$krb); - c$krb$logged = T; + do_log(c); + } + +event krb_ap_request(c: connection, ticket: KRB::Ticket, opts: KRB::AP_Options) &priority=5 + { + if ( set_session(c) ) + return; + } + +event krb_tgs_request(c: connection, msg: KDC_Request) &priority=5 + { + if ( set_session(c) ) + return; + + c$krb$request_type = "TGS"; + c$krb$service = msg$service_name; + if ( msg?$from ) + c$krb$from = msg$from; + c$krb$till = msg$till; + + c$krb$forwardable = msg$kdc_options$forwardable; + c$krb$renewable = msg$kdc_options$renewable; } event krb_tgs_response(c: connection, msg: KDC_Response) &priority=5 { - local info: Info; - - if ( c?$krb && c$krb$logged ) + if ( set_session(c) ) return; - if ( c?$krb ) - info = c$krb; - - if ( ! info?$ts ) + if ( ! c$krb?$client && ( msg?$client_name || msg?$client_realm ) ) { - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; + c$krb$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", + msg?$client_realm ? msg$client_realm : ""); } - if ( ! info?$client && ( msg?$client_name || msg?$client_realm ) ) - info$client = fmt("%s/%s", msg?$client_name ? msg$client_name : "", msg?$client_realm ? msg$client_realm : ""); - - info$service = msg$ticket$service_name; - info$cipher = cipher_name[msg$ticket$cipher]; - info$success = T; - - c$krb = info; + c$krb$service = msg$ticket$service_name; + c$krb$cipher = cipher_name[msg$ticket$cipher]; + c$krb$success = T; } event krb_tgs_response(c: connection, msg: KDC_Response) &priority=-5 { - Log::write(KRB::LOG, c$krb); - c$krb$logged = T; + do_log(c); } event connection_state_remove(c: connection) &priority=-5 { - if ( c?$krb && ! c$krb$logged ) - Log::write(KRB::LOG, c$krb); + do_log(c); } diff --git a/scripts/base/protocols/radius/main.bro b/scripts/base/protocols/radius/main.bro index d9c2d08ca8..ea30b27911 100644 --- a/scripts/base/protocols/radius/main.bro +++ b/scripts/base/protocols/radius/main.bro @@ -10,52 +10,51 @@ export { type Info: record { ## Timestamp for when the event happened. - ts : time &log; + ts : time &log; ## Unique ID for the connection. - uid : string &log; + uid : string &log; ## The connection's 4-tuple of endpoint addresses/ports. - id : conn_id &log; + id : conn_id &log; ## The username, if present. - username : string &log &optional; + username : string &log &optional; ## MAC address, if present. - mac : string &log &optional; - ## Remote IP address, if present. - remote_ip : addr &log &optional; + mac : string &log &optional; + ## The address given to the network access server, if + ## present. This is only a hint from the RADIUS server + ## and the network access server is not required to honor + ## the address. + framed_addr : addr &log &optional; + ## Remote IP address, if present. This is collected + ## from the Tunnel-Client-Endpoint attribute. + remote_ip : addr &log &optional; ## Connect info, if present. - connect_info : string &log &optional; + connect_info : string &log &optional; + ## Reply message from the server challenge. This is + ## frequently shown to the user authenticating. + reply_msg : string &log &optional; ## Successful or failed authentication. - result : string &log &optional; - ## Whether this has already been logged and can be ignored. - logged : bool &optional; + result : string &log &optional; + ## The duration between the first request and + ## either the "Access-Accept" message or an error. + ## If the field is empty, it means that either + ## the request or response was not seen. + ttl : interval &log &optional; + ## Whether this has already been logged and can be ignored. + logged : bool &default=F; }; - ## The amount of time we wait for an authentication response before - ## expiring it. - const expiration_interval = 10secs &redef; - - ## Logs an authentication attempt if we didn't see a response in time. - ## - ## t: A table of Info records. - ## - ## idx: The index of the connection$radius table corresponding to the - ## radius authentication about to expire. - ## - ## Returns: 0secs, which when this function is used as an - ## :bro:attr:`&expire_func`, indicates to remove the element at - ## *idx* immediately. - global expire: function(t: table[count] of Info, idx: count): interval; - ## Event that can be handled to access the RADIUS record as it is sent on - ## to the loggin framework. + ## to the logging framework. global log_radius: event(rec: Info); } redef record connection += { - radius: table[count] of Info &optional &write_expire=expiration_interval &expire_func=expire; + radius: Info &optional; }; const ports = { 1812/udp }; +redef likely_server_ports += { ports }; event bro_init() &priority=5 { @@ -63,64 +62,86 @@ event bro_init() &priority=5 Analyzer::register_for_ports(Analyzer::ANALYZER_RADIUS, ports); } -event radius_message(c: connection, result: RADIUS::Message) +event radius_message(c: connection, result: RADIUS::Message) &priority=5 { - local info: Info; - - if ( c?$radius && result$trans_id in c$radius ) - info = c$radius[result$trans_id]; - else + if ( ! c?$radius ) { - c$radius = table(); - info$ts = network_time(); - info$uid = c$uid; - info$id = c$id; + c$radius = Info($ts = network_time(), + $uid = c$uid, + $id = c$id); } - switch ( RADIUS::msg_types[result$code] ) { + switch ( RADIUS::msg_types[result$code] ) + { case "Access-Request": - if ( result?$attributes ) { + if ( result?$attributes ) + { # User-Name - if ( ! info?$username && 1 in result$attributes ) - info$username = result$attributes[1][0]; + if ( ! c$radius?$username && 1 in result$attributes ) + c$radius$username = result$attributes[1][0]; # Calling-Station-Id (we expect this to be a MAC) - if ( ! info?$mac && 31 in result$attributes ) - info$mac = normalize_mac(result$attributes[31][0]); + if ( ! c$radius?$mac && 31 in result$attributes ) + c$radius$mac = normalize_mac(result$attributes[31][0]); # Tunnel-Client-EndPoint (useful for VPNs) - if ( ! info?$remote_ip && 66 in result$attributes ) - info$remote_ip = to_addr(result$attributes[66][0]); + if ( ! c$radius?$remote_ip && 66 in result$attributes ) + c$radius$remote_ip = to_addr(result$attributes[66][0]); # Connect-Info - if ( ! info?$connect_info && 77 in result$attributes ) - info$connect_info = result$attributes[77][0]; - } + if ( ! c$radius?$connect_info && 77 in result$attributes ) + c$radius$connect_info = result$attributes[77][0]; + } + break; + case "Access-Challenge": + if ( result?$attributes ) + { + # Framed-IP-Address + if ( ! c$radius?$framed_addr && 8 in result$attributes ) + c$radius$framed_addr = raw_bytes_to_v4_addr(result$attributes[8][0]); + + if ( ! c$radius?$reply_msg && 18 in result$attributes ) + c$radius$reply_msg = result$attributes[18][0]; + } break; case "Access-Accept": - info$result = "success"; + c$radius$result = "success"; break; case "Access-Reject": - info$result = "failed"; + c$radius$result = "failed"; break; - } - if ( info?$result && ! info?$logged ) - { - info$logged = T; - Log::write(RADIUS::LOG, info); + # TODO: Support RADIUS accounting. (add port 1813/udp above too) + #case "Accounting-Request": + # break; + # + #case "Accounting-Response": + # break; } - - c$radius[result$trans_id] = info; } +event radius_message(c: connection, result: RADIUS::Message) &priority=-5 + { + if ( c$radius?$result ) + { + local ttl = network_time() - c$radius$ts; + if ( ttl != 0secs ) + c$radius$ttl = ttl; -function expire(t: table[count] of Info, idx: count): interval - { - t[idx]$result = "unknown"; - Log::write(RADIUS::LOG, t[idx]); - return 0secs; - } + Log::write(RADIUS::LOG, c$radius); + + delete c$radius; + } + } + +event connection_state_remove(c: connection) &priority=-5 + { + if ( c?$radius && ! c$radius$logged ) + { + c$radius$result = "unknown"; + Log::write(RADIUS::LOG, c$radius); + } + } diff --git a/scripts/policy/frameworks/software/windows-version-detection.bro b/scripts/policy/frameworks/software/windows-version-detection.bro index 7ed1ab359e..50177b2e9b 100644 --- a/scripts/policy/frameworks/software/windows-version-detection.bro +++ b/scripts/policy/frameworks/software/windows-version-detection.bro @@ -48,6 +48,7 @@ export { ["Microsoft-CryptoAPI/6.2"] = [$name="Windows", $version=[$major=6, $minor=2, $addl="8 or Server 2012"]], ["Microsoft-CryptoAPI/6.3"] = [$name="Windows", $version=[$major=6, $minor=3, $addl="8.1 or Server 2012 R2"]], ["Microsoft-CryptoAPI/6.4"] = [$name="Windows", $version=[$major=6, $minor=4, $addl="10 Technical Preview"]], + ["Microsoft-CryptoAPI/10.0"] = [$name="Windows", $version=[$major=10, $minor=0]], } &redef; } diff --git a/scripts/policy/protocols/krb/ticket-logging.bro b/scripts/policy/protocols/krb/ticket-logging.bro new file mode 100644 index 0000000000..22fd3c810b --- /dev/null +++ b/scripts/policy/protocols/krb/ticket-logging.bro @@ -0,0 +1,33 @@ +##! Add Kerberos ticket hashes to the krb.log + +@load base/protocols/krb + +module KRB; + +redef record Info += { + ## Hash of ticket used to authorize request/transaction + auth_ticket: string &log &optional; + ## Hash of ticket returned by the KDC + new_ticket: string &log &optional; +}; + +event krb_ap_request(c: connection, ticket: KRB::Ticket, opts: KRB::AP_Options) + { + # Will be overwritten when request is a TGS + c$krb$request_type = "AP"; + + if ( ticket?$ciphertext ) + c$krb$auth_ticket = md5_hash(ticket$ciphertext); + } + +event krb_as_response(c: connection, msg: KDC_Response) + { + if ( msg$ticket?$ciphertext ) + c$krb$new_ticket = md5_hash(msg$ticket$ciphertext); + } + +event krb_tgs_response(c: connection, msg: KDC_Response) + { + if ( msg$ticket?$ciphertext ) + c$krb$new_ticket = md5_hash(msg$ticket$ciphertext); + } diff --git a/scripts/test-all-policy.bro b/scripts/test-all-policy.bro index 8d1a9ff054..a022060cd4 100644 --- a/scripts/test-all-policy.bro +++ b/scripts/test-all-policy.bro @@ -72,6 +72,7 @@ @load protocols/http/software.bro @load protocols/http/var-extraction-cookies.bro @load protocols/http/var-extraction-uri.bro +@load protocols/krb/ticket-logging.bro @load protocols/modbus/known-masters-slaves.bro @load protocols/modbus/track-memmap.bro @load protocols/mysql/software.bro diff --git a/src/Frame.cc b/src/Frame.cc index e97b948dbe..f30312aaec 100644 --- a/src/Frame.cc +++ b/src/Frame.cc @@ -33,6 +33,15 @@ Frame::~Frame() Release(); } +void Frame::Reset(int startIdx) + { + for ( int i = startIdx; i < size; ++i ) + { + Unref(frame[i]); + frame[i] = 0; + } + } + void Frame::Release() { for ( int i = 0; i < size; ++i ) diff --git a/src/Frame.h b/src/Frame.h index 85e1dbec2e..0c22fa0e4e 100644 --- a/src/Frame.h +++ b/src/Frame.h @@ -24,6 +24,7 @@ public: frame[n] = v; } + void Reset(int startIdx); void Release(); void Describe(ODesc* d) const; diff --git a/src/Func.cc b/src/Func.cc index ccb2570f70..88da9a7a04 100644 --- a/src/Func.cc +++ b/src/Func.cc @@ -397,6 +397,7 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const bodies[i].stmts->GetLocationInfo()); Unref(result); + f->Reset(args->length()); try { diff --git a/src/RemoteSerializer.cc b/src/RemoteSerializer.cc index 4842f819b6..7d8899d8b9 100644 --- a/src/RemoteSerializer.cc +++ b/src/RemoteSerializer.cc @@ -2730,10 +2730,10 @@ bool RemoteSerializer::ProcessLogCreateWriter() id_val = new EnumVal(id, internal_type("Log::ID")->AsEnumType()); writer_val = new EnumVal(writer, internal_type("Log::Writer")->AsEnumType()); - if ( ! log_mgr->CreateWriter(id_val, writer_val, info, num_fields, fields, - true, false, true) ) + if ( ! log_mgr->CreateWriterForRemoteLog(id_val, writer_val, info, num_fields, fields) ) { - delete_fields_up_to = num_fields; + info = 0; + fields = 0; goto error; } @@ -2803,7 +2803,7 @@ bool RemoteSerializer::ProcessLogWrite() id_val = new EnumVal(id, internal_type("Log::ID")->AsEnumType()); writer_val = new EnumVal(writer, internal_type("Log::Writer")->AsEnumType()); - success = log_mgr->Write(id_val, writer_val, path, num_fields, vals); + success = log_mgr->WriteFromRemote(id_val, writer_val, path, num_fields, vals); Unref(id_val); Unref(writer_val); diff --git a/src/Sessions.cc b/src/Sessions.cc index e2a5fba489..e0a47780dd 100644 --- a/src/Sessions.cc +++ b/src/Sessions.cc @@ -431,7 +431,6 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr return; } #endif - int proto = ip_hdr->NextProto(); if ( CheckHeaderTrunc(proto, len, caplen, pkt, encapsulation) ) @@ -510,6 +509,11 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr uint16 proto_typ = ntohs(*((uint16*)(data + 2))); int gre_version = flags_ver & 0x0007; + // If a carried packet has ethernet, this will help skip it. + unsigned int eth_len = 0; + unsigned int gre_len = gre_header_len(flags_ver); + unsigned int ppp_len = gre_version == 1 ? 1 : 0; + if ( gre_version != 0 && gre_version != 1 ) { Weird(fmt("unknown_gre_version_%d", gre_version), ip_hdr, @@ -519,7 +523,18 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr if ( gre_version == 0 ) { - if ( proto_typ != 0x0800 && proto_typ != 0x86dd ) + if ( proto_typ == 0x6558 && len > gre_len + 14 ) + { + // transparent ethernet bridging + eth_len = 14; + proto_typ = ntohs(*((uint16*)(data + gre_len + 12))); + } + + if ( proto_typ == 0x0800 ) + proto = IPPROTO_IPV4; + else if ( proto_typ == 0x86dd ) + proto = IPPROTO_IPV6; + else { // Not IPv4/IPv6 payload. Weird(fmt("unknown_gre_protocol_%" PRIu16, proto_typ), ip_hdr, @@ -527,7 +542,6 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr return; } - proto = (proto_typ == 0x0800) ? IPPROTO_IPV4 : IPPROTO_IPV6; } else // gre_version == 1 @@ -556,10 +570,7 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr return; } - unsigned int gre_len = gre_header_len(flags_ver); - unsigned int ppp_len = gre_version == 1 ? 1 : 0; - - if ( len < gre_len + ppp_len || caplen < gre_len + ppp_len ) + if ( len < gre_len + ppp_len + eth_len || caplen < gre_len + ppp_len + eth_len ) { Weird("truncated_GRE", ip_hdr, encapsulation); return; @@ -578,9 +589,9 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr proto = (ppp_proto == 0x0021) ? IPPROTO_IPV4 : IPPROTO_IPV6; } - data += gre_len + ppp_len; - len -= gre_len + ppp_len; - caplen -= gre_len + ppp_len; + data += gre_len + ppp_len + eth_len; + len -= gre_len + ppp_len + eth_len; + caplen -= gre_len + ppp_len + eth_len; // Treat GRE tunnel like IP tunnels, fallthrough to logic below now // that GRE header is stripped and only payload packet remains. @@ -607,7 +618,6 @@ void NetSessions::DoNextPacket(double t, const Packet* pkt, const IP_Hdr* ip_hdr // Check for a valid inner packet first. IP_Hdr* inner = 0; int result = ParseIPPacket(caplen, data, proto, inner); - if ( result < 0 ) Weird("truncated_inner_IP", ip_hdr, encapsulation); @@ -794,6 +804,7 @@ void NetSessions::DoNextInnerPacket(double t, const Packet* pkt, // Construct fake packet for DoNextPacket Packet p; p.Init(DLT_RAW, &ts, caplen, len, data, false, ""); + DoNextPacket(t, &p, inner, outer); delete inner; diff --git a/src/analyzer/protocol/ayiya/AYIYA.cc b/src/analyzer/protocol/ayiya/AYIYA.cc index a1e00e9b38..9c4ac237ab 100644 --- a/src/analyzer/protocol/ayiya/AYIYA.cc +++ b/src/analyzer/protocol/ayiya/AYIYA.cc @@ -1,7 +1,6 @@ #include "AYIYA.h" #include "Func.h" -#include "events.bif.h" using namespace analyzer::ayiya; diff --git a/src/analyzer/protocol/ayiya/CMakeLists.txt b/src/analyzer/protocol/ayiya/CMakeLists.txt index ae23c25e2d..50113b72d7 100644 --- a/src/analyzer/protocol/ayiya/CMakeLists.txt +++ b/src/analyzer/protocol/ayiya/CMakeLists.txt @@ -5,6 +5,5 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro AYIYA) bro_plugin_cc(AYIYA.cc Plugin.cc) -bro_plugin_bif(events.bif) bro_plugin_pac(ayiya.pac ayiya-protocol.pac ayiya-analyzer.pac) bro_plugin_end() diff --git a/src/analyzer/protocol/ayiya/ayiya.pac b/src/analyzer/protocol/ayiya/ayiya.pac index b1f3a6ef77..ff0af4d47c 100644 --- a/src/analyzer/protocol/ayiya/ayiya.pac +++ b/src/analyzer/protocol/ayiya/ayiya.pac @@ -2,10 +2,6 @@ %include binpac.pac %include bro.pac -%extern{ -#include "events.bif.h" -%} - analyzer AYIYA withcontext { connection: AYIYA_Conn; flow: AYIYA_Flow; diff --git a/src/analyzer/protocol/gssapi/CMakeLists.txt b/src/analyzer/protocol/gssapi/CMakeLists.txt index 222c3cdf4e..d826d36bf7 100644 --- a/src/analyzer/protocol/gssapi/CMakeLists.txt +++ b/src/analyzer/protocol/gssapi/CMakeLists.txt @@ -5,7 +5,7 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro GSSAPI) bro_plugin_cc(GSSAPI.cc Plugin.cc) -bro_plugin_bif(types.bif events.bif) +bro_plugin_bif(events.bif) bro_plugin_pac( gssapi.pac gssapi-protocol.pac diff --git a/src/analyzer/protocol/gssapi/gssapi.pac b/src/analyzer/protocol/gssapi/gssapi.pac index 07759e8daa..55b7fe4255 100644 --- a/src/analyzer/protocol/gssapi/gssapi.pac +++ b/src/analyzer/protocol/gssapi/gssapi.pac @@ -5,7 +5,6 @@ #include "analyzer/Manager.h" #include "analyzer/Analyzer.h" -#include "types.bif.h" #include "events.bif.h" %} diff --git a/src/analyzer/protocol/gssapi/types.bif b/src/analyzer/protocol/gssapi/types.bif deleted file mode 100644 index 996cee9ad8..0000000000 --- a/src/analyzer/protocol/gssapi/types.bif +++ /dev/null @@ -1 +0,0 @@ -# Empty. diff --git a/src/analyzer/protocol/krb/krb-types.pac b/src/analyzer/protocol/krb/krb-types.pac index a5b2eb1041..3b3b9d1f09 100644 --- a/src/analyzer/protocol/krb/krb-types.pac +++ b/src/analyzer/protocol/krb/krb-types.pac @@ -95,6 +95,7 @@ RecordVal* proc_ticket(const KRB_Ticket* ticket) rv->Assign(1, bytestring_to_val(ticket->realm()->data()->content())); rv->Assign(2, GetStringFromPrincipalName(ticket->sname())); rv->Assign(3, asn1_integer_to_val(ticket->enc_part()->data()->etype()->data(), TYPE_COUNT)); + rv->Assign(4, bytestring_to_val(ticket->enc_part()->data()->ciphertext()->encoding()->content())); return rv; } @@ -161,7 +162,7 @@ type KRB_Encrypted_Data = record { true -> next_meta: ASN1EncodingMeta; false -> none_meta: empty; }; - ciphertext : bytestring &length=have_kvno ? next_meta.length : kvno_meta.length; + ciphertext : ASN1OctetString &length=have_kvno ? next_meta.length : kvno_meta.length; } &let { have_kvno : bool = kvno_meta.index == 1; }; diff --git a/src/analyzer/protocol/pia/CMakeLists.txt b/src/analyzer/protocol/pia/CMakeLists.txt index ff55bcf0aa..02397f7aff 100644 --- a/src/analyzer/protocol/pia/CMakeLists.txt +++ b/src/analyzer/protocol/pia/CMakeLists.txt @@ -5,5 +5,4 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro PIA) bro_plugin_cc(PIA.cc Plugin.cc) -bro_plugin_bif(events.bif) bro_plugin_end() diff --git a/src/analyzer/protocol/pia/PIA.cc b/src/analyzer/protocol/pia/PIA.cc index 7d73624dd0..8f5e23a1ce 100644 --- a/src/analyzer/protocol/pia/PIA.cc +++ b/src/analyzer/protocol/pia/PIA.cc @@ -3,8 +3,6 @@ #include "analyzer/protocol/tcp/TCP_Flags.h" #include "analyzer/protocol/tcp/TCP_Reassembler.h" -#include "events.bif.h" - using namespace analyzer::pia; PIA::PIA(analyzer::Analyzer* arg_as_analyzer) diff --git a/src/analyzer/protocol/pia/events.bif b/src/analyzer/protocol/pia/events.bif deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/analyzer/protocol/zip/CMakeLists.txt b/src/analyzer/protocol/zip/CMakeLists.txt index 814119f9f7..40c64afd6e 100644 --- a/src/analyzer/protocol/zip/CMakeLists.txt +++ b/src/analyzer/protocol/zip/CMakeLists.txt @@ -5,5 +5,4 @@ include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_BINARY_DI bro_plugin_begin(Bro ZIP) bro_plugin_cc(ZIP.cc Plugin.cc) -bro_plugin_bif(events.bif) bro_plugin_end() diff --git a/src/analyzer/protocol/zip/ZIP.cc b/src/analyzer/protocol/zip/ZIP.cc index d14df95673..d44c6353cd 100644 --- a/src/analyzer/protocol/zip/ZIP.cc +++ b/src/analyzer/protocol/zip/ZIP.cc @@ -2,8 +2,6 @@ #include "ZIP.h" -#include "events.bif.h" - using namespace analyzer::zip; ZIP_Analyzer::ZIP_Analyzer(Connection* conn, bool orig, Method arg_method) diff --git a/src/analyzer/protocol/zip/events.bif b/src/analyzer/protocol/zip/events.bif deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/broker/Data.cc b/src/broker/Data.cc index bc4197a974..6420144193 100644 --- a/src/broker/Data.cc +++ b/src/broker/Data.cc @@ -709,3 +709,435 @@ bool bro_broker::DataVal::DoUnserialize(UnserialInfo* info) delete [] serial; return true; } + +static broker::util::optional threading_val_to_data_internal(TypeTag type, const threading::Value::_val& val) + { + switch ( type ) { + case TYPE_BOOL: + return {val.int_val != 0}; + + case TYPE_INT: + return {val.int_val}; + + case TYPE_COUNT: + case TYPE_COUNTER: + return {val.uint_val}; + + case TYPE_PORT: + return {broker::port(val.port_val.port, to_broker_port_proto(val.port_val.proto))}; + + case TYPE_ADDR: + { + IPAddr a; + + switch ( val.addr_val.family ) { + case IPv4: + a = IPAddr(val.addr_val.in.in4); + break; + + case IPv6: + a = IPAddr(val.addr_val.in.in6); + break; + + default: + reporter->InternalError("unsupported protocol family in threading_val_to_data"); + } + + in6_addr tmp; + a.CopyIPv6(&tmp); + return {broker::address(reinterpret_cast(&tmp), + broker::address::family::ipv6, + broker::address::byte_order::network)}; + } + + case TYPE_SUBNET: + { + IPAddr a; + int length; + + switch ( val.subnet_val.prefix.family ) { + case IPv4: + a = IPAddr(val.subnet_val.prefix.in.in4); + length = (val.subnet_val.length - 96); + break; + + case IPv6: + a = IPAddr(val.subnet_val.prefix.in.in6); + length = val.subnet_val.length; + break; + + default: + reporter->InternalError("unsupported protocol family in threading_val_to_data"); + } + + in6_addr tmp; + a.CopyIPv6(&tmp); + + auto s = broker::address(reinterpret_cast(&tmp), + broker::address::family::ipv6, + broker::address::byte_order::network); + return {broker::subnet(s, length)}; + } + + case TYPE_DOUBLE: + return {val.double_val}; + + case TYPE_TIME: + return {broker::time_point(val.double_val)}; + + case TYPE_INTERVAL: + return {broker::time_duration(val.double_val)}; + + case TYPE_ENUM: + return {broker::enum_value(std::string(val.string_val.data, val.string_val.length))}; + + case TYPE_STRING: + case TYPE_FILE: + case TYPE_FUNC: + return {std::string(val.string_val.data, val.string_val.length)}; + + case TYPE_TABLE: + { + auto s = broker::set(); + + for ( int i = 0; i < val.set_val.size; ++i ) + { + auto c = bro_broker::threading_val_to_data(val.set_val.vals[i]); + + if ( ! c ) + return {}; + + s.emplace(*c); + } + + return {move(s)}; + } + + case TYPE_VECTOR: + { + auto s = broker::vector(); + + for ( int i = 0; i < val.vector_val.size; ++i ) + { + auto c = bro_broker::threading_val_to_data(val.vector_val.vals[i]); + + if ( ! c ) + return {}; + + s.emplace_back(*c); + } + + return {move(s)}; + } + + default: + reporter->InternalError("unsupported type %s in threading_val_to_data", + type_name(type)); + } + + return {}; + } + + +broker::util::optional bro_broker::threading_val_to_data(const threading::Value* v) + { + broker::util::optional d; + + if ( v->present ) + { + d = threading_val_to_data_internal(v->type, v->val); + + if ( ! d ) + return {}; + } + + auto type = broker::record::field(static_cast(v->type)); + auto present = broker::record::field(v->present); + auto data = (v->present) ? broker::record::field(*d) : broker::util::optional(); + + return {broker::record({move(type), move(present), move(data)})}; + }; + +struct threading_val_converter { + using result_type = bool; + + TypeTag type; + threading::Value::_val& val; + + result_type operator()(bool a) + { + if ( type == TYPE_BOOL ) + { + val.int_val = (a ? 1 : 0); + return true; + } + + return false; + } + + result_type operator()(uint64_t a) + { + if ( type == TYPE_COUNT || type == TYPE_COUNTER ) + { + val.uint_val = a; + return true; + } + + return false; + } + + result_type operator()(int64_t a) + { + if ( type == TYPE_INT ) + { + val.int_val = a; + return true; + } + + return false; + } + + result_type operator()(double a) + { + if ( type == TYPE_DOUBLE ) + { + val.double_val = a; + return true; + } + + return false; + } + + + result_type operator()(const std::string& a) + { + if ( type == TYPE_STRING || type == TYPE_FILE || type == TYPE_FUNC ) + { + auto n = a.size(); + val.string_val.length = n; + val.string_val.data = new char[n]; + memcpy(val.string_val.data, a.data(), n); + return true; + } + + return false; + } + + result_type operator()(const broker::address& a) + { + if ( type == TYPE_ADDR ) + { + auto bits = reinterpret_cast(&a.bytes()); + auto b = IPAddr(*bits); + + if ( a.is_v4() ) + { + val.addr_val.family = IPv4; + b.CopyIPv4(&val.addr_val.in.in4); + return true; + } + + if ( a.is_v6() ) + { + val.addr_val.family = IPv6; + b.CopyIPv6(&val.addr_val.in.in6); + return true; + } + } + + return false; + } + + result_type operator()(const broker::subnet& s) + { + if ( type == TYPE_SUBNET ) + { + auto bits = reinterpret_cast(&s.network().bytes()); + auto a = IPAddr(*bits); + + val.subnet_val.length = s.length(); + + if ( s.network().is_v4() ) + { + val.subnet_val.prefix.family = IPv4; + a.CopyIPv4(&val.subnet_val.prefix.in.in4); + val.subnet_val.length += 96; + return true; + } + + if ( s.network().is_v6() ) + { + val.subnet_val.prefix.family = IPv6; + a.CopyIPv6(&val.subnet_val.prefix.in.in6); + return true; + } + } + + return false; + } + + result_type operator()(const broker::port& a) + { + if ( type == TYPE_PORT ) + { + val.port_val.port = a.number(); + val.port_val.proto = bro_broker::to_bro_port_proto(a.type()); + return true; + } + + return false; + } + + result_type operator()(const broker::time_point& a) + { + if ( type == TYPE_TIME ) + { + val.double_val = a.value; + return true; + } + + return false; + } + + result_type operator()(const broker::time_duration& a) + { + if ( type == TYPE_INTERVAL ) + { + val.double_val = a.value; + return true; + } + + return false; + } + + result_type operator()(const broker::enum_value& a) + { + if ( type == TYPE_ENUM ) + { + auto n = a.name.size(); + val.string_val.length = n; + val.string_val.data = new char[n]; + memcpy(val.string_val.data, a.name.data(), n); + return true; + } + + return false; + } + + result_type operator()(const broker::set& a) + { + if ( type == TYPE_TABLE ) + { + val.set_val.size = a.size(); + val.set_val.vals = new threading::Value* [val.set_val.size]; + + auto p = val.set_val.vals; + + for ( auto& i : a ) + *p++ = bro_broker::data_to_threading_val(move(i)); + + return true; + } + + return false; + } + + result_type operator()(const broker::table& a) + { + return false; + } + + result_type operator()(const broker::vector& a) + { + if ( type == TYPE_VECTOR ) + { + val.vector_val.size = a.size(); + val.vector_val.vals = new threading::Value* [val.vector_val.size]; + + auto p = val.vector_val.vals; + + for ( auto& i : a ) + *p++ = bro_broker::data_to_threading_val(move(i)); + + return true; + } + + return false; + } + + result_type operator()(const broker::record& a) + { + return false; + } +}; + +threading::Value* bro_broker::data_to_threading_val(broker::data d) + { + auto r = broker::get(d); + + if ( ! r ) + return nullptr; + + auto type = broker::get(*r->get(0)); + auto present = broker::get(*r->get(1)); + auto data = r->get(2); + + if ( ! (type && present) ) + return nullptr; + + if ( *present && ! data ) + return nullptr; + + auto tv = new threading::Value; + tv->type = static_cast(*type); + tv->present = *present; + + if ( *present && ! broker::visit(threading_val_converter{tv->type, tv->val}, *data) ) + { + delete tv; + return nullptr; + } + + return tv; + } + +broker::data bro_broker::threading_field_to_data(const threading::Field* f) + { + auto name = broker::record::field(f->name); + auto type = broker::record::field(static_cast(f->type)); + auto subtype = broker::record::field(static_cast(f->subtype)); + auto optional = broker::record::field(f->optional); + + broker::util::optional secondary; + + if ( f->secondary_name ) + secondary = {f->secondary_name}; + + return move(broker::record({name, secondary, type, subtype, optional})); + } + +threading::Field* bro_broker::data_to_threading_field(broker::data d) + { + auto r = broker::get(d); + + if ( ! r ) + return nullptr; + + auto name = broker::get(*r->get(0)); + auto secondary = r->get(1); + auto type = broker::get(*r->get(2)); + auto subtype = broker::get(*r->get(3)); + auto optional = broker::get(*r->get(4)); + + if ( ! (name && type && subtype && optional) ) + return nullptr; + + if ( secondary && ! broker::is(*secondary) ) + return nullptr; + + return new threading::Field(name->c_str(), + secondary ? broker::get(*secondary)->c_str() : nullptr, + static_cast(*type), + static_cast(*subtype), + *optional); + } diff --git a/src/broker/Data.h b/src/broker/Data.h index 0045ad58ad..526304b00c 100644 --- a/src/broker/Data.h +++ b/src/broker/Data.h @@ -61,6 +61,36 @@ broker::util::optional val_to_data(Val* v); */ Val* data_to_val(broker::data d, BroType* type, bool require_log_attr = false); +/** + * Convert a Bro threading::Value to a Broker data value. + * @param v a Bro threading::Value. + * @return a Broker data value if the Bro threading::Value could be converted to one. + */ +broker::util::optional threading_val_to_data(const threading::Value* v); + +/** + * Convert a Bro threading::Field to a Broker data value. + * @param f a Bro threading::Field. + * @return a Broker data value if the Bro threading::Field could be converted to one. + */ +broker::data threading_field_to_data(const threading::Field* f); + +/** + * Convert a Broker data value to a Bro threading::Value. + * @param d a Broker data value. + * @return a pointer to a new Bro threading::Value or a nullptr if the conversion was not + * possible. + */ +threading::Value* data_to_threading_val(broker::data d); + +/** + * Convert a Broker data value to a Bro threading::Value. + * @param d a Broker data value. + * @return a pointer to a new Bro threading::Value or a nullptr if the conversion was not + * possible. + */ +threading::Field* data_to_threading_field(broker::data d); + /** * A Bro value which wraps a Broker data value. */ diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index 334b7f84f5..eebcd2792f 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -20,10 +20,17 @@ using namespace std; VectorType* bro_broker::Manager::vector_of_data_type; EnumType* bro_broker::Manager::log_id_type; +EnumType* bro_broker::Manager::writer_id_type; int bro_broker::Manager::send_flags_self_idx; int bro_broker::Manager::send_flags_peers_idx; int bro_broker::Manager::send_flags_unsolicited_idx; +struct unref_guard { + unref_guard(Val* v) : val(v) {} + ~unref_guard() { Unref(val); } + Val* val; +}; + bro_broker::Manager::Manager() : iosource::IOSource(), next_timestamp(-1) { @@ -83,6 +90,7 @@ bool bro_broker::Manager::Enable(Val* broker_endpoint_flags) send_flags_unsolicited_idx = require_field(send_flags_type, "unsolicited"); log_id_type = internal_type("Log::ID")->AsEnumType(); + writer_id_type = internal_type("Log::Writer")->AsEnumType(); bro_broker::opaque_of_data_type = new OpaqueType("Broker::Data"); bro_broker::opaque_of_set_iterator = new OpaqueType("Broker::SetIterator"); @@ -206,8 +214,10 @@ bool bro_broker::Manager::Event(std::string topic, broker::message msg, int flag return true; } -bool bro_broker::Manager::Log(EnumVal* stream, RecordVal* columns, RecordType* info, - int flags) +bool bro_broker::Manager::CreateLog(EnumVal* stream, EnumVal* writer, + const logging::WriterBackend::WriterInfo& info, + int num_fields, const threading::Field* const * fields, + int flags, const string& peer) { if ( ! Enabled() ) return false; @@ -221,40 +231,82 @@ bool bro_broker::Manager::Log(EnumVal* stream, RecordVal* columns, RecordType* i return false; } - broker::record column_data; + auto writer_name = writer->Type()->AsEnumType()->Lookup(writer->AsEnum()); - for ( auto i = 0u; i < static_cast(info->NumFields()); ++i ) + if ( ! writer_name ) { - if ( ! info->FieldDecl(i)->FindAttr(ATTR_LOG) ) - continue; + reporter->Error("Failed to remotely log: writer %d doesn't have name", + writer->AsEnum()); + return false; + } - auto field_val = columns->LookupWithDefault(i); + auto writer_info = info.ToBroker(); - if ( ! field_val ) - { - column_data.fields.emplace_back(broker::record::field{}); - continue; - } + broker::vector fields_data; - auto opt_field_data = val_to_data(field_val); - Unref(field_val); + for ( auto i = 0; i < num_fields; ++i ) + { + auto field_data = threading_field_to_data(fields[i]); + fields_data.push_back(move(field_data)); + } - if ( ! opt_field_data ) + // TODO: If peer is given, send message to just that one destination. + + std::string topic = std::string("bro/log/") + stream_name; + auto bstream_name = broker::enum_value(move(stream_name)); + auto bwriter_name = broker::enum_value(move(writer_name)); + broker::message msg{move("create"), move(bstream_name), move(bwriter_name), move(writer_info), move(fields_data)}; + endpoint->send(move(topic), move(msg), flags); + + return true; + } + +bool bro_broker::Manager::Log(EnumVal* stream, EnumVal* writer, string path, int num_vals, const threading::Value* const * vals, int flags) + { + if ( ! Enabled() ) + return false; + + auto stream_name = stream->Type()->AsEnumType()->Lookup(stream->AsEnum()); + + if ( ! stream_name ) + { + reporter->Error("Failed to remotely log: stream %d doesn't have name", + stream->AsEnum()); + return false; + } + + auto writer_name = writer->Type()->AsEnumType()->Lookup(writer->AsEnum()); + + if ( ! writer_name ) + { + reporter->Error("Failed to remotely log: writer %d doesn't have name", + writer->AsEnum()); + return false; + } + + broker::vector vals_data; + + for ( auto i = 0; i < num_vals; ++i ) + { + auto field_data = threading_val_to_data(vals[i]); + + if ( ! field_data ) { reporter->Error("Failed to remotely log stream %s: " - "unsupported type '%s'", - stream_name, - type_name(info->FieldDecl(i)->type->Tag())); + "unsupported type for field #%d", + stream_name, i); return false; } - column_data.fields.emplace_back( - broker::record::field{move(*opt_field_data)}); + vals_data.push_back(move(*field_data)); } - broker::message msg{broker::enum_value{stream_name}, move(column_data)}; std::string topic = std::string("bro/log/") + stream_name; + auto bstream_name = broker::enum_value(move(stream_name)); + auto bwriter_name = broker::enum_value(move(writer_name)); + broker::message msg{move("write"), move(bstream_name), move(bwriter_name), move(path), move(vals_data)}; endpoint->send(move(topic), move(msg), flags); + return true; } @@ -639,6 +691,8 @@ void bro_broker::Manager::Process() { switch ( u.status ) { case broker::outgoing_connection_status::tag::established: + log_mgr->SendAllWritersTo(u.peer_name); + if ( Broker::outgoing_connection_established ) { val_list* vl = new val_list; @@ -684,6 +738,8 @@ void bro_broker::Manager::Process() { switch ( u.status ) { case broker::incoming_connection_status::tag::established: + log_mgr->SendAllWritersTo(u.peer_name); + if ( Broker::incoming_connection_established ) { val_list* vl = new val_list; @@ -809,12 +865,6 @@ void bro_broker::Manager::Process() } } - struct unref_guard { - unref_guard(Val* v) : val(v) {} - ~unref_guard() { Unref(val); } - Val* val; - }; - for ( auto& ls : log_subscriptions ) { auto log_messages = ls.second.q.want_pop(); @@ -826,59 +876,19 @@ void bro_broker::Manager::Process() for ( auto& lm : log_messages ) { - if ( lm.size() != 2 ) + if ( lm.size() < 1 ) { - reporter->Warning("got bad remote log size: %zd (expect 2)", - lm.size()); + reporter->Warning("got bad remote log message, no type field"); continue; } - if ( ! broker::get(lm[0]) ) - { - reporter->Warning("got remote log w/o stream id: %d", - static_cast(broker::which(lm[0]))); - continue; - } - - if ( ! broker::get(lm[1]) ) - { - reporter->Warning("got remote log w/o columns: %d", - static_cast(broker::which(lm[1]))); - continue; - } - - auto stream_id = data_to_val(move(lm[0]), log_id_type); - - if ( ! stream_id ) - { - reporter->Warning("failed to unpack remote log stream id"); - continue; - } - - unref_guard stream_id_unreffer{stream_id}; - auto columns_type = log_mgr->StreamColumns(stream_id->AsEnumVal()); - - if ( ! columns_type ) - { - reporter->Warning("got remote log for unknown stream: %s", - stream_id->Type()->AsEnumType()->Lookup( - stream_id->AsEnum())); - continue; - } - - auto columns = data_to_val(move(lm[1]), columns_type, true); - - if ( ! columns ) - { - reporter->Warning("failed to unpack remote log stream columns" - " for stream: %s", - stream_id->Type()->AsEnumType()->Lookup( - stream_id->AsEnum())); - continue; - } - - log_mgr->Write(stream_id->AsEnumVal(), columns->AsRecordVal()); - Unref(columns); + if ( lm[0] == "create" ) + ProcessCreateLog(std::move(lm)); + else if ( lm[0] == "write" ) + ProcessWriteLog(std::move(lm)); + else + reporter->Warning("got remote log w/o known type: %d", + static_cast(broker::which(lm[0]))); } } @@ -979,6 +989,202 @@ void bro_broker::Manager::Process() next_timestamp = -1; } +bool bro_broker::Manager::ProcessCreateLog(broker::message msg) + { + if ( msg.size() != 5 ) + { + reporter->Warning("got bad remote log create size: %zd (expected 5)", + msg.size()); + return false; + } + + unsigned int idx = 1; // Skip type at index 0. + + // Get stream ID. + + if ( ! broker::get(msg[idx]) ) + { + reporter->Warning("got remote log create w/o stream id: %d", + static_cast(broker::which(msg[idx]))); + return false; + } + + auto stream_id = data_to_val(move(msg[idx]), log_id_type); + + if ( ! stream_id ) + { + reporter->Warning("failed to unpack remote log stream id"); + return false; + } + + unref_guard stream_id_unreffer{stream_id}; + ++idx; + + // Get writer ID. + + if ( ! broker::get(msg[idx]) ) + { + reporter->Warning("got remote log create w/o writer id: %d", + static_cast(broker::which(msg[idx]))); + return false; + } + + auto writer_id = data_to_val(move(msg[idx]), writer_id_type); + + if ( ! writer_id ) + { + reporter->Warning("failed to unpack remote log writer id"); + return false; + } + + unref_guard writer_id_unreffer{writer_id}; + ++idx; + + // Get writer info. + + if ( ! broker::get(msg[idx]) ) + { + reporter->Warning("got remote log create w/o writer info id: %d", + static_cast(broker::which(msg[idx]))); + return false; + } + + auto writer_info = std::unique_ptr(new logging::WriterBackend::WriterInfo); + + if ( ! writer_info->FromBroker(std::move(msg[idx])) ) + { + reporter->Warning("failed to unpack remote log writer info"); + return false; + } + + ++idx; + + // Get log fields. + + auto fields_data = broker::get(msg[idx]); + + if ( ! fields_data ) + { + reporter->Warning("failed to unpack remote log fields"); + return false; + } + + auto num_fields = fields_data->size(); + auto fields = new threading::Field* [num_fields]; + + for ( auto i = 0u; i < num_fields; ++i ) + { + if ( auto field = data_to_threading_field((*fields_data)[i]) ) + fields[i] = field; + else + { + reporter->Warning("failed to convert remote log field # %d", i); + return false; + } + } + + if ( ! log_mgr->CreateWriterForRemoteLog(stream_id->AsEnumVal(), writer_id->AsEnumVal(), writer_info.get(), num_fields, fields) ) + { + ODesc d; + stream_id->Describe(&d); + reporter->Warning("failed to create remote log stream for %s locally", d.Description()); + } + + writer_info.release(); // log_mgr took ownership. + return true; + } + +bool bro_broker::Manager::ProcessWriteLog(broker::message msg) + { + if ( msg.size() != 5 ) + { + reporter->Warning("got bad remote log size: %zd (expected 5)", + msg.size()); + return false; + } + + unsigned int idx = 1; // Skip type at index 0. + + // Get stream ID. + + if ( ! broker::get(msg[idx]) ) + { + reporter->Warning("got remote log w/o stream id: %d", + static_cast(broker::which(msg[idx]))); + return false; + } + + auto stream_id = data_to_val(move(msg[idx]), log_id_type); + + if ( ! stream_id ) + { + reporter->Warning("failed to unpack remote log stream id"); + return false; + } + + unref_guard stream_id_unreffer{stream_id}; + ++idx; + + // Get writer ID. + + if ( ! broker::get(msg[idx]) ) + { + reporter->Warning("got remote log w/o writer id: %d", + static_cast(broker::which(msg[idx]))); + return false; + } + + auto writer_id = data_to_val(move(msg[idx]), writer_id_type); + + if ( ! writer_id ) + { + reporter->Warning("failed to unpack remote log writer id"); + return false; + } + + unref_guard writer_id_unreffer{writer_id}; + ++idx; + + // Get path. + + auto path = broker::get(msg[idx]); + + if ( ! path ) + { + reporter->Warning("failed to unpack remote log path"); + return false; + } + + ++idx; + + // Get log values. + + auto vals_data = broker::get(msg[idx]); + + if ( ! vals_data ) + { + reporter->Warning("failed to unpack remote log values"); + return false; + } + + auto num_vals = vals_data->size(); + auto vals = new threading::Value* [num_vals]; + + for ( auto i = 0u; i < num_vals; ++i ) + { + if ( auto val = data_to_threading_val((*vals_data)[i]) ) + vals[i] = val; + else + { + reporter->Warning("failed to convert remote log arg # %d", i); + return false; + } + } + + log_mgr->WriteFromRemote(stream_id->AsEnumVal(), writer_id->AsEnumVal(), *path, num_vals, vals); + return true; + } + bool bro_broker::Manager::AddStore(StoreHandleVal* handle) { if ( ! Enabled() ) diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 9fb7b9e328..d5701c56b1 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -156,15 +156,34 @@ public: /** * Send a log entry to any interested peers. The topic name used is * implicitly "bro/log/". - * @param stream_id the stream to which the log entry belongs. - * @param columns the data which comprises the log entry. - * @param info the record type corresponding to the log's columns. + * @param stream the stream to which the log entry belongs. + * @param writer the writer to use for outputting this log entry. + * @param path the log path to output the log entry to. + * @param num_vals the number of fields to log. + * @param vals the log values to log, of size num_vals. * @param flags tune the behavior of how the message is send. * See the Broker::SendFlags record type. * @return true if the message is sent successfully. */ - bool Log(EnumVal* stream_id, RecordVal* columns, RecordType* info, - int flags); + bool Log(EnumVal* stream, EnumVal* writer, string path, int num_vals, + const threading::Value* const * vals, int flags); + + /** + * Send a message to create a log stream to any interested peers. + * The log stream may or may not already exist on the receiving side. + * The topic name used is implicitly "bro/log/". + * @param stream the stream to which the log entry belongs. + * @param writer the writer to use for outputting this log entry. + * @param info backend initialization information for the writer. + * @param num_fields the number of fields the log has. + * @param fields the log's fields, of size num_fields. + * @param flags tune the behavior of how the message is send. + * See the Broker::SendFlags record type. + * @param peer If given, send the message only to this peer. + * @return true if the message is sent successfully. + */ + bool CreateLog(EnumVal* stream, EnumVal* writer, const logging::WriterBackend::WriterInfo& info, + int num_fields, const threading::Field* const * fields, int flags, const string& peer = ""); /** * Automatically send an event to any interested peers whenever it is @@ -325,7 +344,6 @@ public: static int send_flags_to_int(Val* flags); private: - // IOSource interface overrides: void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, iosource::FD_Set* except) override; @@ -340,6 +358,9 @@ private: broker::endpoint& Endpoint() { return *endpoint; } + bool ProcessCreateLog(broker::message msg); + bool ProcessWriteLog(broker::message msg); + struct QueueWithStats { broker::message_queue q; size_t received = 0; @@ -360,6 +381,7 @@ private: static VectorType* vector_of_data_type; static EnumType* log_id_type; + static EnumType* writer_id_type; static int send_flags_self_idx; static int send_flags_peers_idx; static int send_flags_unsolicited_idx; diff --git a/src/input/Manager.cc b/src/input/Manager.cc index b84d822101..d029e38092 100644 --- a/src/input/Manager.cc +++ b/src/input/Manager.cc @@ -2380,7 +2380,7 @@ Val* Manager::ValueToVal(const Stream* i, const Value* val, BroType* request_typ bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str()); if ( index == -1 ) { - Warning(i, "Value not '%s' for stream '%s' is not a valid enum.", + Warning(i, "Value '%s' for stream '%s' is not a valid enum.", enum_string.c_str(), i->name.c_str()); have_error = true; diff --git a/src/input/readers/ascii/Ascii.cc b/src/input/readers/ascii/Ascii.cc index 8b609bda04..3440d9565d 100644 --- a/src/input/readers/ascii/Ascii.cc +++ b/src/input/readers/ascii/Ascii.cc @@ -49,6 +49,7 @@ FieldMapping FieldMapping::subType() Ascii::Ascii(ReaderFrontend *frontend) : ReaderBackend(frontend) { mtime = 0; + suppress_warnings = false; } Ascii::~Ascii() @@ -61,6 +62,8 @@ void Ascii::DoClose() bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fields) { + suppress_warnings = false; + separator.assign( (const char*) BifConst::InputAscii::separator->Bytes(), BifConst::InputAscii::separator->Len()); @@ -73,6 +76,9 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f unset_field.assign( (const char*) BifConst::InputAscii::unset_field->Bytes(), BifConst::InputAscii::unset_field->Len()); + fail_on_invalid_lines = BifConst::InputAscii::fail_on_invalid_lines; + fail_on_file_problem = BifConst::InputAscii::fail_on_file_problem; + // Set per-filter configuration options. for ( ReaderInfo::config_map::const_iterator i = info.config.begin(); i != info.config.end(); i++ ) { @@ -87,6 +93,12 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f else if ( strcmp(i->first, "unset_field") == 0 ) unset_field.assign(i->second); + + else if ( strcmp(i->first, "fail_on_invalid_lines") == 0 ) + fail_on_invalid_lines = (strncmp(i->second, "T", 1) == 0); + + else if ( strcmp(i->first, "fail_on_file_problem") == 0 ) + fail_on_file_problem = (strncmp(i->second, "T", 1) == 0); } if ( separator.size() != 1 ) @@ -98,26 +110,51 @@ bool Ascii::DoInit(const ReaderInfo& info, int num_fields, const Field* const* f formatter::Ascii::SeparatorInfo sep_info(separator, set_separator, unset_field, empty_field); formatter = unique_ptr(new formatter::Ascii(this, sep_info)); - file.open(info.source); + return DoUpdate(); + } + +void Ascii::FailWarn(bool is_error, const char *msg, bool suppress_future) + { + if ( is_error ) + Error(msg); + else + { + // suppress error message when we are already in error mode. + // There is no reason to repeat it every second. + if ( ! suppress_warnings ) + Warning(msg); + + if ( suppress_future ) + suppress_warnings = true; + } + } + +bool Ascii::OpenFile() + { + if ( file.is_open() ) + return true; + + file.open(Info().source); + if ( ! file.is_open() ) { - Error(Fmt("Init: cannot open %s", info.source)); - return false; + FailWarn(fail_on_file_problem, Fmt("Init: cannot open %s", Info().source), true); + + return ! fail_on_file_problem; } if ( ReadHeader(false) == false ) { - Error(Fmt("Init: cannot open %s; headers are incorrect", info.source)); + FailWarn(fail_on_file_problem, Fmt("Init: cannot open %s; problem reading file header", Info().source), true); + file.close(); - return false; + return ! fail_on_file_problem; } - DoUpdate(); - + suppress_warnings = false; return true; } - bool Ascii::ReadHeader(bool useCached) { // try to read the header line... @@ -128,7 +165,8 @@ bool Ascii::ReadHeader(bool useCached) { if ( ! GetLine(line) ) { - Error("could not read first line"); + FailWarn(fail_on_file_problem, Fmt("Could not read input data file %s; first line could not be read", + Info().source), true); return false; } @@ -170,8 +208,9 @@ bool Ascii::ReadHeader(bool useCached) continue; } - Error(Fmt("Did not find requested field %s in input data file %s.", - field->name, Info().source)); + FailWarn(fail_on_file_problem, Fmt("Did not find requested field %s in input data file %s.", + field->name, Info().source), true); + return false; } @@ -182,8 +221,9 @@ bool Ascii::ReadHeader(bool useCached) map::iterator fit2 = ifields.find(field->secondary_name); if ( fit2 == ifields.end() ) { - Error(Fmt("Could not find requested port type field %s in input data file.", - field->secondary_name)); + FailWarn(fail_on_file_problem, Fmt("Could not find requested port type field %s in input data file.", + field->secondary_name), true); + return false; } @@ -193,7 +233,6 @@ bool Ascii::ReadHeader(bool useCached) columnMap.push_back(f); } - // well, that seems to have worked... return true; } @@ -224,6 +263,9 @@ bool Ascii::GetLine(string& str) // read the entire file and send appropriate thingies back to InputMgr bool Ascii::DoUpdate() { + if ( ! OpenFile() ) + return ! fail_on_file_problem; + switch ( Info().mode ) { case MODE_REREAD: { @@ -231,8 +273,10 @@ bool Ascii::DoUpdate() struct stat sb; if ( stat(Info().source, &sb) == -1 ) { - Error(Fmt("Could not get stat for %s", Info().source)); - return false; + FailWarn(fail_on_file_problem, Fmt("Could not get stat for %s", Info().source), true); + + file.close(); + return ! fail_on_file_problem; } if ( sb.st_mtime <= mtime ) // no change @@ -254,8 +298,10 @@ bool Ascii::DoUpdate() if ( Info().mode == MODE_STREAM ) { file.clear(); // remove end of file evil bits - if ( !ReadHeader(true) ) - return false; // header reading failed + if ( ! ReadHeader(true) ) + { + return ! fail_on_file_problem; // header reading failed + } break; } @@ -263,17 +309,7 @@ bool Ascii::DoUpdate() file.close(); } - file.open(Info().source); - if ( ! file.is_open() ) - { - Error(Fmt("cannot open %s", Info().source)); - return false; - } - - if ( ReadHeader(false) == false ) - { - return false; - } + OpenFile(); break; } @@ -327,14 +363,23 @@ bool Ascii::DoUpdate() if ( (*fit).position > pos || (*fit).secondary_position > pos ) { - Error(Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", - line.c_str(), pos, (*fit).position, (*fit).secondary_position)); + FailWarn(fail_on_invalid_lines, Fmt("Not enough fields in line %s. Found %d fields, want positions %d and %d", + line.c_str(), pos, (*fit).position, (*fit).secondary_position)); - for ( int i = 0; i < fpos; i++ ) - delete fields[i]; + if ( fail_on_invalid_lines ) + { + for ( int i = 0; i < fpos; i++ ) + delete fields[i]; - delete [] fields; - return false; + delete [] fields; + + return false; + } + else + { + error = true; + break; + } } Value* val = formatter->ParseValue(stringfields[(*fit).position], (*fit).name, (*fit).type, (*fit).subtype); @@ -390,6 +435,9 @@ bool Ascii::DoUpdate() bool Ascii::DoHeartbeat(double network_time, double current_time) { + if ( ! OpenFile() ) + return ! fail_on_file_problem; + switch ( Info().mode ) { case MODE_MANUAL: diff --git a/src/input/readers/ascii/Ascii.h b/src/input/readers/ascii/Ascii.h index 20a459968d..7a7fa52590 100644 --- a/src/input/readers/ascii/Ascii.h +++ b/src/input/readers/ascii/Ascii.h @@ -55,6 +55,11 @@ protected: private: bool ReadHeader(bool useCached); bool GetLine(string& str); + bool OpenFile(); + // Call Warning or Error, depending on the is_error boolean. + // In case of a warning, setting suppress_future to true will suppress all future warnings + // (by setting suppress_warnings to true, until suppress_warnings is set back to false) + void FailWarn(bool is_error, const char *msg, bool suppress_future = false); ifstream file; time_t mtime; @@ -70,6 +75,12 @@ private: string set_separator; string empty_field; string unset_field; + bool fail_on_invalid_lines; + bool fail_on_file_problem; + + // this is an internal indicator in case the read is currently in a failed state + // it's used to suppress duplicate error messages. + bool suppress_warnings; std::unique_ptr formatter; }; diff --git a/src/input/readers/ascii/ascii.bif b/src/input/readers/ascii/ascii.bif index 8bb3a96492..80ff4611e7 100644 --- a/src/input/readers/ascii/ascii.bif +++ b/src/input/readers/ascii/ascii.bif @@ -5,3 +5,5 @@ const separator: string; const set_separator: string; const empty_field: string; const unset_field: string; +const fail_on_invalid_lines: bool; +const fail_on_file_problem: bool; diff --git a/src/input/readers/raw/Plugin.cc b/src/input/readers/raw/Plugin.cc index c7af84e34e..e16a233fe6 100644 --- a/src/input/readers/raw/Plugin.cc +++ b/src/input/readers/raw/Plugin.cc @@ -8,7 +8,6 @@ using namespace plugin::Bro_RawReader; Plugin::Plugin() { - init = false; } plugin::Configuration Plugin::Configure() @@ -23,21 +22,14 @@ plugin::Configuration Plugin::Configure() void Plugin::InitPreScript() { - if ( pthread_mutex_init(&fork_mutex, 0) != 0 ) - reporter->FatalError("cannot initialize raw reader's mutex"); - - init = true; } void Plugin::Done() { - pthread_mutex_destroy(&fork_mutex); - init = false; } -pthread_mutex_t* Plugin::ForkMutex() +std::unique_lock Plugin::ForkMutex() { - assert(init); - return &fork_mutex; + return std::unique_lock(fork_mutex, std::defer_lock); } diff --git a/src/input/readers/raw/Plugin.h b/src/input/readers/raw/Plugin.h index 59a5dfd2be..8b7f9edc10 100644 --- a/src/input/readers/raw/Plugin.h +++ b/src/input/readers/raw/Plugin.h @@ -1,4 +1,6 @@ -// See the file in the main distribution directory for copyright. +// See the file in the main distribution directory for copyright. + +#include #include "plugin/Plugin.h" @@ -16,11 +18,10 @@ public: virtual void InitPreScript(); virtual void Done(); - pthread_mutex_t * ForkMutex(); + std::unique_lock ForkMutex(); private: - bool init; - pthread_mutex_t fork_mutex; + std::mutex fork_mutex; }; diff --git a/src/input/readers/raw/Raw.cc b/src/input/readers/raw/Raw.cc index cfa7b72602..ae1f0939a8 100644 --- a/src/input/readers/raw/Raw.cc +++ b/src/input/readers/raw/Raw.cc @@ -96,24 +96,21 @@ bool Raw::SetFDFlags(int fd, int cmd, int flags) } -bool Raw::LockForkMutex() +std::unique_lock Raw::AcquireForkMutex() { - int res = pthread_mutex_lock(plugin::Bro_RawReader::plugin.ForkMutex()); - if ( res == 0 ) - return true; + auto lock = plugin::Bro_RawReader::plugin.ForkMutex(); - Error(Fmt("cannot lock fork mutex: %d", res)); - return false; - } + try + { + lock.lock(); + } -bool Raw::UnlockForkMutex() - { - int res = pthread_mutex_unlock(plugin::Bro_RawReader::plugin.ForkMutex()); - if ( res == 0 ) - return true; + catch ( const std::system_error& e ) + { + reporter->FatalErrorWithCore("cannot lock fork mutex: %s", e.what()); + } - Error(Fmt("cannot unlock fork mutex: %d", res)); - return false; + return lock; } bool Raw::Execute() @@ -126,12 +123,10 @@ bool Raw::Execute() // never crops up... ("never" meaning I haven't seen in it in // hundreds of tests using 50+ threads where before I'd see the issue // w/ just 2 threads ~33% of the time). - if ( ! LockForkMutex() ) - return false; + auto lock = AcquireForkMutex(); if ( pipe(pipes) != 0 || pipe(pipes+2) || pipe(pipes+4) ) { - UnlockForkMutex(); Error(Fmt("Could not open pipe: %d", errno)); return false; } @@ -139,7 +134,6 @@ bool Raw::Execute() childpid = fork(); if ( childpid < 0 ) { - UnlockForkMutex(); Error(Fmt("Could not create child process: %d", errno)); return false; } @@ -208,8 +202,7 @@ bool Raw::Execute() } } - if ( ! UnlockForkMutex() ) - return false; + lock.unlock(); ClosePipeEnd(stdout_out); diff --git a/src/input/readers/raw/Raw.h b/src/input/readers/raw/Raw.h index 2a166ae322..ec50ade8fe 100644 --- a/src/input/readers/raw/Raw.h +++ b/src/input/readers/raw/Raw.h @@ -4,8 +4,8 @@ #define INPUT_READERS_RAW_H #include -#include #include +#include #include "input/ReaderBackend.h" @@ -37,8 +37,7 @@ protected: private: void ClosePipeEnd(int i); bool SetFDFlags(int fd, int cmd, int flags); - bool LockForkMutex(); - bool UnlockForkMutex(); + std::unique_lock AcquireForkMutex(); bool OpenInput(); bool CloseInput(); @@ -87,7 +86,6 @@ private: }; static const int block_size; - static pthread_mutex_t fork_mutex; }; } diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index e1a314d4d6..8c720137e8 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -919,12 +919,6 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) #endif } -#ifdef ENABLE_BROKER - if ( stream->enable_remote && - ! broker_mgr->Log(id, columns, stream->columns, stream->remote_flags) ) - stream->enable_remote = false; -#endif - Unref(columns); return true; @@ -1121,23 +1115,46 @@ threading::Value** Manager::RecordToFilterVals(Stream* stream, Filter* filter, return vals; } +bool Manager::CreateWriterForRemoteLog(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info, + int num_fields, const threading::Field* const* fields) + { + return CreateWriter(id, writer, info, num_fields, fields, true, false, true); + } + +static void delete_info_and_fields(WriterBackend::WriterInfo* info, int num_fields, const threading::Field* const* fields) + { + for ( int i = 0; i < num_fields; i++ ) + delete fields[i]; + + delete [] fields; + delete info; + } + WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info, - int num_fields, const threading::Field* const* fields, bool local, bool remote, bool from_remote, + int num_fields, const threading::Field* const* fields, bool local, bool remote, bool from_remote, const string& instantiating_filter) { + WriterFrontend* result = 0; + Stream* stream = FindStream(id); if ( ! stream ) + { // Don't know this stream. + delete_info_and_fields(info, num_fields, fields); return 0; + } Stream::WriterMap::iterator w = stream->writers.find(Stream::WriterPathPair(writer->AsEnum(), info->path)); if ( w != stream->writers.end() ) + { // If we already have a writer for this. That's fine, we just // return it. + delete_info_and_fields(info, num_fields, fields); return w->second->writer; + } WriterInfo* winfo = new WriterInfo; winfo->type = writer->Ref()->AsEnumVal(); @@ -1190,7 +1207,11 @@ WriterFrontend* Manager::CreateWriter(EnumVal* id, EnumVal* writer, WriterBacken winfo->info->rotation_interval = winfo->interval; winfo->info->rotation_base = parse_rotate_base_time(base_time); - winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote); +#ifdef ENABLE_BROKER + winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote, stream->remote_flags); +#else + winfo->writer = new WriterFrontend(*winfo->info, id, writer, local, remote, 0); +#endif winfo->writer->Init(num_fields, fields); InstallRotationTimer(winfo); @@ -1207,8 +1228,8 @@ void Manager::DeleteVals(int num_fields, threading::Value** vals) delete [] vals; } -bool Manager::Write(EnumVal* id, EnumVal* writer, string path, int num_fields, - threading::Value** vals) +bool Manager::WriteFromRemote(EnumVal* id, EnumVal* writer, string path, int num_fields, + threading::Value** vals) { Stream* stream = FindStream(id); @@ -1280,6 +1301,34 @@ void Manager::SendAllWritersTo(RemoteSerializer::PeerID peer) } } +void Manager::SendAllWritersTo(const string& peer) + { +#ifdef ENABLE_BROKER + for ( vector::iterator s = streams.begin(); s != streams.end(); ++s ) + { + Stream* stream = (*s); + + if ( ! stream ) + continue; + + for ( Stream::WriterMap::iterator i = stream->writers.begin(); + i != stream->writers.end(); i++ ) + { + WriterFrontend* writer = i->second->writer; + + EnumVal writer_val(i->first.first, internal_type("Log::Writer")->AsEnumType()); + broker_mgr->CreateLog((*s)->id, + &writer_val, + *i->second->info, + writer->NumFields(), + writer->Fields(), + stream->remote_flags, + peer); + } + } +#endif + } + bool Manager::SetBuf(EnumVal* id, bool enabled) { Stream* stream = FindStream(id); diff --git a/src/logging/Manager.h b/src/logging/Manager.h index 5d3372fb9b..56626f39d3 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -129,6 +129,52 @@ public: */ bool Write(EnumVal* id, RecordVal* columns); + /** + * Create a new log writer frontend. This is exposed so that the + * communication system can recreate remote log streams locally. + * + * @param id The enum value corresponding to the log stream. + * + * @param writer The enum value corresponding to the desired log writer. + * + * @param info A fully initialized object defining the + * characteristics of the backend writer instance. The method takes + * ownership of this. + * + * @param num_fields The number of log fields to write. + * + * @param vals An array of log fields to write, of size num_fields. + * The method takes ownership of the array. + * + * @return Returns true if the writer was successfully created. + */ + bool CreateWriterForRemoteLog(EnumVal* id, EnumVal* writer, WriterBackend::WriterInfo* info, + int num_fields, const threading::Field* const* fields); + + /** + * Writes out log entries that have already passed through all + * filters (and have raised any events). This is meant called for logs + * received already processed from remote. + * + * @param stream The enum value corresponding to the log stream. + * + * @param writer The enum value corresponding to the desired log writer. + * + * @param path The path of the target log stream to write to. + * + * @param num_fields The number of log values to write. + * + * @param vals An array of log values to write, of size num_fields. + * The method takes ownership of the array. + */ + bool WriteFromRemote(EnumVal* stream, EnumVal* writer, string path, + int num_fields, threading::Value** vals); + + /** + * Announces all instantiated writers to a given Broker peer. + */ + void SendAllWritersTo(const string& peer); + /** * Sets log streams buffering state. This adjusts all associated * writers to the new state. @@ -203,10 +249,6 @@ protected: int num_fields, const threading::Field* const* fields, bool local, bool remote, bool from_remote, const string& instantiating_filter=""); - // Takes ownership of values.. - bool Write(EnumVal* id, EnumVal* writer, string path, - int num_fields, threading::Value** vals); - // Announces all instantiated writers to peer. void SendAllWritersTo(RemoteSerializer::PeerID peer); diff --git a/src/logging/WriterBackend.cc b/src/logging/WriterBackend.cc index 3e868f067a..ebf5165169 100644 --- a/src/logging/WriterBackend.cc +++ b/src/logging/WriterBackend.cc @@ -119,6 +119,65 @@ bool WriterBackend::WriterInfo::Write(SerializationFormat* fmt) const return true; } +#ifdef ENABLE_BROKER +broker::data WriterBackend::WriterInfo::ToBroker() const + { + auto bpath = broker::record::field(path); + auto brotation_base = broker::record::field(rotation_base); + auto brotation_interval = broker::record::field(rotation_interval); + auto bnetwork_time = broker::record::field(network_time); + + auto t = broker::table(); + + for ( config_map::const_iterator i = config.begin(); i != config.end(); ++i ) + { + auto key = std::string(i->first); + auto value = std::string(i->second); + t.insert(std::make_pair(key, value)); + } + + auto bconfig = broker::record::field(move(t)); + + return move(broker::record({bpath, brotation_base, brotation_interval, bnetwork_time, bconfig})); + } + +bool WriterBackend::WriterInfo::FromBroker(broker::data d) + { + auto r = broker::get(d); + + if ( ! r ) + return false; + + auto bpath = broker::get(*r->get(0)); + auto brotation_base = broker::get(*r->get(1)); + auto brotation_interval = broker::get(*r->get(2)); + auto bnetwork_time = broker::get(*r->get(3)); + auto bconfig = broker::get(*r->get(4)); + + if ( ! (bpath && brotation_base && brotation_interval && bnetwork_time && bconfig) ) + return false; + + path = copy_string(bpath->c_str()); + rotation_base = *brotation_base; + rotation_interval = *brotation_interval; + network_time = *bnetwork_time; + + for ( auto i : *bconfig ) + { + auto k = broker::get(i.first); + auto v = broker::get(i.second); + + if ( ! (k && v) ) + return false; + + auto p = std::make_pair(copy_string(k->c_str()), copy_string(v->c_str())); + config.insert(p); + } + + return true; + } +#endif + WriterBackend::WriterBackend(WriterFrontend* arg_frontend) : MsgThread() { num_fields = 0; diff --git a/src/logging/WriterBackend.h b/src/logging/WriterBackend.h index 2a93e8fefc..f6602cc8ca 100644 --- a/src/logging/WriterBackend.h +++ b/src/logging/WriterBackend.h @@ -7,6 +7,10 @@ #include "threading/MsgThread.h" +#ifdef ENABLE_BROKER +#include "broker/Data.h" +#endif + #include "Component.h" class RemoteSerializer; @@ -110,15 +114,18 @@ public: } } - private: - const WriterInfo& operator=(const WriterInfo& other); // Disable. - - friend class ::RemoteSerializer; - // Note, these need to be adapted when changing the struct's // fields. They serialize/deserialize the struct. bool Read(SerializationFormat* fmt); bool Write(SerializationFormat* fmt) const; + +#ifdef ENABLE_BROKER + broker::data ToBroker() const; + bool FromBroker(broker::data d); +#endif + + private: + const WriterInfo& operator=(const WriterInfo& other); // Disable. }; /** diff --git a/src/logging/WriterFrontend.cc b/src/logging/WriterFrontend.cc index 14e131c755..0c2378890f 100644 --- a/src/logging/WriterFrontend.cc +++ b/src/logging/WriterFrontend.cc @@ -2,6 +2,10 @@ #include "Net.h" #include "threading/SerialTypes.h" +#ifdef ENABLE_BROKER +#include "broker/Manager.h" +#endif + #include "Manager.h" #include "WriterFrontend.h" #include "WriterBackend.h" @@ -97,7 +101,7 @@ private: using namespace logging; -WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote) +WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVal* arg_stream, EnumVal* arg_writer, bool arg_local, bool arg_remote, int arg_remote_flags) { stream = arg_stream; writer = arg_writer; @@ -108,6 +112,7 @@ WriterFrontend::WriterFrontend(const WriterBackend::WriterInfo& arg_info, EnumVa buf = true; local = arg_local; remote = arg_remote; + remote_flags = arg_remote_flags; write_buffer = 0; write_buffer_pos = 0; info = new WriterBackend::WriterInfo(arg_info); @@ -167,12 +172,23 @@ void WriterFrontend::Init(int arg_num_fields, const Field* const * arg_fields) backend->SendIn(new InitMessage(backend, arg_num_fields, arg_fields)); if ( remote ) + { remote_serializer->SendLogCreateWriter(stream, writer, *info, arg_num_fields, arg_fields); +#ifdef ENABLE_BROKER + broker_mgr->CreateLog(stream, + writer, + *info, + arg_num_fields, + arg_fields, + remote_flags); +#endif + } + } void WriterFrontend::Write(int arg_num_fields, Value** vals) @@ -191,12 +207,23 @@ void WriterFrontend::Write(int arg_num_fields, Value** vals) } if ( remote ) + { remote_serializer->SendLogWrite(stream, writer, info->path, num_fields, vals); +#ifdef ENABLE_BROKER + broker_mgr->Log(stream, + writer, + info->path, + num_fields, + vals, + remote_flags); +#endif + } + if ( ! backend ) { DeleteVals(arg_num_fields, vals); diff --git a/src/logging/WriterFrontend.h b/src/logging/WriterFrontend.h index e343f326bf..ae37613261 100644 --- a/src/logging/WriterFrontend.h +++ b/src/logging/WriterFrontend.h @@ -34,16 +34,17 @@ public: * * info: The meta information struct for the writer. * - * writer_name: A descriptive name for the writer's type. - * * local: If true, the writer will instantiate a local backend. * - * remote: If true, the writer will forward all data to remote + * remote: If true, the writer will forward logs to remote * clients. * + * remote_flags: Broker flags controlling where remote logs are + * propagated to. + * * Frontends must only be instantiated by the main thread. */ - WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote); + WriterFrontend(const WriterBackend::WriterInfo& info, EnumVal* stream, EnumVal* writer, bool local, bool remote, int remote_flags); /** * Destructor. @@ -214,6 +215,7 @@ protected: bool buf; // True if buffering is enabled (default). bool local; // True if logging locally. bool remote; // True if loggin remotely. + int remote_flags; // Broker propagation flags. const char* name; // Descriptive name of the WriterBackend::WriterInfo* info; // The writer information. diff --git a/src/main.cc b/src/main.cc index 11df1af0c2..0ca39e9e2d 100644 --- a/src/main.cc +++ b/src/main.cc @@ -596,7 +596,7 @@ int main(int argc, char** argv) break; case 'v': - fprintf(stderr, "%s version %s\n", prog, bro_version()); + fprintf(stdout, "%s version %s\n", prog, bro_version()); exit(0); break; diff --git a/src/plugin/Plugin.h b/src/plugin/Plugin.h index 54451dcfb6..49fa7cdd84 100644 --- a/src/plugin/Plugin.h +++ b/src/plugin/Plugin.h @@ -39,7 +39,7 @@ enum HookType { HOOK_DRAIN_EVENTS, //< Activates Plugin::HookDrainEvents() HOOK_UPDATE_NETWORK_TIME, //< Activates Plugin::HookUpdateNetworkTime. HOOK_BRO_OBJ_DTOR, //< Activates Plugin::HookBroObjDtor. - HOOK_SETUP_ANALYZER_TREE, //< Activates Plugin::HookAddToAnalyzerTree + HOOK_SETUP_ANALYZER_TREE, //< Activates Plugin::HookSetupAnalyzerTree // Meta hooks. META_HOOK_PRE, //< Activates Plugin::MetaHookPre(). @@ -642,6 +642,13 @@ protected: */ virtual void HookUpdateNetworkTime(double network_time); + /** + * Hook that executes when a connection's initial analyzer tree + * has been fully set up. The hook can manipulate the tree at this time, + * for example by adding further analyzers. + * + * @param conn The connection. + */ virtual void HookSetupAnalyzerTree(Connection *conn); /** diff --git a/src/threading/BasicThread.cc b/src/threading/BasicThread.cc index 86d7d7b560..d63b307470 100644 --- a/src/threading/BasicThread.cc +++ b/src/threading/BasicThread.cc @@ -5,6 +5,7 @@ #include "bro-config.h" #include "BasicThread.h" #include "Manager.h" +#include "pthread.h" #ifdef HAVE_LINUX #include @@ -21,7 +22,6 @@ BasicThread::BasicThread() started = false; terminating = false; killed = false; - pthread = 0; buf_len = STD_FMT_BUF_LEN; buf = (char*) safe_malloc(buf_len); @@ -50,6 +50,7 @@ void BasicThread::SetName(const char* arg_name) void BasicThread::SetOSName(const char* arg_name) { + static_assert(std::is_same::value, "libstdc++ doesn't use pthread_t"); #ifdef HAVE_LINUX prctl(PR_SET_NAME, arg_name, 0, 0, 0); @@ -60,7 +61,7 @@ void BasicThread::SetOSName(const char* arg_name) #endif #ifdef FREEBSD - pthread_set_name_np(pthread_self(), arg_name, arg_name); + pthread_set_name_np(thread.native_handle(), arg_name, arg_name); #endif } @@ -108,9 +109,7 @@ void BasicThread::Start() started = true; - int err = pthread_create(&pthread, 0, BasicThread::launcher, this); - if ( err != 0 ) - reporter->FatalError("Cannot create thread %s: %s", name, Strerror(err)); + thread = std::thread(&BasicThread::launcher, this); DBG_LOG(DBG_THREADING, "Started thread %s", name); @@ -147,17 +146,21 @@ void BasicThread::Join() if ( ! started ) return; - if ( ! pthread ) + if ( ! thread.joinable() ) return; assert(terminating); - if ( pthread_join(pthread, 0) != 0 ) - reporter->FatalError("Failure joining thread %s", name); + try + { + thread.join(); + } + catch ( const std::system_error& e ) + { + reporter->FatalError("Failure joining thread %s with error %s", name, e.what()); + } DBG_LOG(DBG_THREADING, "Joined with thread %s", name); - - pthread = 0; } void BasicThread::Kill() @@ -180,6 +183,7 @@ void BasicThread::Done() void* BasicThread::launcher(void *arg) { + static_assert(std::is_same::value, "libstdc++ doesn't use pthread_t"); BasicThread* thread = (BasicThread *)arg; // Block signals in thread. We handle signals only in the main diff --git a/src/threading/BasicThread.h b/src/threading/BasicThread.h index 6386e5ae66..ea829fce54 100644 --- a/src/threading/BasicThread.h +++ b/src/threading/BasicThread.h @@ -2,8 +2,7 @@ #ifndef THREADING_BASICTHREAD_H #define THREADING_BASICTHREAD_H -#include -#include +#include #include "util.h" @@ -35,6 +34,9 @@ public: */ BasicThread(); + BasicThread(BasicThread const&) = delete; + BasicThread& operator =(BasicThread const&) = delete; + /** * Returns a descriptive name for the thread. If not set via * SetName(). If not set, a default name is choosen automatically. @@ -192,11 +194,11 @@ protected: void Done(); private: - // pthread entry function. + // thread entry function. static void* launcher(void *arg); const char* name; - pthread_t pthread; + std::thread thread; bool started; // Set to to true once running. bool terminating; // Set to to true to signal termination. bool killed; // Set to true once forcefully killed. diff --git a/src/threading/MsgThread.h b/src/threading/MsgThread.h index 96da68e1d0..480ab2974c 100644 --- a/src/threading/MsgThread.h +++ b/src/threading/MsgThread.h @@ -2,8 +2,6 @@ #ifndef THREADING_MSGTHREAD_H #define THREADING_MSGTHREAD_H -#include - #include "DebugLogger.h" #include "BasicThread.h" diff --git a/src/threading/Queue.h b/src/threading/Queue.h index 6d21bfd998..9ac9268a7a 100644 --- a/src/threading/Queue.h +++ b/src/threading/Queue.h @@ -1,7 +1,8 @@ #ifndef THREADING_QUEUE_H #define THREADING_QUEUE_H -#include +#include +#include #include #include #include @@ -22,7 +23,7 @@ namespace threading { * * All Queue instances must be instantiated by Bro's main thread. * - * TODO: Unclear how critical performance is for this qeueue. We could like;y + * TODO: Unclear how critical performance is for this qeueue. We could likely * optimize it further if helpful. */ template @@ -71,9 +72,10 @@ public: */ bool MaybeReady() { return (num_reads != num_writes); } - /** Wake up the reader if it's currently blocked for input. This is - primarily to give it a chance to check termination quickly. - **/ + /** + * Wake up the reader if it's currently blocked for input. This is + * primarily to give it a chance to check termination quickly. + */ void WakeUp(); /** @@ -94,14 +96,17 @@ public: * Returns statistics about the queue's usage. * * @param stats A pointer to a structure that will be filled with - * current numbers. */ + * current numbers. + */ void GetStats(Stats* stats); private: static const int NUM_QUEUES = 8; - pthread_mutex_t mutex[NUM_QUEUES]; // Mutex protected shared accesses. - pthread_cond_t has_data[NUM_QUEUES]; // Signals when data becomes available + std::vector> LocksForAllQueues(); + + std::mutex mutex[NUM_QUEUES]; // Mutex protected shared accesses. + std::condition_variable has_data[NUM_QUEUES]; // Signals when data becomes available std::queue messages[NUM_QUEUES]; // Actually holds the queued messages int read_ptr; // Where the next operation will read from @@ -115,17 +120,18 @@ private: uint64_t num_writes; }; -inline static void safe_lock(pthread_mutex_t* mutex) +inline static std::unique_lock acquire_lock(std::mutex& m) { - int res = pthread_mutex_lock(mutex); - if ( res != 0 ) - reporter->FatalErrorWithCore("cannot lock mutex: %d(%s)", res, strerror(res)); - } - -inline static void safe_unlock(pthread_mutex_t* mutex) - { - if ( pthread_mutex_unlock(mutex) != 0 ) - reporter->FatalErrorWithCore("cannot unlock mutex"); + try + { + return std::unique_lock(m); + } + catch ( const std::system_error& e ) + { + reporter->FatalErrorWithCore("cannot lock mutex: %s", e.what()); + // Never gets here. + throw std::exception(); + } } template @@ -136,50 +142,28 @@ inline Queue::Queue(BasicThread* arg_reader, BasicThread* arg_writer) num_reads = num_writes = 0; reader = arg_reader; writer = arg_writer; - - for( int i = 0; i < NUM_QUEUES; ++i ) - { - if ( pthread_cond_init(&has_data[i], 0) != 0 ) - reporter->FatalError("cannot init queue condition variable"); - - if ( pthread_mutex_init(&mutex[i], 0) != 0 ) - reporter->FatalError("cannot init queue mutex"); - } } template inline Queue::~Queue() { - for( int i = 0; i < NUM_QUEUES; ++i ) - { - pthread_cond_destroy(&has_data[i]); - pthread_mutex_destroy(&mutex[i]); - } } template inline T Queue::Get() { - safe_lock(&mutex[read_ptr]); + auto lock = acquire_lock(mutex[read_ptr]); int old_read_ptr = read_ptr; if ( messages[read_ptr].empty() && ! ((reader && reader->Killed()) || (writer && writer->Killed())) ) { - struct timespec ts; - ts.tv_sec = time(0) + 5; - ts.tv_nsec = 0; - - pthread_cond_timedwait(&has_data[read_ptr], &mutex[read_ptr], &ts); - safe_unlock(&mutex[read_ptr]); - return 0; + if ( has_data[read_ptr].wait_for(lock, std::chrono::seconds(5)) == std::cv_status::timeout ) + return nullptr; } - else if ( messages[read_ptr].empty() ) - { - safe_unlock(&mutex[read_ptr]); - return 0; - } + if ( messages[read_ptr].empty() ) + return nullptr; T data = messages[read_ptr].front(); messages[read_ptr].pop(); @@ -187,15 +171,13 @@ inline T Queue::Get() read_ptr = (read_ptr + 1) % NUM_QUEUES; ++num_reads; - safe_unlock(&mutex[old_read_ptr]); - return data; } template inline void Queue::Put(T data) { - safe_lock(&mutex[write_ptr]); + auto lock = acquire_lock(mutex[write_ptr]); int old_write_ptr = write_ptr; @@ -203,43 +185,59 @@ inline void Queue::Put(T data) messages[write_ptr].push(data); - if ( need_signal ) - pthread_cond_signal(&has_data[write_ptr]); - write_ptr = (write_ptr + 1) % NUM_QUEUES; ++num_writes; - safe_unlock(&mutex[old_write_ptr]); + if ( need_signal ) + { + lock.unlock(); + has_data[old_write_ptr].notify_one(); + } } template inline bool Queue::Ready() { - safe_lock(&mutex[read_ptr]); + auto lock = acquire_lock(mutex[read_ptr]); bool ret = (messages[read_ptr].size()); - safe_unlock(&mutex[read_ptr]); - return ret; } +template +inline std::vector> Queue::LocksForAllQueues() + { + std::vector> locks; + + try + { + for ( int i = 0; i < NUM_QUEUES; i++ ) + locks.emplace_back(std::unique_lock(mutex[i])); + } + + catch ( const std::system_error& e ) + { + reporter->FatalErrorWithCore("cannot lock all mutexes: %s", e.what()); + // Never gets here. + throw std::exception(); + } + + return locks; + } + template inline uint64_t Queue::Size() { // Need to lock all queues. - for ( int i = 0; i < NUM_QUEUES; i++ ) - safe_lock(&mutex[i]); + auto locks = LocksForAllQueues(); uint64_t size = 0; for ( int i = 0; i < NUM_QUEUES; i++ ) size += messages[i].size(); - for ( int i = 0; i < NUM_QUEUES; i++ ) - safe_unlock(&mutex[i]); - return size; } @@ -248,29 +246,19 @@ inline void Queue::GetStats(Stats* stats) { // To be safe, we look all queues. That's probably unneccessary, but // doesn't really hurt. - for ( int i = 0; i < NUM_QUEUES; i++ ) - safe_lock(&mutex[i]); + auto locks = LocksForAllQueues(); stats->num_reads = num_reads; stats->num_writes = num_writes; - - for ( int i = 0; i < NUM_QUEUES; i++ ) - safe_unlock(&mutex[i]); } template inline void Queue::WakeUp() { for ( int i = 0; i < NUM_QUEUES; i++ ) - { - safe_lock(&mutex[i]); - pthread_cond_signal(&has_data[i]); - safe_unlock(&mutex[i]); - } + has_data[i].notify_all(); } } - #endif - diff --git a/testing/btest/Baseline/broker.remote_log/recv.recv.out b/testing/btest/Baseline/broker.remote_log/recv.recv.out index 2f4a31df51..e69de29bb2 100644 --- a/testing/btest/Baseline/broker.remote_log/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_log/recv.recv.out @@ -1,6 +0,0 @@ -wrote log, [msg=ping, nolog=no, num=0] -wrote log, [msg=ping, nolog=no, num=1] -wrote log, [msg=ping, nolog=no, num=2] -wrote log, [msg=ping, nolog=no, num=3] -wrote log, [msg=ping, nolog=no, num=4] -wrote log, [msg=ping, nolog=no, num=5] diff --git a/testing/btest/Baseline/broker.remote_log/recv.test.log b/testing/btest/Baseline/broker.remote_log/recv.test.log index 0d6dae756c..b79e1b53b9 100644 --- a/testing/btest/Baseline/broker.remote_log/recv.test.log +++ b/testing/btest/Baseline/broker.remote_log/recv.test.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path test -#open 2015-01-26-22-47-11 +#open 2017-02-27-16-21-20 #fields msg num #types string count ping 0 @@ -12,4 +12,4 @@ ping 2 ping 3 ping 4 ping 5 -#close 2015-01-26-22-47-11 +#close 2017-02-27-16-21-20 diff --git a/testing/btest/Baseline/broker.remote_log/send.test.log b/testing/btest/Baseline/broker.remote_log/send.test.log index 0d6dae756c..93862c656b 100644 --- a/testing/btest/Baseline/broker.remote_log/send.test.log +++ b/testing/btest/Baseline/broker.remote_log/send.test.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path test -#open 2015-01-26-22-47-11 +#open 2017-02-27-16-21-19 #fields msg num #types string count ping 0 @@ -12,4 +12,4 @@ ping 2 ping 3 ping 4 ping 5 -#close 2015-01-26-22-47-11 +#close 2017-02-27-16-21-20 diff --git a/src/analyzer/protocol/ayiya/events.bif b/testing/btest/Baseline/broker.remote_log_types/recv.recv.out similarity index 100% rename from src/analyzer/protocol/ayiya/events.bif rename to testing/btest/Baseline/broker.remote_log_types/recv.recv.out diff --git a/testing/btest/Baseline/broker.remote_log_types/recv.test.log b/testing/btest/Baseline/broker.remote_log_types/recv.test.log new file mode 100644 index 0000000000..eb2b066cd4 --- /dev/null +++ b/testing/btest/Baseline/broker.remote_log_types/recv.test.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2017-02-11-02-17-35 +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string set[count] set[string] set[string] vector[count] vector[string] func +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1486779455.703438 100.000000 hurz 1 AA (empty) 10,20,30 (empty) foo\x0a{ \x0aif (0 < i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +#close 2017-02-11-02-17-35 diff --git a/testing/btest/Baseline/broker.remote_log_types/send.send.out b/testing/btest/Baseline/broker.remote_log_types/send.send.out new file mode 100644 index 0000000000..632279e697 --- /dev/null +++ b/testing/btest/Baseline/broker.remote_log_types/send.send.out @@ -0,0 +1 @@ +Broker::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/Baseline/broker.remote_log_types/send.test.log b/testing/btest/Baseline/broker.remote_log_types/send.test.log new file mode 100644 index 0000000000..59987c5998 --- /dev/null +++ b/testing/btest/Baseline/broker.remote_log_types/send.test.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2017-02-11-02-17-35 +#fields b i e c p sn a d t iv s sc ss se vc ve f +#types bool int enum count port subnet addr double time interval string set[count] set[string] set[string] vector[count] vector[string] func +T -42 Test::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1486779455.703438 100.000000 hurz 1 AA (empty) 10,20,30 (empty) foo\x0a{ \x0aif (0 < i) \x0a\x09return (Foo);\x0aelse\x0a\x09return (Bar);\x0a\x0a} +#close 2017-02-11-02-17-36 diff --git a/testing/btest/Baseline/core.erspan/tunnel.log b/testing/btest/Baseline/core.erspan/tunnel.log new file mode 100644 index 0000000000..76d2784a7a --- /dev/null +++ b/testing/btest/Baseline/core.erspan/tunnel.log @@ -0,0 +1,10 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path tunnel +#open 2017-02-03-20-27-11 +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action +#types time string addr port addr port enum enum +1442309933.472798 CHhAvVGS1DHFjwGM9 10.200.0.3 0 10.200.0.224 0 Tunnel::GRE Tunnel::DISCOVER +#close 2017-02-03-20-27-11 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index 6587112ef2..d53b14ce58 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-11-02-17-25-26 +#open 2017-02-28-17-15-30 #fields name #types string scripts/base/init-bare.bro @@ -58,7 +58,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/top-k.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro - build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro build/scripts/base/bif/plugins/Bro_BackDoor.events.bif.bro build/scripts/base/bif/plugins/Bro_BitTorrent.events.bif.bro build/scripts/base/bif/plugins/Bro_ConnSize.events.bif.bro @@ -74,7 +73,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Gnutella.events.bif.bro - build/scripts/base/bif/plugins/Bro_GSSAPI.types.bif.bro build/scripts/base/bif/plugins/Bro_GSSAPI.events.bif.bro build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro build/scripts/base/bif/plugins/Bro_HTTP.events.bif.bro @@ -96,7 +94,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_NTLM.types.bif.bro build/scripts/base/bif/plugins/Bro_NTLM.events.bif.bro build/scripts/base/bif/plugins/Bro_NTP.events.bif.bro - build/scripts/base/bif/plugins/Bro_PIA.events.bif.bro build/scripts/base/bif/plugins/Bro_POP3.events.bif.bro build/scripts/base/bif/plugins/Bro_RADIUS.events.bif.bro build/scripts/base/bif/plugins/Bro_RDP.events.bif.bro @@ -150,7 +147,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_XMPP.events.bif.bro - build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro @@ -171,4 +167,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2016-11-02-17-25-26 +#close 2017-02-28-17-15-30 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 7a7b127752..e11edefe16 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2016-11-02-17-25-18 +#open 2017-02-28-17-19-41 #fields name #types string scripts/base/init-bare.bro @@ -58,7 +58,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/top-k.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro - build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro build/scripts/base/bif/plugins/Bro_BackDoor.events.bif.bro build/scripts/base/bif/plugins/Bro_BitTorrent.events.bif.bro build/scripts/base/bif/plugins/Bro_ConnSize.events.bif.bro @@ -74,7 +73,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_FTP.events.bif.bro build/scripts/base/bif/plugins/Bro_FTP.functions.bif.bro build/scripts/base/bif/plugins/Bro_Gnutella.events.bif.bro - build/scripts/base/bif/plugins/Bro_GSSAPI.types.bif.bro build/scripts/base/bif/plugins/Bro_GSSAPI.events.bif.bro build/scripts/base/bif/plugins/Bro_GTPv1.events.bif.bro build/scripts/base/bif/plugins/Bro_HTTP.events.bif.bro @@ -96,7 +94,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_NTLM.types.bif.bro build/scripts/base/bif/plugins/Bro_NTLM.events.bif.bro build/scripts/base/bif/plugins/Bro_NTP.events.bif.bro - build/scripts/base/bif/plugins/Bro_PIA.events.bif.bro build/scripts/base/bif/plugins/Bro_POP3.events.bif.bro build/scripts/base/bif/plugins/Bro_RADIUS.events.bif.bro build/scripts/base/bif/plugins/Bro_RDP.events.bif.bro @@ -150,7 +147,6 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_Teredo.events.bif.bro build/scripts/base/bif/plugins/Bro_UDP.events.bif.bro build/scripts/base/bif/plugins/Bro_XMPP.events.bif.bro - build/scripts/base/bif/plugins/Bro_ZIP.events.bif.bro build/scripts/base/bif/plugins/Bro_FileEntropy.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.events.bif.bro build/scripts/base/bif/plugins/Bro_FileExtract.functions.bif.bro @@ -359,4 +355,4 @@ scripts/base/init-default.bro scripts/base/misc/find-filtered-trace.bro scripts/base/misc/version.bro scripts/policy/misc/loaded-scripts.bro -#close 2016-11-02-17-25-18 +#close 2017-02-28-17-19-41 diff --git a/testing/btest/Baseline/language.uninitialized-local2/out b/testing/btest/Baseline/language.uninitialized-local2/out new file mode 100644 index 0000000000..75d09294e6 --- /dev/null +++ b/testing/btest/Baseline/language.uninitialized-local2/out @@ -0,0 +1,2 @@ +error in /home/jon/projects/bro/bro/testing/btest/.tmp/language.uninitialized-local2/uninitialized-local2.bro, line 19: value used but not set (var_b) +var_a is, baz diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 9b22c34b71..420d20ae12 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -247,7 +247,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1485327769.512366, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1488302456.440387, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, , (Conn::LOG)) -> @@ -377,7 +377,7 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -> -0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1485327769.512366, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, , (PacketFilter::LOG, [ts=1488302456.440387, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(NetControl::check_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(NetControl::init, , ()) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, , ()) -> @@ -416,7 +416,6 @@ 0.000000 MetaHookPost LoadFile(../main) -> -1 0.000000 MetaHookPost LoadFile(../plugin) -> -1 0.000000 MetaHookPost LoadFile(./Bro_ARP.events.bif.bro) -> -1 -0.000000 MetaHookPost LoadFile(./Bro_AYIYA.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_AsciiReader.ascii.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_AsciiWriter.ascii.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_BackDoor.events.bif.bro) -> -1 @@ -440,7 +439,6 @@ 0.000000 MetaHookPost LoadFile(./Bro_FileHash.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_Finger.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_GSSAPI.events.bif.bro) -> -1 -0.000000 MetaHookPost LoadFile(./Bro_GSSAPI.types.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_GTPv1.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_Gnutella.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_HTTP.events.bif.bro) -> -1 @@ -465,7 +463,6 @@ 0.000000 MetaHookPost LoadFile(./Bro_NetBIOS.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_NoneWriter.none.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_PE.events.bif.bro) -> -1 -0.000000 MetaHookPost LoadFile(./Bro_PIA.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_POP3.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_RADIUS.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_RDP.events.bif.bro) -> -1 @@ -528,7 +525,6 @@ 0.000000 MetaHookPost LoadFile(./Bro_X509.functions.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_X509.types.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./Bro_XMPP.events.bif.bro) -> -1 -0.000000 MetaHookPost LoadFile(./Bro_ZIP.events.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./acld) -> -1 0.000000 MetaHookPost LoadFile(./addrs) -> -1 0.000000 MetaHookPost LoadFile(./analyzer.bif.bro) -> -1 @@ -968,7 +964,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1485327769.512366, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, , (PacketFilter::LOG, [ts=1488302456.440387, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, , (Conn::LOG)) @@ -1098,7 +1094,7 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, , (Weird::LOG, [columns=, ev=Weird::log_weird, path=weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (X509::LOG, [columns=, ev=X509::log_x509, path=x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, , (mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql])) -0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1485327769.512366, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, , (PacketFilter::LOG, [ts=1488302456.440387, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(NetControl::check_plugins, , ()) 0.000000 MetaHookPre CallFunction(NetControl::init, , ()) 0.000000 MetaHookPre CallFunction(Notice::want_pp, , ()) @@ -1137,7 +1133,6 @@ 0.000000 MetaHookPre LoadFile(../main) 0.000000 MetaHookPre LoadFile(../plugin) 0.000000 MetaHookPre LoadFile(./Bro_ARP.events.bif.bro) -0.000000 MetaHookPre LoadFile(./Bro_AYIYA.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_AsciiReader.ascii.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_AsciiWriter.ascii.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_BackDoor.events.bif.bro) @@ -1161,7 +1156,6 @@ 0.000000 MetaHookPre LoadFile(./Bro_FileHash.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_Finger.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_GSSAPI.events.bif.bro) -0.000000 MetaHookPre LoadFile(./Bro_GSSAPI.types.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_GTPv1.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_Gnutella.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_HTTP.events.bif.bro) @@ -1186,7 +1180,6 @@ 0.000000 MetaHookPre LoadFile(./Bro_NetBIOS.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_NoneWriter.none.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_PE.events.bif.bro) -0.000000 MetaHookPre LoadFile(./Bro_PIA.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_POP3.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_RADIUS.events.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_RDP.events.bif.bro) @@ -1249,7 +1242,6 @@ 0.000000 MetaHookPre LoadFile(./Bro_X509.functions.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_X509.types.bif.bro) 0.000000 MetaHookPre LoadFile(./Bro_XMPP.events.bif.bro) -0.000000 MetaHookPre LoadFile(./Bro_ZIP.events.bif.bro) 0.000000 MetaHookPre LoadFile(./acld) 0.000000 MetaHookPre LoadFile(./addrs) 0.000000 MetaHookPre LoadFile(./analyzer.bif.bro) @@ -1688,7 +1680,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1485327769.512366, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1488302456.440387, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1818,7 +1810,7 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird, path=weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509, path=x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql, path=mysql]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1485327769.512366, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1488302456.440387, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction NetControl::check_plugins() 0.000000 | HookCallFunction NetControl::init() 0.000000 | HookCallFunction Notice::want_pp() @@ -2297,6 +2289,7 @@ 1362692527.080972 MetaHookPost CallFunction(Conn::determine_service, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> 1362692527.080972 MetaHookPost CallFunction(Conn::set_conn, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> 1362692527.080972 MetaHookPost CallFunction(HTTP::get_file_handle, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) -> +1362692527.080972 MetaHookPost CallFunction(KRB::do_log, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> 1362692527.080972 MetaHookPost CallFunction(KRB::fill_in_subjects, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) -> 1362692527.080972 MetaHookPost CallFunction(Log::__write, , (Conn::LOG, [ts=1362692526.869344, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], proto=tcp, service=http, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={}])) -> 1362692527.080972 MetaHookPost CallFunction(Log::write, , (Conn::LOG, [ts=1362692526.869344, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], proto=tcp, service=http, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={}])) -> @@ -2327,6 +2320,7 @@ 1362692527.080972 MetaHookPre CallFunction(Conn::determine_service, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) 1362692527.080972 MetaHookPre CallFunction(Conn::set_conn, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) 1362692527.080972 MetaHookPre CallFunction(HTTP::get_file_handle, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T)) +1362692527.080972 MetaHookPre CallFunction(KRB::do_log, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) 1362692527.080972 MetaHookPre CallFunction(KRB::fill_in_subjects, , ([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=])) 1362692527.080972 MetaHookPre CallFunction(Log::__write, , (Conn::LOG, [ts=1362692526.869344, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], proto=tcp, service=http, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={}])) 1362692527.080972 MetaHookPre CallFunction(Log::write, , (Conn::LOG, [ts=1362692526.869344, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], proto=tcp, service=http, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={}])) @@ -2358,6 +2352,7 @@ 1362692527.080972 | HookCallFunction Conn::determine_service([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=]) 1362692527.080972 | HookCallFunction Conn::set_conn([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) 1362692527.080972 | HookCallFunction HTTP::get_file_handle([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=], T) +1362692527.080972 | HookCallFunction KRB::do_log([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=]) 1362692527.080972 | HookCallFunction KRB::fill_in_subjects([id=[orig_h=141.142.228.5, orig_p=59856<...>/plain], current_entity=, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={}, current_request=1, current_response=1, trans_depth=1], irc=, krb=, modbus=, mysql=, ntlm=, radius=, rdp=, rfb=, sip=, sip_state=, snmp=, smtp=, smtp_state=, socks=, ssh=, syslog=]) 1362692527.080972 | HookCallFunction Log::__write(Conn::LOG, [ts=1362692526.869344, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], proto=tcp, service=http, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={}]) 1362692527.080972 | HookCallFunction Log::write(Conn::LOG, [ts=1362692526.869344, uid=CHhAvVGS1DHFjwGM9, id=[orig_h=141.142.228.5, orig_p=59856<...>/tcp], proto=tcp, service=http, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=, local_resp=, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={}]) diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out b/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out new file mode 100644 index 0000000000..3406639d29 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.invalid-lines/out @@ -0,0 +1,26 @@ +{ +[-43] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242 HOHOHO, sc={ +2, +4, +1, +3 +}, ss={ +BB, +AA, +CC +}, se={ + +}, vc=[10, 20, 30], ve=[]], +[-42] = [b=T, e=SSH::LOG, c=21, p=123/unknown, sn=10.0.0.0/24, a=1.2.3.4, d=3.14, t=1315801931.273616, iv=100.0, s=hurz, ns=4242, sc={ +2, +4, +1, +3 +}, ss={ +BB, +AA, +CC +}, se={ + +}, vc=[10, 20, 30], ve=[]] +} diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.missing-enum/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.input.missing-enum/bro..stderr index 20207bcf94..8cd0c5ab6c 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.input.missing-enum/bro..stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.input.missing-enum/bro..stderr @@ -1,2 +1,2 @@ -warning: Value not 'IdoNot::Exist' for stream 'enum' is not a valid enum. +warning: Value 'IdoNot::Exist' for stream 'enum' is not a valid enum. received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/bro..stderr b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/bro..stderr new file mode 100644 index 0000000000..337cdcda87 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/bro..stderr @@ -0,0 +1,8 @@ +warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat +warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat +warning: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat +error: ../does-not-exist.dat/Input::READER_ASCII: Init: cannot open ../does-not-exist.dat +error: ../does-not-exist.dat/Input::READER_ASCII: Init failed +error: ../does-not-exist.dat/Input::READER_ASCII: terminating thread +warning: ../does-not-exist.dat/Input::READER_ASCII: Could not get stat for ../does-not-exist.dat +received termination signal diff --git a/testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/bro..stdout b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/bro..stdout new file mode 100644 index 0000000000..39309bc8cf --- /dev/null +++ b/testing/btest/Baseline/scripts.base.frameworks.input.missing-file-initially/bro..stdout @@ -0,0 +1,5 @@ +now it does +and more! +now it does +and more! +Streaming still works diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output index dfa922f88f..a01d119b81 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.expire-item/output @@ -3,20 +3,31 @@ #empty_field (empty) #unset_field - #path intel -#open 2016-06-15-19-11-06 +#open 2017-02-09-18-29-44 #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p seen.indicator seen.indicator_type seen.where seen.node matched sources fuid file_mime_type file_desc #types time string addr port addr port string enum enum string set[enum] set[string] string string string -1466017866.348490 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - -1466017867.349583 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - -1466017868.349656 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - -#close 2016-06-15-19-11-12 +1486664984.510411 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - +1486664987.510937 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1 - - - +1486664990.511265 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1,source2 - - - +1486664993.512024 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1,source2 - - - +1486664996.512265 - - - - - 1.2.3.4 Intel::ADDR SOMEWHERE bro Intel::ADDR source1,source2 - - - +#close 2017-02-09-18-30-03 +-- Run 1 -- Trigger: 1.2.3.4 Seen: 1.2.3.4 +-- Run 2 -- +Trigger: 1.2.3.4 +Reinsert: 1.2.3.4 +Seen: 1.2.3.4 +-- Run 3 -- Trigger: 1.2.3.4 Seen: 1.2.3.4 +-- Run 4 -- +Trigger: 1.2.3.4 +Seen: 1.2.3.4 +-- Run 5 -- Trigger: 1.2.3.4 Seen: 1.2.3.4 Expired: 1.2.3.4 -Trigger: 1.2.3.4 -Trigger: 1.2.3.4 +-- Run 6 -- Trigger: 1.2.3.4 diff --git a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output index 9cb4a7c9ff..69606c1407 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output +++ b/testing/btest/Baseline/scripts.base.frameworks.intel.remove-non-existing/output @@ -3,9 +3,9 @@ #empty_field (empty) #unset_field - #path reporter -#open 2016-09-20-22-35-58 +#open 2017-02-11-16-36-40 #fields ts level message location #types time enum string string -0.000000 Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). /home/jgras/devel/bro/scripts/base/frameworks/intel/./main.bro, lines 507-508 +0.000000 Reporter::INFO Tried to remove non-existing item '192.168.1.1' (Intel::ADDR). /home/johanna/bro/master/scripts/base/frameworks/intel/./main.bro, lines 520-521 0.000000 Reporter::INFO received termination signal (empty) -#close 2016-09-20-22-35-59 +#close 2017-02-11-16-36-40 diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log deleted file mode 100644 index 974349e229..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol.log +++ /dev/null @@ -1,23 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path netcontrol -#open 2016-08-12-17-38-49 -#fields ts rule_id category cmd state action target entity_type entity mod msg priority expire location plugin -#types time string enum string enum string enum string string string string int interval string string -1471023529.752740 - NetControl::MESSAGE - - - - - - - activating plugin with priority 0 - - - Debug-All -1471023529.752740 - NetControl::MESSAGE - - - - - - - activation finished - - - Debug-All -1471023529.752740 - NetControl::MESSAGE - - - - - - - plugin initialization done - - - - -1471023532.819263 2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 connection drop worker-1 Debug-All -1471023532.819263 worker-1:2 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All -1471023532.819263 2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 connection drop worker-1 Debug-All -1471023532.819263 worker-1:2 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All -1471023532.920126 worker-1:2 NetControl::RULE EXPIRE NetControl::TIMEOUT NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All -1471023532.920126 worker-1:2 NetControl::RULE REMOVE NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All -1471023532.921768 worker-1:2 NetControl::RULE REMOVE NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 0.100000 direct drop worker-1 Debug-All -1471023534.308087 2 NetControl::RULE REMOVE NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - worker-2 0 600.000000 connection drop worker-1 Debug-All -1471023534.308087 2 NetControl::RULE REMOVE NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 192.168.18.50/32 - - 0 600.000000 connection drop worker-1 Debug-All -1471023534.308087 4 NetControl::RULE ADD NetControl::REQUESTED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 3600.000000 Re-drop by catch-and-release: direct cr worker-1 Debug-All -1471023534.308087 4 NetControl::RULE ADD NetControl::SUCCEEDED NetControl::DROP NetControl::FORWARD NetControl::ADDRESS 8.8.8.8/32 - - 0 3600.000000 Re-drop by catch-and-release: direct cr worker-1 Debug-All -#close 2016-08-12-17-38-54 diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log deleted file mode 100644 index 54202fffbe..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/manager-1.netcontrol_catch_release.log +++ /dev/null @@ -1,18 +0,0 @@ -#separator \x09 -#set_separator , -#empty_field (empty) -#unset_field - -#path netcontrol_catch_release -#open 2016-08-12-17-38-52 -#fields ts rule_id ip action block_interval watch_interval blocked_until watched_until num_blocked location message -#types time string addr enum interval interval time time count string string -1471023532.819263 2 192.168.18.50 NetControl::DROP 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 - -1471023532.819263 2 192.168.18.50 NetControl::DROPPED 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 - -1471023532.819263 worker-1:2 8.8.8.8 NetControl::ADDED 600.000000 3600.000000 - 1471027132.819263 1 direct cr worker-1 Address already blocked outside of catch-and-release. Catch and release will monitor and only actively block if it appears in network traffic. -1471023532.920126 worker-1:2 8.8.8.8 NetControl::UNBLOCK 600.000000 3600.000000 - 1471027132.819263 1 direct cr worker-1 - -1471023534.308087 2 192.168.18.50 NetControl::INFO 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 Block seen while in rule_entities. No action taken. -1471023534.308087 2 192.168.18.50 NetControl::UNBLOCK 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 worker-2 -1471023534.308087 4 8.8.8.8 NetControl::SEEN_AGAIN 3600.000000 86400.000000 1471027134.308087 1471109934.308087 2 direct cr worker-1 - -1471023534.308087 4 8.8.8.8 NetControl::DROPPED 3600.000000 86400.000000 1471027134.308087 1471109934.308087 2 direct cr worker-1 - -1471023532.239980 2 192.168.18.50 NetControl::INFO 600.000000 3600.000000 1471024132.819263 1471027132.819263 1 connection drop worker-1 Already blocked using catch-and-release - ignoring duplicate -#close 2016-08-12-17-38-54 diff --git a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout b/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout deleted file mode 100644 index f61fe92474..0000000000 --- a/testing/btest/Baseline/scripts.base.frameworks.netcontrol.catch-and-release-cluster/worker-2..stdout +++ /dev/null @@ -1,9 +0,0 @@ -Suspend, worker-2 -New block, 192.168.18.50, [block_until=1471027194.791177, watch_until=1471030194.791177, num_reblocked=0, current_interval=0, current_block_id=2, location=connection drop worker-1] -New block, 8.8.8.8, [block_until=, watch_until=1471030194.791177, num_reblocked=0, current_interval=0, current_block_id=worker-1:2, location=direct cr worker-1] -Resume, worker-2 -Connection established -Info, [block_until=1471027194.791177, watch_until=1471030194.791177, num_reblocked=0, current_interval=0, current_block_id=2, location=connection drop worker-1] -Delete block, 192.168.18.50 -New block, 8.8.8.8, [block_until=1471030196.295249, watch_until=1471112996.295249, num_reblocked=1, current_interval=1, current_block_id=4, location=direct cr worker-1] -remote connection closed diff --git a/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output b/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output index 0bec7ee13c..c42038b5d0 100644 --- a/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output +++ b/testing/btest/Baseline/scripts.base.protocols.krb.kinit/output @@ -1,3 +1,3 @@ KRB_AP_REQUEST -[pvno=5, realm=VLADG.NET, service_name=krbtgt/VLADG.NET, cipher=18] +[pvno=5, realm=VLADG.NET, service_name=krbtgt/VLADG.NET, cipher=18, ciphertext={\x9fY\xd0f\x8dS\xf4I\x88\x04\xfa\xc1\xd8m\xa2\xb7+\xbb\x19\xcag\x0c\x13\xd1g*\xfc\x18\xd1\xb1\x80!\xbd\x85\xec\xf9\x9b\xfa-\x18\xb6\xf5h\x91\xe7\x99\xf4\xdb\x93\xa0\xc7\x90\x1e\xa9\x95v\xd3\x12\xfa,9\x1d\x0b\xd0\xa1\xd25\x0f\x1f[G\xdf\xd0\xbbd\x06$2\xd1\xae\x130qZiY\x07@\xe9\xf9\xff\xa4\x9a\xd4\x09\xf0\x0d\xc1R\x10M\xbdKOV\xfd\xf6\x13\xf6\x9a\x95N\xdf!\xf6x\x94\xd8j\xa5\xdcp\xa8\x04\x99\x02x\xdb$\xd8\xfa_o\x8dV\xc8\x0a\xfe\x00\xf3&c\x0c8\xd1\xd0\xe9\x8e\xab\xfe&\xfe\x00\x8d$\x98I\xe5\x8d\x94rM4%\xd8\xfe\xa9\x08\x06\xc6\x95H7\xf7HCq\xb9\x0d$\x95?\x83B\x82\xdd\xea\xc3f3\xcc\xbb\x09\x0d-\x09;\xa6i%\xcd\xba\x11\xd4\xe0\x12w\xd0G&\xdaj\x82\x7f;\xf3\x1d\x10\xa4l\x06\x16l\x1bc\xa1\xd1\x15!\x00\x8a\xff\x8a\x06\xe7U^:recv.out" +# @TEST-EXEC: btest-bg-run send "bro -b ../common.bro ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff recv/test.log +# @TEST-EXEC: btest-diff send/send.out +# @TEST-EXEC: btest-diff send/test.log + +@TEST-START-FILE common.bro + + +global quit_receiver: event(); +global quit_sender: event(); + + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + b: bool; + i: int; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of count; + ve: vector of string; + f: function(i: count) : string; + } &log; + +} + +event bro_init() &priority=5 + { + Broker::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info]); + } + +@TEST-END-FILE + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Broker::subscribe_to_logs("bro/log/"); + Broker::subscribe_to_events("bro/event/"); + Broker::listen(broker_port, "127.0.0.1"); + } + +event quit_receiver() + { + terminate(); + } + +@TEST-END-FILE + + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Broker::enable_remote_logs(Test::LOG); + Broker::publish_topic("bro/event/"); + Broker::connect("127.0.0.1", broker_port, 1secs); + } + +event quit_sender() + { + terminate(); + } + +function foo(i : count) : string + { + if ( i > 0 ) + return "Foo"; + else + return "Bar"; + } + +event Broker::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Broker::outgoing_connection_established", peer_address, peer_port; + + local empty_set: set[string]; + local empty_vector: vector of string; + + Log::write(Test::LOG, [ + $b=T, + $i=-42, + $e=Test::LOG, + $c=21, + $p=123/tcp, + $sn=10.0.0.1/24, + $a=1.2.3.4, + $d=3.14, + $t=network_time(), + $iv=100secs, + $s="hurz", + $sc=set(1), # set(1,2,3,4), # Output not stable for multi-element sets. + $ss=set("AA"), # set("AA", "BB", "CC") # Output not stable for multi-element sets. + $se=empty_set, + $vc=vector(10, 20, 30), + $ve=empty_vector, + $f=foo + ]); + + local args = Broker::event_args(quit_receiver); + Broker::send_event("bro/event/", args); + schedule 1sec { quit_sender() }; + } + +@TEST-END-FILE diff --git a/testing/btest/core/erspan.bro b/testing/btest/core/erspan.bro new file mode 100644 index 0000000000..eb05cdcf5a --- /dev/null +++ b/testing/btest/core/erspan.bro @@ -0,0 +1,4 @@ +# @TEST-EXEC: bro -C -b -r $TRACES/erspan.trace %INPUT +# @TEST-EXEC: btest-diff tunnel.log + +@load base/frameworks/tunnels diff --git a/testing/btest/istate/pybroccoli.py b/testing/btest/istate/pybroccoli.py index 7600c2b7d4..0d7106d592 100644 --- a/testing/btest/istate/pybroccoli.py +++ b/testing/btest/istate/pybroccoli.py @@ -4,6 +4,7 @@ # @TEST-REQUIRES: test -e $BUILD/aux/broccoli/bindings/broccoli-python/_broccoli_intern.so # # @TEST-EXEC: btest-bg-run bro bro %INPUT $DIST/aux/broccoli/bindings/broccoli-python/tests/test.bro +# @TEST-EXEC: sleep 2 # @TEST-EXEC: btest-bg-run python PYTHONPATH=$DIST/aux/broccoli/bindings/broccoli-python/:$BUILD/aux/broccoli/bindings/broccoli-python python $DIST/aux/broccoli/bindings/broccoli-python/tests/test.py # @TEST-EXEC: btest-bg-wait -k 20 # @TEST-EXEC: btest-diff bro/.stdout diff --git a/testing/btest/language/uninitialized-local2.bro b/testing/btest/language/uninitialized-local2.bro new file mode 100644 index 0000000000..f11a5fda10 --- /dev/null +++ b/testing/btest/language/uninitialized-local2.bro @@ -0,0 +1,25 @@ +# @TEST-EXEC: bro -b %INPUT >out 2>&1 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + +event test() + { + local var_a: string = "foo"; + } + +event test() + { + if ( F ) + { + local var_b: string = "bar"; + } + + local var_a: string = "baz"; + + print "var_a is", var_a; + print "var_b is", var_b; + } + +event bro_init() + { + event test(); + } diff --git a/testing/btest/scripts/base/frameworks/input/invalid-lines.bro b/testing/btest/scripts/base/frameworks/input/invalid-lines.bro new file mode 100644 index 0000000000..83be1efd09 --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/invalid-lines.bro @@ -0,0 +1,67 @@ +# @TEST-EXEC: btest-bg-run bro bro -b %INPUT +# @TEST-EXEC: btest-bg-wait 10 +# @TEST-EXEC: btest-diff out + +redef exit_only_after_terminate = T; +redef InputAscii::fail_on_invalid_lines = F; + +@TEST-START-FILE input.log +#separator \x09 +#path ssh +#fields b i e c p sn a d t iv s sc ss se vc ve ns +#types bool int enum count port subnet addr double time interval string table table table vector vector string +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 +T -42 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 +T -43 SSH::LOG 21 123 10.0.0.0/24 1.2.3.4 3.14 1315801931.273616 100.000000 hurz 2,4,1,3 CC,AA,BB EMPTY 10,20,30 EMPTY 4242 HOHOHO +T -41 +@TEST-END-FILE + +@load base/protocols/ssh + +global outfile: file; + +redef InputAscii::empty_field = "EMPTY"; + +module A; + +type Idx: record { + i: int; +}; + +type Val: record { + b: bool; + e: Log::ID; + c: count; + p: port; + sn: subnet; + a: addr; + d: double; + t: time; + iv: interval; + s: string; + ns: string; + sc: set[count]; + ss: set[string]; + se: set[string]; + vc: vector of int; + ve: vector of int; +}; + +global servers: table[int] of Val = table(); +global servers2: table[int] of Val = table(); + +event bro_init() + { + outfile = open("../out"); + # first read in the old stuff into the table... + Input::add_table([$source="../input.log", $name="ssh", $idx=Idx, $val=Val, $destination=servers]); + Input::add_table([$source="../input.log", $name="ssh2", $idx=Idx, $val=Val, $destination=servers2, $config=table(["fail_on_invalid_lines"] = "T")]); + } + +event Input::end_of_data(name: string, source:string) + { + print outfile, servers; + Input::remove("ssh"); + close(outfile); + terminate(); + } diff --git a/testing/btest/scripts/base/frameworks/input/invalidtext.bro b/testing/btest/scripts/base/frameworks/input/invalidtext.bro index 1de4e96671..3f5b590dec 100644 --- a/testing/btest/scripts/base/frameworks/input/invalidtext.bro +++ b/testing/btest/scripts/base/frameworks/input/invalidtext.bro @@ -13,6 +13,7 @@ @TEST-END-FILE redef exit_only_after_terminate = T; +redef InputAscii::fail_on_invalid_lines = T; global outfile: file; diff --git a/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro b/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro new file mode 100644 index 0000000000..73fd57284e --- /dev/null +++ b/testing/btest/scripts/base/frameworks/input/missing-file-initially.bro @@ -0,0 +1,49 @@ +# This tests files that don't exist initially and then do later during +# runtime to make sure the ascii reader is resilient to files missing. +# It does a second test at the same time which configures the old +# failing behavior. + +# @TEST-EXEC: btest-bg-run bro bro %INPUT +# @TEST-EXEC: sleep 2; cp does-exist.dat does-not-exist.dat +# @TEST-EXEC: sleep 2; mv does-not-exist.dat does-not-exist-again.dat; echo "Streaming still works" >> does-not-exist-again.dat +# @TEST-EXEC: btest-bg-wait -k 3 +# @TEST-EXEC: btest-diff bro/.stdout +# @TEST-EXEC: btest-diff bro/.stderr + +@TEST-START-FILE does-exist.dat +#separator \x09 +#fields line +#types string +now it does +and more! +@TEST-END-FILE + +redef exit_only_after_terminate = T; + +@load base/frameworks/input + +module A; + +type Val: record { + line: string; +}; + +event line(description: Input::EventDescription, tpe: Input::Event, v: Val) + { + print v$line; + } + +event line2(description: Input::EventDescription, tpe: Input::Event, v: Val) + { + print "DONT PRINT THIS LINE"; + } + + +event bro_init() + { + Input::add_event([$source="../does-not-exist.dat", $name="input", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line, $want_record=T]); + Input::add_event([$source="../does-not-exist.dat", $name="inputstream", $reader=Input::READER_ASCII, $mode=Input::STREAM, $fields=Val, $ev=line, $want_record=T]); + Input::add_event([$source="../does-not-exist.dat", $name="inputmanual", $reader=Input::READER_ASCII, $mode=Input::MANUAL, $fields=Val, $ev=line, $want_record=T]); + Input::add_event([$source="../does-not-exist.dat", $name="input2", $reader=Input::READER_ASCII, $mode=Input::REREAD, $fields=Val, $ev=line2, $want_record=T, + $config=table(["fail_on_file_problem"] = "T")]); + } diff --git a/testing/btest/scripts/base/frameworks/input/missing-file.bro b/testing/btest/scripts/base/frameworks/input/missing-file.bro index 08adfe2150..2ec3bb937f 100644 --- a/testing/btest/scripts/base/frameworks/input/missing-file.bro +++ b/testing/btest/scripts/base/frameworks/input/missing-file.bro @@ -3,6 +3,7 @@ # @TEST-EXEC: btest-diff bro/.stderr redef exit_only_after_terminate = T; +redef InputAscii::fail_on_file_problem = T; global outfile: file; global try: count; diff --git a/testing/btest/scripts/base/frameworks/intel/expire-item.bro b/testing/btest/scripts/base/frameworks/intel/expire-item.bro index df9170b669..dd915b3a03 100644 --- a/testing/btest/scripts/base/frameworks/intel/expire-item.bro +++ b/testing/btest/scripts/base/frameworks/intel/expire-item.bro @@ -20,11 +20,28 @@ redef table_expire_interval = 3sec; global runs = 0; event do_it() { + ++runs; + print fmt("-- Run %s --", runs); + print "Trigger: 1.2.3.4"; Intel::seen([$host=1.2.3.4, $where=SOMEWHERE]); - ++runs; + if ( runs == 2 ) + { + # Reinserting the indicator should reset the expiration + print "Reinsert: 1.2.3.4"; + local item = [ + $indicator="1.2.3.4", + $indicator_type=Intel::ADDR, + $meta=[ + $source="source2", + $desc="this host is still bad", + $url="http://some-data-distributor.com/2"] + ]; + Intel::insert(item); + } + if ( runs < 6 ) schedule 3sec { do_it() }; } diff --git a/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro b/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro deleted file mode 100644 index fd7de7e442..0000000000 --- a/testing/btest/scripts/base/frameworks/netcontrol/catch-and-release-cluster.bro +++ /dev/null @@ -1,132 +0,0 @@ -# @TEST-SERIALIZE: comm -# -# @TEST-EXEC: btest-bg-run manager-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=manager-1 bro %INPUT" -# @TEST-EXEC: sleep 1 -# @TEST-EXEC: btest-bg-run worker-1 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-1 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT" -# @TEST-EXEC: btest-bg-run worker-2 "cp ../cluster-layout.bro . && CLUSTER_NODE=worker-2 bro --pseudo-realtime -C -r $TRACES/tls/ecdhe.pcap %INPUT" -# @TEST-EXEC: btest-bg-wait 20 -# @TEST-EXEC: TEST_DIFF_CANONIFIER='grep -v ^# | $SCRIPTS/diff-remove-timestamps' btest-diff manager-1/netcontrol.log -# @TEST-EXEC: btest-diff manager-1/netcontrol_catch_release.log -# @TEST-EXEC: btest-diff worker-2/.stdout - -@TEST-START-FILE cluster-layout.bro -redef Cluster::nodes = { - ["manager-1"] = [$node_type=Cluster::MANAGER, $ip=127.0.0.1, $p=37757/tcp, $workers=set("worker-1", "worker-2")], - ["worker-1"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37760/tcp, $manager="manager-1", $interface="eth0"], - ["worker-2"] = [$node_type=Cluster::WORKER, $ip=127.0.0.1, $p=37761/tcp, $manager="manager-1", $interface="eth0"], -}; -@TEST-END-FILE - -redef exit_only_after_terminate = T; - -redef Log::default_rotation_interval = 0secs; - -@load base/frameworks/netcontrol -redef NetControl::catch_release_warn_blocked_ip_encountered = T; - -global ready_for_data_1: event(); -global ready_for_data_2: event(); -redef Cluster::manager2worker_events += /^ready_for_data_(1|2)$/; - -@if ( Cluster::local_node_type() == Cluster::MANAGER ) - -global peer_count = 0; -event remote_connection_handshake_done(p: event_peer) &priority=-5 - { - ++peer_count; - print "remote_connection_handshake_done", peer_count; - if ( peer_count == 2 ) - { - event ready_for_data_1(); - schedule 1.5sec { ready_for_data_2() }; - } - } - -@endif - -@if ( Cluster::local_node_type() == Cluster::WORKER ) -event bro_init() - { - print "Suspend", Cluster::node; - suspend_processing(); - } - -event remote_connection_closed(p: event_peer) { - print "remote connection closed"; - terminate(); -} -@endif - -@if ( Cluster::node == "worker-1" ) -event ready_for_data_1() - { - print "Resume", Cluster::node; - continue_processing(); - } -@endif - -@if ( Cluster::node == "worker-2" ) -event ready_for_data_2() - { - print "Resume", Cluster::node; - continue_processing(); - } -@endif - -event NetControl::init() - { - local netcontrol_debug = NetControl::create_debug(T); - NetControl::activate(netcontrol_debug, 0); - } - -global i: count = 0; - -event connection_established(c: connection) - { - print "Connection established"; - local id = c$id; - local info = NetControl::get_catch_release_info(id$orig_h); - print "Info", info; - NetControl::drop_address_catch_release(id$orig_h, cat("connection drop ", Cluster::node)); - if ( info$current_block_id != "" ) - { - NetControl::unblock_address_catch_release(id$orig_h, Cluster::node); - } - } - -@if ( Cluster::node == "worker-1" ) -event connection_established(c: connection) - { - NetControl::drop_address(8.8.8.8, 0.1secs, cat("direct drop ", Cluster::node)); - NetControl::drop_address_catch_release(8.8.8.8, cat("direct cr ", Cluster::node)); - } -@endif - -@if ( Cluster::node == "worker-2" ) -event connection_established(c: connection) - { - NetControl::catch_release_seen(8.8.8.8); - } -@endif - -event NetControl::catch_release_block_new(a: addr, b: NetControl::BlockInfo) - { - print "New block", a, b; - } - -event NetControl::catch_release_block_delete(a: addr) - { - print "Delete block", a; - } - -event terminate_me() { - terminate(); -} - -@if ( Cluster::local_node_type() == Cluster::MANAGER ) -event NetControl::rule_added(r: NetControl::Rule, p: NetControl::PluginState, msg: string) - { - print "Scheduling terminate"; - schedule 3sec { terminate_me() }; - } -@endif diff --git a/testing/btest/scripts/base/protocols/radius/radius-multiple-attempts.test b/testing/btest/scripts/base/protocols/radius/radius-multiple-attempts.test new file mode 100644 index 0000000000..473e492355 --- /dev/null +++ b/testing/btest/scripts/base/protocols/radius/radius-multiple-attempts.test @@ -0,0 +1,6 @@ +# Test a more complicated radius session with multiple attempts + +# @TEST-EXEC: bro -b -C -r $TRACES/radius/radius_localhost.pcapng %INPUT +# @TEST-EXEC: btest-diff radius.log + +@load base/protocols/radius diff --git a/testing/btest/scripts/policy/protocols/krb/ticket-logging.bro b/testing/btest/scripts/policy/protocols/krb/ticket-logging.bro new file mode 100644 index 0000000000..0bc0a33d5d --- /dev/null +++ b/testing/btest/scripts/policy/protocols/krb/ticket-logging.bro @@ -0,0 +1,6 @@ +# This test makes sure that krb ticket hashes are logged correctly. + +# @TEST-EXEC: bro -b -r $TRACES/krb/auth.trace %INPUT +# @TEST-EXEC: btest-diff kerberos.log + +@load protocols/krb/ticket-logging