diff --git a/.gitmodules b/.gitmodules index afac9bfc6b..87826d2ef6 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,24 +1,24 @@ [submodule "aux/bro-aux"] path = aux/bro-aux - url = ../bro-aux + url = git://git.bro.org/bro-aux [submodule "aux/binpac"] path = aux/binpac - url = ../binpac + url = git://git.bro.org/binpac [submodule "aux/broccoli"] path = aux/broccoli - url = ../broccoli + url = git://git.bro.org/broccoli [submodule "aux/broctl"] path = aux/broctl - url = ../broctl + url = git://git.bro.org/broctl [submodule "aux/btest"] path = aux/btest - url = ../btest + url = git://git.bro.org/btest [submodule "cmake"] path = cmake - url = ../cmake + url = git://git.bro.org/cmake [submodule "magic"] path = magic - url = ../bromagic + url = git://git.bro.org/bromagic [submodule "src/3rdparty"] path = src/3rdparty - url = ../bro-3rdparty + url = git://git.bro.org/bro-3rdparty diff --git a/CHANGES b/CHANGES index a50370c236..00cd9b9176 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,21 @@ +2.1-1387 | 2013-09-23 11:54:48 -0700 + + * Change submodules to fixed URL. (Jon Siwek) + + * Updating NEWS. (Robin Sommer) + + * Fixing an always false condition. (Robin Sommer) + + * Fix required for compiling with clang 3.3. (Robin Sommer) + +2.1-1377 | 2013-09-20 14:38:15 -0700 + + * Updates to the scripting introduction. (Scott Runnels) + + * Kill raw input reader's child by process group to reliably clean + it up. (Jon Siwek) + 2.1-1368 | 2013-09-19 20:07:57 -0700 * Add more links in the GeoLocation document (Daniel Thayer) @@ -18750,3138 +18767,3 @@ name: string, data: string) smb_com_read_andx(c: connection, is_orig: bool, data: string) smb_com_write_andx(c: connection, is_orig: bool, data: string) - - This analyzer is still experimental. - -- Greater support for vectors (Umesh Shankar), much of it taken from - the 'S' language. - - You can use a boolean vector as an index into another vector (providing - both are the same length) and each 'T' value in the index extracts the - corresponding element from the indexed vector. For example, "x[x > 3]" - returns a vector whose elements are those elements of x that are greater - than, while if y is a vector of the same length as x then "y[x > 3]" - extracts those elements of y that have the same position as the elements - in x that are greater than 3. - - You can also use an arithmetic vector to index another vector. Each - element present in the index is extracted. So, for example: - - global a: vector of count; - global b: vector of string; - - a[1] = 3; - a[2] = 3; - a[3] = 1; - - b[1] = "foo"; - b[2] = "bar"; - b[3] = "bletch"; - - print b[a]; - - prints: - - [bletch, bletch, foo] - -- The new built-ins any_set() and all_set() return true if for a given - boolean vector any element is true or all of the elements is true - (Umesh Shankar). So, for example, "any_set(x < 0)" returns T if - an element of x is less than zero. - -- The new built-in sort() takes a vector as an argument and sorts it - *in place* (Umesh Shankar). (The in-place semantics may change in the - future.) An optional second argument can be used to specify a - function to call for comparing elements, and is required for non-arithmetic - vectors. For example, the following could be used to sort a vector - of strings based solely on the length of the strings: - - function string_compare(a : string, b: string): int - { - local la = byte_len(a); - local lb = byte_len(b); - - return (la < lb) ? -1 : ((lb > la) ? 1 : 0); - } - -- The new function order() has the same calling sequence as sort(), - but instead of returning (and altering in place) the sorted vector, - it returns a "vector of count" giving the *indices* that if used - to index the vector will return it sorted. So, for example, - given two vectors x and y of the same length (but not necessarily - of the same type), - - local x_sort_indices = order(x); - x = x[x_sort_indices]; - y = y[x_sort_indices]; - - will assign x to a sorted version of itself and also rearrange y such - that elements of y that were paired with elements of x originally - remain paired after the sorting. - -- The ICMP analyzer now has a general notion of "context", i.e., the packet - associated with ICMP status messages such as Unreachable or Time Exceeded - (Ruoming Pang). This changes the parameters to the icmp_unreachable - event. A new event, icmp_time_exceed, is now also available. - -- The tcp_segment even has been replaced by a pair of new events (Ruoming Pang): - - tcp_packet(c: connection, is_orig: bool, flags: string, seq: count, - ack: count, len: count, payload: string) - - is invoked for each TCP packet. "flags" is a string containing "SFAPU" - for the SYN/FIN/etc TCP flags. - - tcp_contents(c: connection, is_orig: bool, seq: count, - contents: string) - - is invoked for each chunk of the byte-stream that has been reassembled - in sequence, providing it satisfies tcp_content_delivery_ports_{orig,resp}, - per the next item. - -- You can specify the set of ports for which contents should be reassembled - for the originator (responder, respectively) stream using the new sets - tcp_content_delivery_ports_{orig,resp} (Ruoming Pang). This can be - useful for user-level stream analysis for protocols not known to Bro's - event engine. These controls may change to a "table of bool" in the future, - in order to support an &default attribute. - -- New built-in functions (Ruoming Pang): - - function interval_to_double(i: interval): double - Converts a value of type "interval" to "double". - - function write_file(f: file, data: string): bool - Writes the given string to the given file, returning - F on error. - - function is_ascii(str: string): bool - Returns T if the given string consists entirely of - ASCII characters (i.e., in the range 0..127). - - function sqrt(x: double): double - Returns the square-root of x, or -1 and a run-time error - if x is < 0. - - function uuid_to_string(uuid: string): string - Takes a UUID and returns its string representation, where - UUID = Universal Unique Identifier as defined per - http://www.opengroup.org/onlinepubs/9629399/apdxa.htm#tagcjh_20 - - function string_to_ascii_hex(s: string): string - Returns the ASCII hex representation of the given string. - For example, string_to_ascii_hex("foo") returns "666f6f". - - function match_pattern(s: string, p:pattern): pattern_match_result - Matches the given pattern against the given string, returning - a record with three fields: - - matched: bool; # T if a match was found, F otherwise - str: string; # portion of string that first matched - off: count; # 1-based offset where match starts - - For example, - match_pattern("foobar", /o*[a-k]/) - returns - [matched=T, str=f, off=1] - because the *first* match is for zero o's followed by an [a-k], - while - match_pattern("foobar", /o+[a-k]/) - returns - [matched=T, str=oob, off=2] - -- Functions that terminate without returning a value when they were declared - to do so now generate a run-time warning (Christian Kreibich). Functions - in the standard set of policy scripts that did this have been fixed. - -- The new event non_dns_request(c: connection, msg: string) is generate - to make the contents of malformed DNS requests available for analysis, - with the assumption that these are actually some other protocol entirely - (Ruoming Pang). - -- If you redef truncate_http_URI to have a value >= 0, then any HTTP - URIs generated by the event engine will be truncated to this length - (Ruoming Pang). This can be convenient when analyzing traffic that - generates huge URIs (as do some automated attacks). - -- "SEARCH" is now recognized as a standard HTTP method (Ruoming Pang). - -- The new event connection_EOF(c: connection, is_orig: bool) is generated - when one side of a connection closes (Ruoming Pang). - -- synflood.bro and the corresponding event engine internals now works - in terms of probabilities (0.0-1.0) instead of percentages (0-100) - (Robin Sommer). The script has had several tweaks, including using - new_connection() rather than connection_attempt(), which gives it - quicker response and broader coverage (it'll detect non-TCP flooding, - too, so "synflood" is now a bit of a misnomer), at the cost of perhaps - more CPU load. - -- A signature for Witty has been added to policy/sigs/worm.sig (Ruoming Pang). - -- Makefile now has a "test" target. Currently this only works for internal - development (we haven't put together a public test suite yet; that will - take some time due to the need to make sure no sensitive information leaks). - -- The built-in function generator now knows about "double" as a built-in type - (Ruoming Pang). - -- Some generated files have been removed from the Bro distribution since - they're redundant (Ruoming Pang). - -- A bug has been fixed in which contents files might not be correctly - written upon termination of Bro (Ruoming Pang). - -- A bug has been fixed in which UDP connections didn't generate - new_connection events (Ruoming Pang). - -- Support for the Linux "cooked capture" link layer (Ruoming Pang). - -- BPF support has been factored into a separate class, BPF_Program, which - makes for easier portability (Christian Kreibich). - -- A serious low-level Dictionary bug has been fixed (Christian Kreibich). - -- A bug that could cause Bro to crash if it receives an event from another - Bro that it isn't able to process has been fixed (Christian Kreibich). - -- A bug in set file descriptors non-blocking has been fixed - (Christian Kreibich). - -- A bug that could cause some error messages to generate crashes has - been fixed. - -- The global skip_event_tcp_option has been removed. - - -0.9a1 Mon Jun 7 01:33:00 PDT 2004 - -- 0.8a85 is the new STABLE release, to be updated only for bug (and - portability) fixes. 0.9a1 is the new CURRENT release, which will have - both bug fixes and new functionality. - -- Support for FTP EPRT, EPSV (Holger Dreger). - -- Change to timer management to recover memory more quickly (Robin Sommer). - -- Tweaks to eliminate a number of compiler warnings (Robin Sommer). - -- Statistics now report number of connections terminated due - to inactivity (Robin Sommer). - -- New Makefile target, pub-tar-no-doc builds a public tarball but - without the (large) doc/ subdirectory. - -- Bug fix for identd requests with illegal port numbers. - -- The example of a scanning source to skip in scan.bro has been changed - from an AltaVista robot (now obsolete) to a Google robot. - -- Some previously fatal internal errors have now been turned into - "internal warnings", which Bro is able to continue operating - in their presence. - - -0.8a84 Wed May 26 23:33:39 PDT 2004 - -- autoconf tweaks for portability to Darwin (Christian Kreibich). - -- Fixed subtle bug in chunked-IO reads (Christian Kreibich). - -- Fixed bug for the "discarder" framework in which specifying an - IP discarder would cause other TCP/UDP/ICMP discarders not being - invoked (Christian Kreibich). - -- Fatal bug in signature matching fixed (Robin Sommer). - -- Missing member variable initialization fixed (Robin Sommer). (Needed for - compilation with new versions of gcc.) - -- Makefile bug for "make install" fixed. - -- Fixed bug that could lead to Bro crashing if an SMTP session - had data sent after a RST. - -- Removed some out-of-date SMTP analysis warning messages. - - -0.8a82 Tue Apr 27 11:53:24 PDT 2004 - -- Fixed inactivity timer loop when a packet arrives exactly when - the timer is set to expire. - - -0.8a81 Mon Apr 26 22:46:37 PDT 2004 - -- A bunch of memory leaks fixed (Chema Gonzalez). - -- A new HTTP analyzer variable, content_truncation_limit, controls - how much of an HTTP request/reply contents element (i.e., what's - passed to http_entity_data - this is *not* the entire content, but - the next chunk's worth) is logged in the log file. It defaults - to 40 bytes. Setting it to 0 means "log all of it". - -- Fix to avoid crashing for malformed RPC requests. - -- Improved OpenSSL auto-configuration (Robin Sommer). - -- Fix for compiling without OpenSSL. - -- A new built-in, double_to_count(), converts a value of type "double" - to the corresponding "count" (Chema Gonzalez). We should probably - add floor(), ceil(), etc. - -- Parameterization of trw.bro tweaked (Jaeyeon Jung). - - -0.8a79 Wed Mar 24 22:02:53 PST 2004 - -- Bro now has an SSL analyzer, written by Michael Kuhn and Benedikt Ostermaier, - with further development by Scott Campbell. It generates the following - events: - - event process_X509_extensions(c: connection, ex: X509_extension) - event ssl_X509_error(c: connection, err: int, err_string: string) - event ssl_certificate(c: connection, cert: X509, is_server: bool) - event ssl_certificate_seen(c: connection, is_server: bool) - event ssl_conn_alert(c: connection, version: count, level: count, - description: count) - event ssl_conn_attempt(c: connection, version: int) - event ssl_conn_established(c: connection, version: int, - cipher_suite: count) - event ssl_conn_reused(c: connection, session_id: sessionID) - event ssl_conn_server_reply(c: connection, version: int) - event ssl_conn_weak(name: string, c: connection) - event ssl_session_insertion(c: connection, id: sessionID) - - Note, it still has a lot of rough edges; particularly, handling - non-conformant input. It also generates unnecessary ContentGap alerts - due to the way it runs multiple analyzers (SSLv2 and SSLv3) on a single - connection. This will be fixed in the fairly near-term future. - -- The manual has been updated with chapters on signatures (Robin Sommer) - and using the interactive debugger (Umesh Shankar), along with a partial - description of the new SSL analyzer (Michael Kuhn and Benedikt Ostermaier) - and a number of updates to the documentation of built-in functions (Umesh - Shankar), though this latter is still not complete since Umesh actually - contributed this quite a while ago. - -- Ruoming Pang has contributed a crude analyzer for DCE/RPC (used for Windows). - It generates simple dce_rpc_request and dce_rpc_reply events. It should - not be considered stable. - -- The traditional connection logging format (traditional_conn_format) - is no longer the default. The 0.8a70 release notes stated that this - was the case but this time it really is :-). - -- An experimental "vector" type has been added (Umesh Shankar). A vector - is an aggregate type. For example: - - local suspect_hosts: vector of addr; - - You can randomly access elements starting with the first as 1, e.g., - - suspect_hosts[1] = 10.0.0.8; - - and can also add elements at later postions even if there are gaps: - - suspect_hosts[31337] = 10.0.0.9; - - *The semantics and capabilities of vectors will be changing considerably.* - -- Umesh Shankar has developed a framework for generating IDMEF messages. - Currently it needs a modified version of libidmef, which is not included - in this distribution. Contact me or Umesh if you want a copy. - -- A new attribute &synchronized causes the given global variable to - be *synchronized* across concurrent instances of Bro (which are - intercommunicating via remote.bro). Any change made by one of them - to the variable will be reflected (soon after) in the copy at - the others. A new event remote_state_inconsistency is generated - if two instances both change the value before they're synchronized. - (Contributed by Robin Sommer.) - -- trw.bro implements a new scan detection algorithm, Threshold Random Walk - (Jaeyeon Jung). It's described in an upcoming IEEE S&P symposium paper. - The analyzer generates two events: - - TRWAddressScan, # source flagged as scanner by TRW algorithm - TRWScanSummary, # summary of scanning activities reported by TRW - - TRW is generally much more sensitive than Bro's regular detection algorithm. - -- vlan.bro provides support for VLAN encapsulation. More generally, Bro - now has support for any sort of constant-offset encapsulation (Vinod - Yegneswaran). You specify the header size by redef'ing encap_hdr_size. - You can also redef tunnel_port to be a UDP port which Bro treats as being - the encapsulation (in the packet stream - not addressed to it) rather - than all traffic. - -- If you turn on statistics (i.e., via @load statistics) and also redef - segment_profiling to T, then Bro will generate to the statistics file - a trace of its "segment" processing. A segment is a unit of internal - execution. Profiles look like: - - 1058517499.615430 segment-processing-packet dt=0.000013 dmem=0 - 1058517499.615430 segment-draining-events dt=0.000012 dmem=0 - 1058517499.615671 segment-expiring-timers dt=0.000010 dmem=0 - 1058517499.615671 segment-processing-packet dt=0.000010 dmem=0 - 1058517499.615671 segment-draining-events dt=0.000012 dmem=0 - 1058517499.615671 segment-policy/conn.bro:282 dt=0.000011 dmem=0 - 1058517499.615671 segment-policy/conn.bro:253 dt=0.000012 dmem=0 - - The first line states that at the given (packet) timestamp, the event - engine processed a packet, taking 13 usec of CPU time to do so, and - not consuming any memory (from the kernel's perspective; this is *not* - fine-grained memory consumption). The next lines indicate 12 usec were - spent draining events and 10 usec expiring timers. The last two lines - indicate that the functions at lines 282 and 253 in conn.bro were - executed, requiring 11 usec and 12 usec, respectively. - - Note #1: timings are just what rusage() reports, so not necessarily - very accurate for small times. - - Note #2: there's a bug in tracking function line numbers that hasn't - been ferreted out yet, so they're only roughly correct. - -- The inactivity_timeout global has been split into tcp_inactivity_timeout/ - udp_inactivity_timeout/icmp_inactivity_timeout (Robin Sommer). Using - this, the default inactivity timeouts for UDP and ICMP have been changed - from "no timeout" to 10 seconds. This is needed because otherwise - analyzing a stream of UDP or ICMP traffic generally gobbles up memory - quickly and never recovers it; and there's seems little point in trying - to consolidate long-lived-but-often-inactive UDP/ICMP streams. - -- The new policy script cpu-adapt.bro is an extension to load-levels.bro - (see CHANGES for 0.8a37 below) to adapt the packet filter based on the - current CPU load. If the load is below cpu_lower_limit (default 40%), - the load-level is decreased. If it's above cpu_upper_limit (default - 90%), it's increased. (Robin Sommer) - -- The new policy script hand-over.bro can be used for a new running - instance of Bro to smoothly take over operation from an old instance, - i.e., it implements hand-over of state between two Bro instances when - checkpointing (Robin Sommer). First, all persistent data (i.e. variables - declared &persistent and connections for which make_connection_persistent() - has been called) is transferred from the old instance to the new instance. - Then the old instance terminates itself and the new one starts processing. - - The host from which we want to take over the state has to be added to - remote_peers_clear (or remote_peers_ssl), setting hand_over to T. The - host which is allowed to perform a hand-over with us has to be added - with a port of 0/tcp and hand_over=T. An example for a handover between - two instances on the same machine: - - @load hand-over - redef remote_peers_clear += { - [127.0.0.1, 47756/tcp] = [$hand_over = T], - [127.0.0.1, 0/tcp] = [$hand_over = T] - }; - - (This interface may be subject to change in the future.) - -- New script functions (Robin Sommer): - - function terminate() - Terminates Bro via SIGTERM. - - function dump_config() - Dumps Bro's full configuration into state_dir (one file per - variable/type/function, etc.) - - function send_state(ip: addr, p: port) - Send all of persistent state to the remote host. - - function set_accept_state(ip: addr, p: port, accept: bool) - If accept is true, state sent by the given host will be - accepted (default: false) - - function make_connection_persistent(c: connection) - Declare the given connection state to be persistent (i.e. - to be saved upon termination and exchanged by send_state). - checkpoint.bro uses this to declare some services to be - persistent by default. - - function is_local_interface(ip: addr): bool - Returns true if the given address is assigned to a local interface. - -- Printing of sets and tables now includes timestamps indicating when the - element was added (Robin Sommer): - - ID did_ssh_version = { - [129.187.20.9, F] = 1 @11/01-15:55, - [212.144.77.26, T] = 2 @11/01-15:55, - [141.84.116.26, T] = 10 @11/01-15:55, - [217.232.245.249, T] = 1 @11/01-15:55, - [217.235.217.149, T] = 1 @11/01-15:55, - [129.187.39.13, F] = 2 @11/01-15:55, - [129.187.208.139, F] = 1 @11/01-15:55, - } - - The format may change in the future, and will probably be made an option. - -- Similarly, you can print functions to get both a timestamp of the last - time the given block was executed and a count of how often (Robin Sommer): - - ID record_connection = record_connection - (@11/01-16:03 #6549) - { - id = c$id; - local_init = is_local_addr(id$orig_h); - local_addr = local_init ? id$orig_h : id$resp_h; - remote_addr = local_init ? id$resp_h : id$orig_h; - flags = local_init ? "L" : ""; - if (remote_addr in neighbor_nets) - (@ #0) - flags = cat(flags, "U"); - - if ("" == flags) - (@11/01-16:03 #2110) - flags = "X"; - - is_tcp = is_tcp_port(id$orig_p); - ; - if (is_tcp) - (@11/01-16:03 #6549) - { - if (c$orig$state in conn_closed || c$resp$state in conn_closed -) - (@11/01-16:03 #4739) - duration = fmt("%.6g", c$duration); - else - (@11/01-16:03 #1810) - duration = "?"; - [...] - -- You can now specify numbers using hex constants, e.g., 0xabcd = 43981 - (Michael Kuhn and Benedikt Ostermaier). - -- A new function, join_string_array(sep: string, a: string_array) concatenates - strings in 'a' and inserts 'sep' between every two adjacent elements - (Ruoming Pang). E.g., join_string_array("", {"a", "b", "c"}) returns - "a b c", and join_string_array("", a) is the same as cat_string_array(a). - -- checkpoint.bro now makes some services persistent by default - (Robin Sommer). - -- The new_packet event now includes both the associated connection - and a pkt_hdr describing the packet (Robin Sommer). - -- The functions functions connect_ssl() and connect_clear() have been replaced - by a single connect() function taking an additional parameter to - differentiate the types (Robin Sommer). - -- The new function stop_listening() unbinds the listen port (Robin Sommer). - -- A new flag packet_filter_default says whether the Bro-level packet-filter - will by default accept all or reject everything (Robin Sommer). - -- Function calls can now be sent to remote Bro's, though there isn't yet - an interface for accessing this from the script level (Robin Sommer). - -- Bro now has an generalized internal framework for serializing objects - and monitoring access to state (Robin Sommer). - -- Better memory allocation accounting (Robin Sommer). - -- A minor tweak to the output generated by statistics.bro. - -- Improved localization of source code for functions in messages (but - there are still some bug). - -- Autoconf looks for -ltermcap (Robin Sommer). - -- Fixes for bugs in the management of table expiration values (Chema Gonzalez). - -- A bug in printing "void" values has been fixed (Chema Gonzalez). - -- -t bug fixed (Chema Gonzalez). - -- A bug has been fixed in which sometimes "expression value ignored" - was erroneously generated. - -- A bug with packet_contents and UDP packets with checksum errors - has been fixed (Ruoming Pang). - -- A memory leak in packet timestamp sorting via packet_sort_window - has been fixed (Ruoming Pang). - -- A bug has been fixed in expanding NULs when printing strings (Ruoming Pang). - -- Bug fixes for extracting connection contents via contents.bro (Ruoming Pang). - -- Bogus error message "Can't install default pcap filter" when using -F - removed. - - -0.8a70 Sun Feb 8 14:19:45 PST 2004 - -- Bro has a new home page at - - http://www-nrg.ee.lbl.gov/bro.html - - It includes a "wish list" of Bro development projects: - - http://www-nrg.ee.lbl.gov/bro-wishlist.html - -- The "match" expression has been completely overhauled (Umesh Shankar). - It now has the syntax: - - match EXPR1 using EXPR2 - - Its semantics are complicated, but it's very powerful (see its use for - alert filtering below). EXPR1 can have any type T. EXPR2 must be of - type "set[R]", where R is a record type. R must have the following fields: - - $pred - type is "function(T): bool". This is the predicate - associated with the record. It is passed in EXPR1's - value and returns true or false. - - $result - can have any type T'. This is the value to use when - if $pred returns true for EXPR1. - - $priority - type must be arithmetic (count, int, double). This - is the priority associated with the match of EXPR1 - if $pred returns true. - - The way the expression works is that EXPR1 is evaluated yielding a - value V. EXPR2 is then evaluated yielding a set of records whose - type includes the above fields. Bro then spins through each of the - records in the set and tests whether its $pred predicate holds for V. - If so, it records the given $result and the associated $priority. - It then returns for the value of the entire expression the $result - with the highest $priority. - - Here's an example. The following script: - - global match_stuff = { - [$pred = function(a: count): bool { return a > 5; }, - $result = "it's big", - $priority = 2], - - [$pred = function(a: count): bool { return a > 15; }, - $result = "it's really big", - $priority = 3], - - [$pred = function(a: count): bool { return T; }, - $result = "default", - $priority = 0], - }; - - print match 0 using match_stuff; - print match 10 using match_stuff; - print match 20 using match_stuff; - - when executed will print: - - default - it's big - it's really big - - (Note that Bro actually will first evalute $priority before evaluating - $pred, and if it already has a better (higher) priority result, it - will not bother calling $pred.) - -- There's a new syntax for designating function values (Umesh Shankar). - It currently only works when associating a function value with a - record field in a record constructor: - - [$foo(bad_guy: addr) = { launch_counter_DDOS(bad_guy); return 3.14; }] - - is equivalent to: - - [$foo = function(bad_guy: addr): double = { - launch_counter_DDOS(bad_guy); - return 3.14; - }] - - The goal is to make such functions easier on the eye to express. - The changes are (1) no "function" keywork necessary, (2) no function - return type necessary (note, it is inferred from the "return" statement - in the function body; eventually this will work for all functions, not - just those in record constructors), (3) the '=' sign comes after the ')' - rather than before the keyword "function". - - Given this syntax, we can rewrite the initialization of match_stuff - in the example above as: - - global match_stuff = { - [$pred(a: count) = { return a > 5; }, - $result = "it's big", - $priority = 2], - - [$pred(a: count) = { return a > 15; }, - $result = "it's really big", - $priority = 3], - - [$pred(a: count) = { return T; }, - $result = "default", - $priority = 0], - }; - -- The motivation behind these elaborate new mechanisms is to provide a - powerful and streamlined way to filter alerts. According, alert.bro - now processes any alerts generated via ALERT() through a new global, - alert_policy. alert_policy's type is set[alert_policy_item], where - alert_policy_item is: - - type alert_policy_item: record { - result: AlertAction; - pred: function(a: alert_info): bool; - priority: count; - }; - - The idea is that you specify your alert post-filtering by redef'ing - new elements into alert_policy. For example, here are two post-filtering - rules used at LBL to weed out uninteresting alerts: - - # Ignore connections marked as sensitive because they're - # NTP to otherwise-sensitive hosts (first clause) or they happen - # to involve 2766/tcp (Solaris Listen), which happens frequently - # to ftp.ee.lbl.gov if Bro misses the PORT negotiation. - [$pred(a: alert_info) = - { - return a$alert == SensitiveConnection && - (a$conn$id$resp_p == 123/udp || # NTP - a$msg == /Solaris listen service/); - }, - $result = ALERT_FILE, - $priority = 1], - - - # Ignore sensitive URIs if the request was unsuccessful (code 404, - # or not answered.) - [$pred(a: alert_info) = - { - return a$alert == HTTP::HTTP_SensitiveURI && - a$msg == /.*((\(404)|(no reply)).*/; - }, - $result = ALERT_FILE, - $priority = 1], - - These rules are part of: - - redef alert_policy += { - ... these records and others ... - }; - - The idea behind them is to demote certain alerts that would ordinarily - be syslog'd (i.e., the associated action is ALERT_LOG_ALWAYS) to instead - just be recorded in the alert.$BRO_ID file. Naturally, there are - many other possibilities. For example: - - [$pred(a: alert_info) = { - if ( a$alert == FTP::FTP_Sensitive && - a$msg == /.*crown_jewels.*/ ) - { - system("page_the_duty_officer \"crown jewels theft!\""); - return T; - } - else - return F; - }, - $result = ALERT_LOG_ALWAYS, - $priority = 1000], - - would run the program page_the_duty_officer with the argument "crown - jewels theft!" if an FTP_Sensitive alert was generated and the log message - included the text "crown_jewels". More generally, post-filtering needn't - just be about deciding on how the alert is logged; the processing can - run programs, update tables, etc., just like any other function call might. - -- You can use the new function tally_alert_type in an alert_action_filters - initialization in order to suppress immediate logging of an alert and - instead have Bro generate a summary of how many times the given alert - was seen when it exits. You can use another new function, file_alert, - to specify an alert_action_filters initialization that causes the alerts - to just be written to the alert.$BRO_ID file but not otherwise logged. - - For example: - - redef alert_action_filters += { - # Just summarize various packet capture glitches. - [[RetransmissionInconsistency, ContentGap, DroppedPackets, - AckAboveHole]] = - tally_alert_type, - - [RemoteWorm] = file_alert, - }; - - would specify that RetransmissionInconsistency (etc.) alerts should just - be reported in the log file (log.$BRO_ID) as a total count, and - RemoteWorm should only be put in the alert.$BRO_ID file, but not - otherwise logged or counted. - - You could get the same functionality by writing alert_policy records, - but they're quite a bit bulkier than the above. Note that - alert_action_filters entries take precedence over alert_policy - records, but are evaluated *after* the "match" on alert_policy, - so if you have predicates in alert_policy with side-effects (like the - invocation of page_the_duty_officer in the example above), those - will still happen. - -- The alert_info record (which is used in calls to ALERT) now has - slots for some more additional information: - - user: string; # can hold an assocaited username - filename: string; # an associated filename - method: string; # associated HTTP method - URL: string; # associated URL - n: count; # any associated count/number/status code - - (These are all &optional, so you don't need to specify them if they're - not appropriate.) A number of ALERT's in the default policy scripts - have been changed to include these. The intent is to add more such - information in the future. Ideally, alert_policy records shouldn't - be doing checks like "a$msg == /.*((\(404)|(no reply)).*/" but instead - extracting the status code directly from a field of 'a' (which is an - alert_info record). - -- ALERT now fills in the '$id' field in the alert_info record with - the $id of the associated connection, if the caller didn't suppy - a $id but did supply a $conn. Likewise, it will fill in $src with - the $orig_h from $id (after first filling in $id). The net result - is that you can rely on $id and $src being set for any alert that - has an associated connection. - -- The HTTP analysis scripts (policy/http*.bro) have been converted to - use the "module" facility, similar to how ftp.bro was converted for - 0.8a48. This may require changing some of your own scripts, generally - just to add "HTTP::" qualifiers. - -- Now that the variables associated with FTP analysis are part of an - "FTP" module, the "ftp_" prefix associated with: - - ftp_excessive_filename_len - ftp_excessive_filename_trunc_len - ftp_guest_ids - ftp_hot_cmds - ftp_hot_files - ftp_hot_guest_files - ftp_ignore_invalid_PORT - ftp_ignore_privileged_PASVs - ftp_log - ftp_skip_hot - - has been removed, and these are now called: - - excessive_filename_len - excessive_filename_trunc_len - guest_ids - hot_cmds - hot_files - hot_guest_files - ignore_invalid_PORT - ignore_privileged_PASVs - log_file - skip_hot - - To get to them from other scripts, you specify, for example, - - redef FTP::guest_ids = { .... }; - - whereas before you had to use: - - redef FTP::ftp_guest_ids = { .... }; - -- The new connection logging format introduced in 0.8a57 is now the - default, unless you redef the new variable "traditional_conn_format" - to be T (Robin Sommer). Connections using unidentified ephemeral - ports now have a service of simply "other" rather than other-XXXX. - The 'U' connection status flag has been removed (unless you're using - traditional_conn_format). - -- Tables can now be directly indexed by records, and indexing using records - is no longer interchangeable with using a corresponding list of indices - (Umesh Shankar). This may require adjustments to existing policy - scripts. - -- Hostnames such as www.google.com now have type set[addr] rather than - a funky internal list type. - -- The new function dump_current_packet(file_name: string) dumps a copy of - the current packet to the file with the given name, appending it if the - file already exists (Robin Sommer). The file is in tcpdump format. - A handy use for this is in an event handler for signature_match(), - to record packets that match given signatures. - -- The event new_packet() is invoked for each new packet (Robin Sommer). - It currently doesn't provide the packet contents but soon will in - a fashion similar to secondary-filter.bro. - -- "cf -f fmt" specifies a strtime() format. -u specifics UTC time rather - than local time (Mark Delow and Craig Leres). cf now has a man page - (Craig Leres). - -- Two new variables, backdoor_ignore_local and backdoor_ignore_remote, - can be used to specify backdoor signatures that should be ignored - if the server is local/remote. - -- A bug has been fixed in which a "next" executed in the final iteration - of a for loop would mangle the subsequent processing of the outer - statements (Chema Gonzalez). - -- Bug fixes for MIME and Base64 processing (Ruoming Pang). - -- pcap.bro now builds its filter in the opposite order (restrict_filters - first), which can improve packet filtering performance (Robin Sommer). - -- A bug in &default has been fixed. - -- More SSL autoconf tweaks (Robin Sommer). - -- Portability for different pcap_compile_nopcap() calling sequences - (Chema Gonzalez). - -- Some tweaks for a minor reduction in memory consumption. - -- A memory leak for secondary packet filters has been fixed. - -- The localization of error messages (what script line they correspond to) - has been improved. - - -0.8a58 Tue Dec 16 08:55:47 PST 2003 - -- Compatibility with older versions of libpcap (Chema Gonzalez). - - -0.8a57 Tue Dec 9 10:14:30 PST 2003 - -- The format of Bro's connection summaries is changing. The new format - looks like - - 1069437569.904605 0.230644 1.2.3.4 5.6.7.8 http 59377 80 tcp 610 275 S3 L - - That is, , , , , - , , , , - , , . (Robin Sommer) - - The script variable traditional_conn_format=T specifies to use the old - format rather than this new one. This is *currently* the default, but - will change soon to default to F instead. If you have comments on this - new format, we'd like to hear them. - -- The SigAction's available in signatures.bro have been extended (Robin Sommer). - SIG_FILE_BUT_NO_SCAN is like SIG_FILE but without any horizontal/vertical - processing; SIG_LOG_ONCE logs only an alert only the first time it occurs; - SIG_LOG_PER_ORIG logs only the first instance of an alert generated by a - particular originator; SIG_COUNT has been renamed SIG_COUNT_PER_RESP; and - SIG_SUMMARY suppresses logging of individual alerts but generates a - per-originator summary. - -- A new -p option for snort2bro tells it to only process signatures that - include matching on payload (Robin Sommer). - -- You can now explicitly include or exclude particular SIDs when - running snort2bro by specifying a configuration file via -c (Robin - Sommer). The format is simple, just "include" or "ignore" followed - by the SID number: - - # sid-526 BAD TRAFFIC data in TCP SYN packet - ignore 526 - - # sid-623 matches a null-flags stealth scan. Include it even - # if we build with -p, since it doesn't tend to generate any - # false positives. - include 623 - - The new "snort2bro.cfg" file gives examples (i.e., the above). - -- Bro can now serialize functions and event handlers, meaning that these - can be passed as values between Bro's and dumped using -g (Robin Sommer). - One of the main goals in supporting this is to allow in situ alteration - of the Bro's configuration (e.g., you can edit a function and change its - functioning and have a running Bro pick up the change without having to - stop and be restarted). Such dynamic reconfiguration is experimentally - supported via -g (see below). - -- &persistent state is now stored in the *directory* given by state_dir - (default: "./.state"), one file per variable, rather than a single file - (Robin Sommer). - -- Storing &persistent state to disk is now done incrementally: after writing - each file, there's a delay of state_write_delay (default: 0.1 secs) before - working on the next file (Robin Sommer). This may introduce small - inconsistencies, but prevents load spikes that can lead to packet drops. - Currently, there is no mechanism to incrementally store a single variable - (like a large table), although there is already some framework in place - to eventually support this. - -- The *experimental* new -g option dumps the script-level configuration - (excluding things defined in internal default scripts like bro.init) - into the directory . These files may be printed with "bro -x ", - or copied into the state_dir of a running Bro, which will then pick up - the change if it has loaded checkpoint.bro. (When picking up changes, - event handlers are always added, while functions, types, and variables - replace the current ones). - -- Table values are now incrementally expired rather than all at once - (Robin Sommer). That is, if the expiration timer goes off and hundreds - of values should now be expired, the work of doing so is spread over - chunks of table_expire_size (default: 50) at a time, separated by a - delay of table_expire_delay (default: 0.1 secs). This change aims to - prevent large processing spikes that can lead to packet drops. - -- New built-ins sub() and gsub() act like awk's functions of the same - name, changing substrings (either first, or all) that match a given - regular expression to a given target string. (Note, the calling sequence - differs from the order used by awk.) - -- The new auxiliary script aux/scripts/mvlog is a handy way to manage - checkpointed logs. See the script for documentation. - -- The &expire_func function now takes two arguments. The second is - of type "any" and corresponds to the index(es) of the element being - expired. To access the individual indices, you use a new assignment form: - - [a, b, c] = index_val; - - (where index_val is the second argument of type "any"). This assigns - a to the first index, b to the second, and c to the third. NOTE: the - use of "any" types here is *temporary* and will be changing in the - future to a general "tuple" notion. (Robin Sommer) - -- scan.bro and conn.bro have been reworked to consume less memory and to - support more flexible state expiration (Robin Sommer). - -- The new builtin rescan_state() causes Bro to re-read any persistent - data values (Robin Sommer). - -- snort2bro now supports continued lines ("\") (Robin Sommer). - -- The calling sequences of the software_version_found() and - software_parse_error() events has changed, and a new event, - software_unparsed_version_found(), is invoked for raw version - strings (i.e., the version string prior to the event engine - attempting to parse it into version/major/minor) (Robin Sommer). - -- Software version tracking for clients now tracks all versions, not just - the latest version (Robin Sommer). - -- alert_info records now include an optional field event_src, which is the - source of the event if it was received from an external Bro (Robin Sommer). - -- Regular expressions now support {} iteration values of 0, and generate - better error messages. - -- Output generated by icmp.bro is now redirected into an "icmp" log file - (Robin Sommer). - -- autoconf tweaks for configuring OpenSSL on Linux (Ruoming Pang, Robin Sommer). - Tested on RedHat (thanks to Anton Chuvakin), Debian, FreeBSD, Solaris. - -- You can now turn off using OpenSSL even if the OS supports it, via - configuring with --disable-openssl (Robin Sommer). - -- Variable size computations (per global_sizes()) are now more accurate - (Robin Sommer). - -- A bug with combining file encryption and log rotation has been - fixed (Robin Sommer). - -- A problem tracking directionality in signatures fixed (Robin Sommer). - -- Bro now continues running if DNS is not functioning (Robin Sommer). - -- Rewriter memory use has been significantly reduced (Ruoming Pang). - -- Some bugs with -A/-w interaction have been fixed (Ruoming Pang). - - -0.8a48 Tue Oct 21 15:56:13 PDT 2003 - -- There is now a mechanism in place for multiple Bro's to communicate with - one another via sockets (Robin Sommer). *This is quite experimental at - this point* and may have significant bugs and/or need significant - enhancements. - - By loading listen-clear.bro or listen-ssl.bro, an instance of Bro starts - to listen on a TCP port. The first of these listens for unencrypted - connections and the second for connections encrypted via OpenSSL. To - connect to a listening Bro, you load remote-clear.bro or remote-ssl.bro. - For this connection, you specify which events you want to receive by - giving a regular expression (e.g. "/http_*/" for all HTTP events), - although only those events for which you have defined a local handler - are actually requested. Communication is uni-directional in the sense - that for a certain connection the events go only from Bro-A to Bro-B but - not from B to A (you could set up a second connection for the other - direction, though). - - The OpenSSL certificate used to authorize remote Bro's is specified in - the script variable "ssl_ca_certificate" and the private key and certificate - for the local Bro via "ssl_private_key". - - If Bro A connects to Bro B, by default it sends over its capture_filter. - But Bro B uses it only if it has loaded remote-pcap.bro. This is the - beginning of structuring inter-Bro trust mechanisms. It is done via two - new events, remote_capture_filter(ip: addr, p: port, filter: string) and - send_capture_filter(ip: addr, p: port, s: string) : bool. - - The implementation forks a second process which does the socket - communication, so that the main process should not be affected too much. - The two processes communicate via a pipe. - - You can call is_remote_event() to determine whether the event currently - being handled was originated by a remote Bro (if T) or the local Bro - (if F). - - If a connection with a remote Bro terminates (for whatever reason), Bro - may try to reconnect automatically. - - A new function, get_event_source(), returns a record event_source - describing the source that raised the last event. - - See doc/ssl.txt for an explanation of how to create the keys/certificates. - -- A fledgling Gnutella analyzer has been contributed (Mark Allman). - It generates the following events: - - event gnutella_text_msg(c: connection, orig: bool, headers: string) - event gnutella_binary_msg(c: connection, orig: bool, msg_type: count, - ttl: count, hops: count, msg_len: count, - payload: string, payload_len: count, - trunc: bool, complete: bool) - event gnutella_partial_binary_msg(c: connection, orig: bool, - msg: string, len: count) - event gnutella_establish(c: connection) - event gnutella_not_establish(c: connection) - event gnutella_http_notify(c: connection) - -- Bro now supports a secondary channel for acquiring packets (Chema Gonzalez). - You access it by redef'ing the new global "secondary_filters", adding - table[string] of event(filter: string, pkt: pkt_hdr). The string - specifies a tcpdump filter; anytime a packet matches the filter - (including packets that would *not* otherwise be matched by - capture_filter), then the given event handler is invoked. - - For example, - - redef secondary_filters += { - ["tcp[13] & 7 != 0"] = rst_syn_fin_flag, - } - - will invoke rst_syn_fin_flag() anytime a TCP packet is seen for - which the SYN/FIN/RST bits are non-zero. The event handler will - be passed the string "tcp[13] & 7 != 0" (so it can tell which - of possibly multiple filters matched) and a pkt_hdr value, which - includes the IP header and, if present, the TCP, UDP or ICMP header. - - Another example, and what motivated the addition, is: - - redef secondary_filters += { - ["ip[10:2] & 0xffc == 0x398"] = sampled_1_in_1024_packet, - } - - which will invoke sampled_1_in_1024_packet() any time the given - 10 bits in the IP checksum match the pattern 0x398. If the checksum - field is uniformly distributed then this roughly corresponds to - 1-in-1024 random sampling. (Chema has also developed BPF additions - to support true random sampling.) - - See policy/secondary-filter.bro for examples. - -- Bro now does a much better job of keeping track of how much memory - has been allocated for different structures (Robin Sommer). - - This includes more accurate computations for global_size(). - - In addition, if you redef "statistics_interval" to be a non-zero time - interval, then with that periodicity a summary of memory usage (including - memory used by event engine components) is dumped to the file - "statistics_file". In addition, at this point a "do_statistics" event - is generated. You can also call the new built-in statistics_update() - to generate memory statistics on demand. - - The above structure is likely to change in the future. statistics_interval - will probably go away, to be replaced by either explicit calls to - statistics_update() (which you can do on a timer if you like by using - "schedule"), or by a built-in function that returns a record of all - the statistics, that you can then format however you want. - -- A major memory leak in HTTP analysis has been fixed (Ruoming Pang). - -- New attributes &rotate = and - &postprocessor = can be associated with a file - variable in order to specify how often the file should be rotated to a - new filename on disk, and, when rotation occurs, the name of a shell - program to run on the now-older version as a postprocessor (Robin Sommer). - -- Similarly, log_postprocessor and log_rotate_interval specify the default - values for files. Unless redef'd, these themselves default to the empty - string (= no postprocessing) and 0.0 seconds (no rotation). (Robin Sommer) - -- A new attribute, &encrypt, applies to variables of "file" type. It specifies - that the version on disk should be encrypted, using either the key specified - as the value of the attribute, or, if no value is specified, using the - value of the new script variable log_encryption_key. The key is an OpenSSL - public key; it's used to then embed a Blowfish session key. (Robin Sommer) - - A new utility, aux/bdcat/bdcat ("Bro decrypt cat") can be used to decrypt - the files. - -- The internal structure of TCP analysis has been significantly altered. - Previously, TCP_Endpoint tracked endpoint state and TCP_EndpointContents - (derived from it) did stream reassembly. These have now been separated; - TCP_Endpoint still tracks endpoint state, but TCP_EndointContents has - become TCP_Contents, and is only loosely coupled with TCP_Endpoint. - The reason for this change is to lay the groundwork for (1) applying - an application analyzer to a connection after several packets for - the connection have already been read, and (2) applying *multiple* - application analyzers to a single connection. - -- Bro now supports the universal hashing if USE_UHASH is defined - (Ruoming Pang). Universal hashing is a lighter-weight alternative - to MD5 hashing that retains the property of being very difficult - for an attacker to guess. It comes in two flavors, a 32-bit - version (which you get with USE_UHASH) and a faster 16-bit version - (which you get if you also define USE_NH). Bro will likely switch - to using these by default in the near future, as their performance - gain over MD5 is significant. - -- New built-in functions srand() and rand() provide access to the - usual C-style random number seeding & generation (Chema Gonzalez). - -- You can now specify server/client addresses to leave in the clear in - IP address anonymization (via the built-in variables preserve_orig_addr - and preserve_resp_addr). Correspondingly, the built-in function for - IP anonymization now takes a parameter to specify the type of the address - (orig, resp, or other), instead of the method of anonymization - (seq, random, prefix-preserving). (Ruoming Pang) - -- Trace anonymization now has prelminary support for handling TCP options - via the new event "tcp_option" (Ruoming Pang). It is only invoked - if skip_event_tcp_option is F (it defaults to T). - -- A new event, tcp_segment, is similar to the event "packet_content" - but provides more information: is_orig (directionality), sequence - number, and acknowledgement number (Ruoming Pang). - -- ./configure finds OpenSSL if it's in some standard location. Otherwise, - you may specify it --with-openssl=. If OpenSSL is not available, - Bro compiles cleanly without and gives warnings if a script tries use SSL. - (Robin Sommer) - -- The internal links in manual/entire.html have been fixed so they - now work (Chema Gonzalez). - -- A new policy script, blaster.bro, detects instances of the W32.Blaster - worm (Robin Sommer). - -- Signature files (for input to the signature engine) now reside in - policy/sigs/*.sig. This directory is now on the default $BROPATH. - -- sig.ex.ssl-worm.bro and sig.ex.web-rules.bro have been updated - to reflect changes in keywords (Robin Sommer). They've been - renamed to ex.ssl-worm.sig and ex.web-rules.sig and reside - in policy/sigs/, per the above. - -- The module facility has been changed to have its scope limited to - the current file plus any additional files that are automatically - processed based on its name plus $BRO_PREFIXES. - -- As an experiment, ftp.bro has been modified to a style that includes - using "module FTP". Likely other policy scripts will be converted - in the near future, and their variable names changed accordingly - (e.g., "ftp_hot_files" will become "FTP::hot_files"). - -- The new "match" expression has been modified to allow any yield type - rather than just strings. It is likely to change significantly again - soon. - -- Iterating over multi-dimensional tables/sets now works (Robin Sommer). - For example: - - const remote_peers_ssl : table[addr, port] of Peer &redef; - [...] - for ( [ip, p] in remote_peers_ssl ) - connect_ssl(ip, p, remote_peers_ssl[ip, p]$retry); - -- Checkpointing of persistent state on SIGHUP now happens via bro.init - (Robin Sommer). Not tested. - -- fmt() now supports %x for hexadecimal formatting (Chema Gonzalez). - -- alert.bro logs the source for remote alerts; by redefining the new - "event_source_description: string" locally, you can specify how a - Bro is to refered to on the remote side. (Robin Sommer) - -- software.bro now tracks HTTP clients, too (Robin Sommer). This - will be extended in the near future. - -- Some FreeBSD 5.X porting tweaks (Sergey Osokin). - - -0.8a37 Wed Sep 3 23:20:21 PDT 2003 - -- A new attribute, "&persistent", indicates that a global variable's - state should persist on disk (Robin Sommer). Currently, they - reside in the file "state.bst". - - For example, given the following script: - - global a = 0 &persistent; - - event bro_init() - { - print ++a; - } - - then every time you run it, a increases by one. - - You can dump the state file using "bro -x state.bst ". - (There's also a partially-implemented XML dumper which you invoke via - "bro -X state.bst ".) - - If you send Bro a HUP signal, then it will synchronize all persistent - state to disk (checkpoint) at that point. More generally, the policy - script can do so at any time by calling the new built-in checkpoint_state(). - - By including the new policy script "checkpoint", you can have Bro - automatically checkpoint every time checkpoint_interval elapses - (default 15 minutes). - -- You can also record events to disk by loading the "capture-events" - policy script, or calling the new capture_events() built-in (Robin Sommer). - The events are written to the file events.bst. You can dump these - using bro -x (or -X), as above. You can also replay them using - "bro -R ". By replaying them using the same policy script as - generated them, you should get the same output. For example: - - bro -r tracefile myscript capture-events - - ... - - bro -R events.log myscript - -- An experimental module facility has been added (Umesh Shankar). - - The module facility implements namespaces. Everything is in some namespace - or other. The default namespace is called "GLOBAL" and is searched by - default when doing name resolution. The scoping operator is "::" as in - C++. You can only access things in the current namespace, things in the - GLOBAL namespace, or things that have been explicitly exported from a - different namespace. Exported variables and functions still require - fully-qualified names. The syntax is as follows: - - module foo; # Sets the current namespace to "foo" - export { - int i; - int j; - } - int k; - - module bar; - int i; - - foo::i = 1; - bar::i = 2; - print i; # bar::i (since we're currently in module bar) - j = 3; # ERROR: j is exported, but the fully qualified name - # foo::j is required - foo::k = 4; # ERROR: k is not exported - - The same goes for calling functions. - - One restriction currently in place is that variables not in the "GLOBAL" - namespace can't shadow those in GLOBAL, so you can't have - - module GLOBAL; - global i: int; - - module other_module; - global i: int; - - It is a little confusing that the "global" declaration really only means - that the variable i is global to the current module, not that it is truly - global and thus visible everywhere (that would require that it be in - GLOBAL, or if using the full name is ok, that it be exported). Perhaps - there will be a change to the syntax in the future to address this. - - The "module" statement cuts across @load commands, so that if you say: - - module foo; - @load other_script; - - then other_script will be in module foo. Likewise if other_script changes - to module bar, then the current module will be module bar even after - other_script is done. However, this functionality may change in the future - if it proves problematic. - - The policy scripts in the Bro distribution have not yet been updated to - use it, but there is a backward-compatibility feature so that existing - scripts should work without modification. In particular, everything is - put in GLOBAL by default. - -- The hooks are now in place for communicating events between running - Bro's. An experimental implementation of doing so (written by Robin - Sommer) will be integrated shortly. - -- A side-effect of getting those hooks in place is that event handlers must - now be directly specified (by naming them) rather than indirectly - (for example, by indexing a table whose value yields an event handler). - This may be fixed soon. - -- An experimental "match" expression scans a list of predicates to find - the first one that's both true and has the highest priority (Umesh Shankar). - - The general form is: - - match some_record on { - [ priority, class_name, predicate ], - ... - } - - where "predicate" is evaluated in the context of the value (and type) of - some_record. For example, if some_record has a field "num_alerts" - then predicate could be "$num_alerts > 5". - - "priority" is a non-negative integer (i.e., of type "count"), and, - for now, "class_name" is a string. - - For example, - - global c: conn_id; - - c = [ $orig_h = 0.0.0.0, $orig_p = 0/tcp, - $resp_h = 1.1.1.1, $resp_p = 1/tcp ]; - - print match c on { - [ 2, "emptyweb", $orig_h == 0.0.0.0 && $resp_p == 80/tcp ], - [ 1, "emptyhost", $orig_h == 0.0.0.0 ], - [ 0, "should not match", 1 == 0 ] - }; - - will print "emptyhost". - - The initial intent behind this is to eventually provide more flexible - customization of alert processing, though it clearly has broader - applicable. *It is very likely that the semantics and perhaps the syntax - of "match" will change in the near future.* - -- Bro's packet filter is now computed via pcap.bro (which is automatically - included via bro.init). It uses two new built-ins: - - precompile_pcap_filter(id: PcapFilterID, s: string): bool - install_pcap_filter(id: PcapFilterID): bool - - The first is for precompiling a pcap filter so it can be installed - or turned off dynamically. Associating an ID with the filter, you can - then activate the filter by calling the second function (installing a - new filter replaces the current one). (Robin Sommer) - - Be default, pcap.bro is responsible for building a pcap string based on - the capture/restrict_filters defined by the various analyzers. It compiles - and installs this filter, so there is no observable difference in usage - to the old implementation, except capture/restrict_filter are now *tables* - rather than strings, and are written as plural rather than singular. - So the analyzers need to define something like this: - - [finger.bro] - redef capture_filters += { ["finger"] = "port finger" }; - - This then allows "finger" to be used as the name for the corresponding - filter element (see the next item). - -- load-level.bro is an experimental policy script for allowing Bro to - shed or add on load (in terms of which protocols it analyzes). It - provides three interface functions for setting the current loadlevel: - - # level is LoadLevel1, ..., LoadLevel10 - function set_load_level(level: PcapFilterID): bool - - function increase_load_level() - function decrease_load_level() - - load-levels.bro defines ten different load levels (from 1 to 10, with 10 - being the one with the most load imposed on the system), which are - configured by defining capture_load_level/restrict_load_levels: Example: - - redef capture_load_levels += { - ["dns"] = LoadLevel1, - ["smtp"] = LoadLevel2, - ["http-reply"] = LoadLevel3, - ["http-request"] = LoadLevel8, - }; - - This means for example: "include the capture_filter associated with - 'http-reply' if the current load level is 3 or below". There's a similar - mechanism for restrict_filters: - - redef restrict_filters += { - ["cs-only"] = "net 131.159.0.0/16", - }; - - redef restrict_load_levels += { - ["cs-only"] = LoadLevel7, - }; - - This applies the given restrict_filter if the current load level is 7 - or *above*. - - The pcap filters belonging to the ten load levels are built and pre-compiled - on startup. The three functions shown above just call install_pcap_filter() - then. (Robin Sommer) - -- drop-adapt.bro tries to adjust the load level based on the current - packet drop rate (Robin Sommer). - -- synflood.bro is an experimental policy script for detecting SYN floods. - It is not yet documented, other than brief comments in the script. - (Robin Sommer) - -- Subnet lookups now use Patricia trees instead of brute-force, which should - make them significantly more efficient for large lists, e.g., local nets. - (Robin Sommer) - -- Due to the addition of modules, which use a C++-style "::" scope - designator, you now need to use "0x" to introduce IPv6 address constants - that use "::" to specify a series of empty octets. For example, you - used to be able to specify "deadbeef::cafe", but now this needs to be - "0xdeadbeef::cafe". Note that "1::2" can still be written without needing - a "0x" prefix; it's just hex constants that start with letters that need - the prefix. - -- A new built-in, escape_string(), takes a string and returns a copy of - it that uses escape sequences for any problematic characters (Robin Sommer). - -- A number of low-level bug fixes and portability tweaks (Robin Sommer, - Ruoming Pang, Christian Kreibich, Chema Gonzalez). - -- A new timer, status_update_timer, fires for each ongoing connection - every connection_status_update_interval seconds (default = 0 seconds, - which means "don't fire). (Robin Sommer) - -- An additional Bro-level packet filter can filter/sample packets - based on their src/dest ip/subnet (using a Patricia tree for - efficiency; Robin Sommer). install_src_addr_filter(ip, flags, p) drops - packets originating from ip with probability p (0..100) if none of the - given TCP flags is set. install_src_net_filter, install_dst_addr_filter - and install_dst_net_filter" work similarly. The corresponding "uninstall_*" - functions remove the filters again. - -- The @if/@else/@endif mechanisms have been made more powerful (Robin Sommer). - -- New configure option --enable-debug to compile without optimization - (Robin Sommer). - -- Small tweaks to the mpatrol support (Robin Sommer). - -- SMTP is now one of the services which can use a small inactivity timeout - in inactivity.bro (Robin Sommer). - -- Alerts for signatures which are triggered by worms may be suppressed if - we already know that the triggering host is indeed infected by a worm - (Robin Sommer). - -- Matches of a signature can now be counted per destination host - (Robin Sommer). - -- snort2bro now ignores sid-526 ("BAD TRAFFIC data in TCP SYN packet"; - Robin Sommer). Due to different semantics of Bro (matching stream-wise) - and Snort (matching packet-wise) this signature generates a lot of - false positives. - - -0.8a34 Sun Jul 13 09:11:32 PDT 2003 - -- The new "subnet" type corresponds to a CIDR prefix (Robin Sommer). - You can use the '/' operator to convert an address to a subnet. The - "in" operator tests whether an address matches a subnet (e.g., "1.2.3.4 - in 1.2.255.255/16" yields T). - - You can index tables with index type "subnet" using addresses to retrieve - the table entry with the longest-matching prefix for that address, and - a number of tables/sets in the default scripts have been converted to - this form. So for example the local_16_nets and local_24_nets variables - have been replaced by local_nets, which has type "set[subnet]", - is_local_addr() now refers to it, and skip_scan_nets_{16,24} have - likewise been consolidated into skip_scan_nets. - - One present deficiency is that subnets can't be used in tables/sets - with multiple indices. Fixing this is going to take some time ... - Another deficiency is that the longest-match lookup is not very - efficient. - - Caution: this feature has been only briefly tested with IPv6. - -- ALERT now generates an event "alert_action" which includes the alert_info - and AlertAction associated with an alert. You can define your own handler - to further customize alert processing. - -- The "snort2bro" conversion utility has been extended to recognize some - of the new Snort options (depth, distance, within; Robin Sommer). For - example: - - alert .... ( ... content:"|00 01 86 B8|"; - content:"|00 00 00 02|"; distance:4; within:4; - content:"%x %x"; distance:16; within:256; - ... ) - - is converted to: - - signature sid-1891 { - ... - payload /.*\x00\x01\x86\xB8.{4}\x00\x00\x00\x02.{16}.{0,251}%x %x/ - } - - Note that not all of the new Snort options are supported yet. - -- You can refer to script variables within signatures (Robin Sommer). - For example, the following is now possible: - - signature sid-547 { - ip-proto == tcp - src-ip != local_nets - dst-ip == local_nets - dst-port == 21 - event "FTP \"MKD \" possible warez site" - tcp-state established - payload /[mM][kK][dD] / - } - - This makes the signatures independent of the environment ("local_nets" - in the above examle). snort2bro now converts some well-known Snort - variables into corresponding Bro variables, a number of which are listed - in policy/site.bro. - -- The default action for signature matches is now logging insted of only - writing it to a file (Robin Sommer). - -- You can now use the '^' and '$' regular expression operators inside - subpatterns. For example, /foo$/ | /^bar/ works now. - -- You can now use "msec" for milliseconds (in addition to "usec" for - microseconds, "sec" for seconds, etc). - -- The log_file_name and open_log_file functions are now redef'able - if you want to change their behavior. - -- Bro now exits more cleanly upon exhaustion of memory (Robin Sommer). - -- A bug was fixed for the case of getrlimit() returning RLIM_INFINITY for - the maximum number of open files. - -- Numerous additions of std:: scoping to address porting issues (Robin Sommer). - -- gcc 3.X and Solaris portability fixes. - -- A new event RemoteWorm is the complement of LocalWorm (Robin Sommer). - -- A bug in which the FTP analyzer would complain about failing to be able - to look up a connection has been fixed (Robin Sommer). - -- You can now configure Bro using "--with-mpatrol" to activate MPatrol - debugging hooks. When built in this mode, -m dumps the leak table - and -M shows unfreed parts of the heap. These dumps can also be - triggered using SIGUSR1 / SIGUSR2, respectively. (Robin Sommmer) - -- A script function get_contents_type() which returns the type of reassembling - being done for a connection (none/orig-side/resp-side/both; Robin Sommer). - -- A minor bug fix for the regular expression matcher (Robin Sommer). - - -0.8a32 Thu Jun 12 23:33:21 PDT 2003 - -- The low-level hash functions have been reimplemented to use HMAC-MD5 to - counter the "algorithmic complexity attacks" discussed in the USENIX - Security 2003 paper by Scott Crosby & Dan Wallach - (http://www.cs.rice.edu/~scrosby/hash/) (Ruoming Pang and Vern Paxson) - -- Ruoming Pang has made extensive changes to the anonymization/transformation - framework. A paper describing it will appear in SIGCOMM 2003, and will - be included with the Bro distribution once the final copy is ready. - -- Internal hash tables now resize incrementally, and more quickly. - This makes a big difference in avoiding lengthy processing pauses when - processing large volumes of traffic. (Craig Leres) - -- gcc 3.1 and Linux portability tweaks. - -- The calling sequence of http_request() has changed. It now includes both - the original URI and the escaped URI. This was made explicit because - we found that if only the original was passed in, it was too easy to forget - to expand the escapes in it; and if only the escaped was passed in, - some attacks could be missed. (Ruoming Pang) - -- Signature rules can now refer to "finger" payload as well as HTTP and FTP - (Robin Sommer). - -- The signature engine now includes keywords "dst-ip", "dst-port", - "ip-proto", "src-ip", "src-port". (Robin Sommer) - -- Packet sorting now defaults to off. - -- The FTP analysis now attempts to track the current directory (Ruoming Pang). - -- A number of scan detection additions have been added (Scott Campbell): - - activate_landmine_check = F - if T, then access to more than landmine_thresh_trigger - addresses in landmine_address (a set of addresses) - constitutes a scan - - activate_priv_port_check = T - if T, then inbound access to more than priv_scan_trigger - privileged ports (i.e., ports < 1024) is considered a port - scan. You can exclude particular services from the count - via troll_skip_service, which defaults to { smtp, ftp, - ssh, 20/tcp, http }. - -- The SMTP analysis now includes a new alert, HotEmailRecipient, which - is triggered by mail sent to any of the addresses specified in - the pattern hot_recipients (which defaults to a bare /@/). - -- The new built-in cat_string_array_n() is like cat_string_array() except - it takes starting and ending indices as additional arguments (Ruoming Pang). - -- The new built-in sort_string_array() takes a string array and returns - a sorted version (Ruoming Pang). It currently uses an N^2 algorithm - so shouldn't be used on large arrays. - -- The new built-in subst() substitutes all instances of a given pattern - in a given string with another string (Ruoming Pang). For example, - subst("fooboo", /oo/, "xxx") returns "fxxxbxxx". - -- The new built-in cut_tail() trims characters from the end of a string - (Ruoming Pang). For example, cut_tail("fooboo", 2) returns "foob". - -- sub_bytes() can now take a negative argument, which is with respect to - the end of the string rather than the beginning (Ruoming Pang). - -- The new built-in md5_hmac() returns an HMAC-MD5 hash of the given string - (Ruoming Pang). The HMAC secret key is generated from available entropy - when Bro starts up, or it can be specified for repeatability using - the new -K flag. - -- The new built-in split_all() is like split() except that the returned - array also includes the parts of the string that match the pattern - used for the splitting (Ruoming Pang). - -- The new built-in split_n() splits up to a given number of instances, - optionally returning the parts matching the split pattern (Ruoming Pang). - -- The new built-in split_complete() is the most general of the split functions - (the others are written in terms of it). It splits a given string - with separators that appear in either a given pattern or a given set - of strings, optionally including the text matching the separators in - its return value, and optionally limiting the number of matches to - a specified maximum. (Ruoming Pang) - -- The new built-in to_string_literal() returns an escaped version of a string - suitable to feeding into Bro's parser. For example, calling it on - "foo\nbar" (where '\n' is an embedded newline) returns "foo\x0abar". - (Ruoming Pang) - -- subst_substring() has been renamed subst_string (Ruoming Pang). - -- unescape_URI() no longer takes the connection as an argument (Ruoming Pang). - -- config.guess and config.sub updated - -- String escape expansion has been refined (Ruoming Pang) and some bugs - fixed. It now supports a format that's consistent with Bro string literals. - -- Scanning of octal and hex escape sequences now stops after 3 or 2 - characters, respectively. For example, "\0007" now expands to a - NUL followed by the character '7'. (Ruoming Pang) - -- Bug fixes for handling of bare and 's when expected - (Ruoming Pang), and associated "weird" events. - -- A bug in signature matching reassembly has been fixed (Robin Sommer). - -- A bug in reporting "bad connection size" for connection sizes > 2GB - has been fixed. - -- A bug in computing sizes for large partial connections has been fixed. - -- A bug in delayed generation of connection closed events has been fixed. - -- A framework has been added for compression of some "weird" events - to only be generated once-per-connection (Ruoming Pang). - -- Some of the "weird"'s generated by the HTTP analyzer have been regularized. - -- Some memory management fixes. - -- A performance problem in allocating List objects was fixed. - -- The copyright dates have been updated (Craig Leres). - - -0.8a22 Wed Jan 15 16:47:18 PST 2003 - -- There is now a "conditional compilation" feature similar to C's preprocessor - (contributed by Robin Sommer). "@if (expr)" will include the remaining - input lines up to "@endif" only if the given expression evaluates to true. - "@ifdef (id)" will do so only if the given identifier has been defined, - and "@ifndef (id)" only if it has not been defined. There's currently - no "@else" directive, and conditionals cannot be nested. Both of these - will be added in the future. - -- New built-in functions (contributed by Ruoming Pang): parse_dotted_addr - takes a string in A1.A2.A3.A4 form an returns a corresponding "addr" value. - unescape_URI takes a URI that possibly has embedded escape sequences - ("%61" for 'a') and expands the sequences. - -- The URIs in HTTP requests are no longer automaticaly escaped. Instead, - they need to be manually escaped using unescape_URI. As this is likely - error-prone (users forgetting to do so), the plan is to change the - calling sequence of http_request in the near future so that *both* the - escaped and the unescaped URI are passed in. - -- A number of g++ 3.0 porting tweaks have been added (thanks to Sean Irvine). - -- The term "rule" has been systematically changed to "signature" (Robin Sommer). - -- The functionality of monitoring packet drops has been moved into its - own policy script, netstats.bro (Robin Sommer). - -- A number of rewriter and location bugs have been fixed. - -- Some missing HTTP "weird"'s have now been included in weird.bro. - - -0.8a21 Thu Nov 28 23:31:38 PST 2002 - -- A new mechanism will reorder packets within a window of packet_sort_window - if due to their sequence numbers they appear to be out of order. This - can help a great deal when reading from multiple NICs. (Contributed by - Ruoming Pang.) - -- A bug in regular-expression matching in which for example - "2a1" == /[0-9]+/ would evaluate as true was fixed (Ruoming Pang). - -- There's now a rewriter/anonymizer for FTP (Ruoming Pang). - -- The rewriter/transformation framework now supports a notion of - delaying transformation decisions until later (Ruoming Pang). - -- An incompatibility with bison 1.75 has been identified. The problem - is that bison 1.75 generates incomplete location information for empty - grammar productions (those for which the RHS is empty). This will - be fixed soon. - -- Some bugs in the signature engine have been fixed (Robin Sommer). - -- The sources no longer use the not-fully-portable hash_map template - (Umesh Shankar). - -- Some bugs with the debugger getting confused about source line number - locations, and also with the -t option, have been fixed (Umesh Shankar). - -- If a content gap occurs wholly inside an HTTP entity, then the analyzer - will skip over the entity and continue processing any subseqeuent HTTP - items, rather than giving up on processing the connection (Ruoming Pang). - -- The following new built-in functions have been contributed by - Ruoming Pang: - - function cat_string_array(a: string_array): string - function split_all(str: string, re: pattern): string_array - function strstr(big: string, little: string): count - function subst_substring(s: string, from: string, to: string): string - function int_to_count(n: int): count - function fmt_ftp_port(a: addr, p: port): string - - -0.8a20 Sun Nov 17 20:09:31 PST 2002 - -- This is the first "public" Bro release (one accessible directly via - the Web). The README has been correspondingly updated. - -- The user manual has been significantly expanded, and source for it is - now included in the release. - -- Some "active mapping" bug fixes (contributed by Umesh Shankar). - -- The configuration script now checks for the presence of the necessary - BIND libraries/headers, and also for bogus --enable options (contributed - by Craig Leres). - -- backdoor.bro now includes a tcpdump filter for detecting (some) KaZaA - traffic. - -- http-reply.bro now tracks the sizes of request/response entities. - -- http-request.bro now treats an URL accessing variants of /etc/netconfig - as sensitive. - - -0.8a18 Sun Oct 27 15:28:23 PST 2002 - -- Improvements to the performance of the SMTP analyzer (Ruoming Pang). - A new function, skip_smtp_data, skips over the content of an SMTP - message. - -- If you're doing trace rewriting and specify -w but don't specify -A, - then the rewritten traffic goes to the -w save file, along with the - usual non-transformed traffic (Ruoming Pang). If a connection is - being transformed (which is specified by the new rewriting_smtp_trace - and rewriting_http_trace globals), then only its transformed packets - are written to the -w file; not the original packets. This allows - a potentially large degree of trace compression. There's also a - new variable omit_rewrite_place_holder, which directs that when rewriting - a trace, packets that are placeholders for now-missing packets (because - you've excised content) are *not* generated. This diminishes the - timing fidelity of the transformed trace, but also can save a large - amount of space. - -- SMTP relay analysis is now standalone from regular SMTP analysis - (Ruoming Pang). - -- Some memory management and error propagation fixes (Ruoming Pang and - Vern Paxson). - - -0.8a16 Wed Oct 23 23:48:40 PDT 2002 - -- "--enable-brov6" was broken by some recent changes. It now works again. - -- Some "make distclean" tweaks. - -- Error checking for "redef enum". - - -0.8a15 Tue Oct 22 00:02:51 PDT 2002 - -- Fixed Makefile bug which left out wide-spread dependencies on - event.bif.netvar_h. - - -0.8a14 Mon Oct 21 01:16:46 PDT 2002 - -- The "add" statement has been implemented. This allows you to add - elements to sets. A bunch of policy scripts that used to use - "table [] of bool"'s for this purpose have been modified to instead - use sets. - -- You can now extend the elements of an enum using "redef enum foo += { ... }" - where the additional names are listed in the braces (contributed by - Umesh Shankar). A number of policy scripts have been tweaked to use - this for the (fairly) new Alert enum. This allows you to create new - Alert's without alert.bro having to know about them. - -- Some bugs in identifying error locations have been fixed. - -- -A now supports anonymizing IP addresses (contributed by Ruoming Pang). - This includes four new functions: preserve_prefix, preserve_subnet, and - preserve_net (which direct that the corresponding prefix/net/subnet - not be anonymized) and anonymize_addr (which returns the anonymized - version of the given address. - -- Some bugs in HTTP -A have been fixed (thanks to Ruoming Pang). - -- The beginnings of support for CIDR prefixes has been added by - Ruoming Pang. - -- Porting tweaks (use of map's rather than hash_map's, courtesy of - Umesh Shankar; libedit comments fixed). - -- http-detail.bro has gone away. - -- Some more copyright and $Header -> $Id fixes. - -- There is now a function string_cat() which concatenates two strings - and returns the result (contributed by Ruoming Pang). This function - is useful in place of the regular cat() because the latter will - expand escape sequences etc. It probably shouldn't, and that may - change in the future. - - -0.8a11 Sun Oct 13 10:53:07 PDT 2002 - -- The framework for defining built-in functions has been extended - so it's also now the way to specify the interface to event handlers. - See event.bif. (Contributed by Ruoming Pang) - -- A new policy script, http-abstract.bro, can be loaded when doing HTTP - transformation (via http-rewriter.bro), contributed by Ruoming Pang. - It trims HTTP text responses to the first "http_abstract_max_length" bytes. - -- A new built-in, skip_http_entity_data, specifies that the entity - associated with the current HTTP request/response should be skipped - (Ruoming Pang). - -- More changes have been made to the mechanisms for tracking source - file locations in order to pinpoint error messages. If you encounter - problems, please let me know. - -- If you try to configure but your resolve library lacks res_mkquery(), - this is now a fatal error (Craig Leres). Ideally, someone will modify - DNS_Mgr.cc to work even if only blocking DNS is available ... - -- In most (not all, yet) of the sources, the copyright notices have - been updated, $Header has been changed to $Id, and config.h is included - (Craig Leres). - - -0.8a10 Tue Oct 8 16:05:42 PDT 2002 - -- The way that Bro tracks the source-file location associated with - different objects has been tweaked, with a resulting savings of - about 10% in Bro's memory use and a smidgen of CPU time, too. - -- Built-in functions now are better about identifying source-file locations - associated with errors. - -- The http.$BRO_ID log file format has changed to no longer track - individual request streams. (So it is now closer to how it used - to operate in 0.7.) - -- The autoconf setup has been tweaked so that you use: - - --enable-brov6 enable IPV6 processing - --enable-activemapping enable active mapping processing - --enable-expire-dfa-states enable DFA state expiration - - rather than --with-XXX as was the case in 0.8a9. - - -0.8a9 Mon Oct 7 10:15:12 PDT 2002 - -- A bunch of configuration/autoconf/portability tweaks (Craig Leres). - These include compiling under Solaris. - - When running ../configure, you can now specify: - - --with-brov6 enable IPV6 processing - --with-activemapping enable active mapping processing - --with-expire-dfa-states enable DFA state expiration - - You no longer set these via editing the Makefile. - -- Some bugs with HTTP 1.0 keep-alive connections fixed (Ruoming Pang). - -- The "hf" suite of utilities is no longer distributed in aux/. Instead, - get it directly from ftp://ftp.ee.lbl.gov/hf.tar.gz. - -- bro_logchk has been renamed bro-logchk.pl and is now distributed in aux/ - (it was supposed to be before, but was inadvertantly left out of the list - of distribution files). - - -0.8a7 Fri Oct 4 22:24:30 PDT 2002 - -- HTTP reply analysis has been split into a number of policy script files: - - http-body.bro - http-detail.bro - http-entity.bro - http-event.bro - http-header.bro - http-reply.bro - http-rewriter.bro - - so you can readily trade off how detailed the HTTP processing is vs. - the CPU/memory it consumes (contributed by Ruoming Pang). - -- Bro now generates login_prompt events when the $TTYPROMPT environment - variable is passed during a login session, in order to detect the - recently publicized Solaris remote /bin/login exploit. - -- Ruoming Pang has extended the framework for defining "rewriter" functions - to now also serve as the way to define general built-in functions. - -- bro.init has been rearranged to have a more logical structure (courtesy - of Ruoming Pang). - -- Craig Leres contributed a number of portability & autoconf tweaks. - -- Craig Leres has extended nb_dns.{h,c} to support IPv6 DNS lookups. - Bro does not yet take advantage of these extensions. - -- The beginnings of portability to gcc 3.2 were added. There unfortunately - is more work to do here! - -- The README has finally been updated to have more correspondence with - the 0.8 release. - - -0.8a6 Wed Oct 2 18:58:12 PDT 2002 - -- Upgrade to autoconf 2.53 (Craig Leres). - - -0.8a5 Tue Oct 1 19:04:53 PDT 2002 - -- The regular expression matcher how has a mechanism to stop scanning when - no further match is possible (Robin Sommer). If you find problems with - regular expression matching, especially if you're using EXPIRE_DFA_STATES, - please let us know! - -- Rule/signature files are now searched for using $BROPATH (Robin Sommer). - In addition, you can define a list of signature files to incorporate - using the new global "signature_files". For example, - - redef signature_files += "web-stuff" - - will look for web-stuff and web-stuff.sig along $BROPATH. - -- The tcp_deliver_undelivered variable is now tcp_match_undelivered and - only applies to delivering otherwise-undelivered data to the signature - matcher; such data is *not* delivered to any associated analyzer. - (Robin Sommer) - -- The framework for tracking version numbers now allows negative as - well as positive versions (Robin Sommer). - - -0.8a4 Tue Oct 1 15:54:58 PDT 2002 - -- Support for extracting the contents of HTTP replies (and POST's), - and for transforming/anonymizing HTTP traffic, contributed by Ruoming Pang. - -- Some minor internal tweaks to the timer management to help track patterns - of timer expiration. - - -0.8a3 Mon Sep 23 22:48:07 PDT 2002 - -- HTTP reply handling refined in policy scripts. - -- New built-in functions to_int(), to_count() convert strings to - integers/counts. - -- Bug fixes for DNS_Mgr and IPv6. - -- AckAboveHole alerts now ignored (just written to alert.$BRO_ID) by default. - -- Packets belong to ignored connections (for example, partial connections if - partial_connection_ok is false) are no longer recorded to the save file. - -- Some minor formatting/naming tweaks. - - -0.8a1 Sat Sep 21 22:09:23 PDT 2002 - -- IPv6 support enabled if you build using -DBROv6. Deficiencies: Bro - doesn't yet look up hostnames for AAAA records; no handling of extension - headers (if you have traces of these, please send them to me!); no - handling of FTP PORT/PASV w/ IPv6 addresses (again, if you have traces, - please send them!); DNS analyzer doesn't understand AAAA yet (again, - please send me traces!); you have to change the capture_filter line - in tcp.bro (as indicated in the script) in order to process TCP traffic, - due to deficiencies in libpcap's support for IPv6. - -- Bro is migrating towards a more structured way of handling log messages / - alerts. Analyzers now @load alert.bro, which has a function ALERT() - for processing alerts. Soon this function will provide a variety of - filtering/processing hooks; expect changes. - -- Bro now has an HTTP response analyzer (contributed by Ruoming Pang). - The HTTP policy scripts have been split up into http.bro (just general - definitions), http-request.bro (handles requests; loaded by http.bro), - http-reply.bro (handles replies; you need to explicitly load this), and - http-detail.bro (handles individual headers). http-reply.bro will be - undergoing some significant reworking in the near future; probably the - scripts will be merged back into a single http.bro plus http-detail.bro. - -- ssl-worm.bro contains a prototype policy script for detecting the - Scalper SSL worm (contributed by Robin Sommer). It uses the signature - file sig.ex.ssl-worm.bro. If someone has traces of Scalper in action - to send us, that would be great. - -- A new policy script, contents.bro, extracts the contents of each - Bro connection into its own pair of files (one file for each - direction). Use in conjunction with -f or discarder_XXX() to - extract specific connections. - -- A new built-in function, strcmp(), returns the usual comparison between - two strings (contributed by Robin Sommer). - -- A new event, content_gap(), is generated when Bro detects that it is - forced to skip over data in a reconstructed TCP stream because it is - missing from the packet input. - -- BIND8 is no longer included with the distribution. If this causes you - problems, let me know. - -- aux/scripts/bro_logchk is a Perl script for parsing Bro HTTP & FTP logs - (contributed by Jim Barlow). - -- You can now compare addresses to see which is larger. a < b means - that in network order, the octets making up 'a' are ordered before - those for 'b'. E.g., 128.2.3.4 < 128.2.3.5 < 129.0.0.1. Note that - IPv4 addresses are all < IPv6 addresses (other than IPv4 addresses - that are embedded in IPv6 addresses, e.g., ::128.2.3.4 < 128.2.3.5). - -- Serious bug in TCP option handling fixed. - -- Some bugs in CRLF handling fixed (courtesy Ruoming Pang). - -- Bug in the implementation of &optional fixed. - -- Bug in computing memory statistics when not reading packets (from - an interface or the trace file) fixed. - -- You can now include a trailing comma after the last item in an - "enum" enumeration list. - -- port-name.bro now maps 389/tcp to "ldap". - -- A bug has been fixed in loading files multiple times - - -v0.7a175 Thu Aug 29 21:14:34 PDT 2002 - -- bro -s reads in a *signature* file to search for regular expressions - in packet payloads or TCP byte streams (written by Robin Sommer). - See policy/rules.bro for an example of a policy script for processing - the matches. - - Note that this feature is experimental, and will be evolving in the - near future. - -- The python script "snort2bro" reads in Snort signatures and translates - them into Bro signature rules, suitable for processing using -s. - - An example of its operation is seen by running - - python snort2bro < sig.ex.web-rules.snort - - which, after reading in sig.ex.classification.config and - sig.ex.reference.config, generates the output given in - sig.ex.web-rules.bro, which is suitable to use as input to - bro -s. - -- bro -d invokes a gdb-like debugger (written by Umesh Shankar). You can - set breakpoints and watchpoints, examine tracebacks, print Bro expressions, - and the like. Type "help" for on-line help. - -- bro -t turns on tracing of the policy script execution, - written to the given file. - -- Bro now includes an SMTP analyzer, which includes processing MIME - message bodies (written by Ruoming Pang). See smtp.bro and mime.bro - for related policy scripts. smtp.bro includes several experimental - techniques for detecting mail relaying. - -- You can now define enumerated types, such as - - type scanner_type: enum { - SCANNER_STEALTH, SCANNER_HIGH_SPEED, SCANNER_AMBIGUOUS, - }; - - Enumerated types can be compared for equality with one another, and used - as table indices, but cannot be converted to/from integers. - -- bro -A invokes an experimental, general trace transformation/ - anonymization framework (written by Ruoming Pang) which writes a modified - tcpdump trace file from the input (which can be the network or another - trace file) with potentially extensive modifications to the recorded - packets. - - Transformers are built from .rw files (currently, {finger,ftp,ident,smtp}.rw), - which are processed by the utility "rwcl" to generate both event engine - analyzer components and rewriter policy scripts (for example, after - configuring and building Bro, you'll find the scripts - policy/{finger,ftp,ident,smtp}.rw.bro). - - See policy/smtp-rewriter.bro for an example of a policy script that - performs transformation/anonymization. - -- New built-ins: - - split(s: string, p: pattern): string_array; - - takes a string and splits it into pieces at each occurrence of - the regular expression pattern p. (The functionality is like - that in awk.) It returns a string_array, which is a table[count] - of string that is indexed starting at 1, giving the different - pieces. - - For example, - - split("foobar", /o/) - - returns a 3-element table, for which [1] is the string "f", - [2] is the string "" (empty), and [3] is the string "bar". - - split1(s: string, p: pattern): string_array; - - split1() does the same thing as split(), but only performs splitting - at the first occurrence, so it returns either a one-element table - (if the pattern doesn't appear in the string) or a two-element - table. split1("foobar", /o/) returns a 2-element table for which - [1] is "f" and [2] is "obar". - - md5_hash(s: string): string - - returns (in human-readable form) the MD5 hash of a given string. - - So, for example, - - md5_hash("foobar") - - yields "3858f62230ac3c915f300c664312c63f". - - to_addr(s: string): addr - - takes a string representing an address in "dotted quad" format - and returns the correponding "addr" value. - - set_buf(f: file, buffered: bool) - - sets the given file to have its writes buffered or unbuffered - depending on the value of "buffered". It does not return a value. - - connection_exists: function(c: conn_id): bool - - returns T if the given connection identifier corresponds to a - currently instantiated connection (one for which the event engine - has state), F otherwise. - - lookup_connection(c: conn_id): connection - - returns the "connection" record associated with the given - connection identifier, or a fatal run-time error if there - isn't one. - - set_inactivity_timeout(c: conn_id, timeout: interval): interval - - sets the inactivity timeout for the given connection to the - given interval, returning the old interval. - - If the interval is non-zero, then when no packets have been - processed for a connection after that much time has elapsed, - the connection is deleted, and an "inactivity_timeout" event - generated. - - get_matcher_stats(): matcher_stats - - used for gathering statistics about the signature matcher - - rewriting_trace(): bool - - returns T if -A was specified (anonymize/rewrite a trace), - F otherwise. - -- New events: - - connection_state_remove(c: connection); - - Invoked when the event engine has removed the connection from - its state. - - connection_SYN_packet(c: connection, pkt: SYN_packet); - - Invoked for each SYN/SYN-ACK packet. - - connection_timeout(c: connection); - - Invoked when the event engine times out a connection - for - example, because the originator sent a SYN that was never - answered, so the connection was never established. - - connection_reused: event(c: connection); - - Invoked when the event engine decides that a new SYN for - an existing connection reflects a reuse of the connection - four-tuple, rather than belonging to the existing connection. - -- New globals: - - const ignore_checksums = F &redef; - - If true, then the event engine does not verify checksums (and - hence will not discard packets with bad checksums). - - const tcp_deliver_undelivered = F &redef; - - If true, then when the event engine closes a connection, if - that connection has a chunk of data not yet delivered to its - analyzer (which only happens if the data is above a sequence - hole, indicating either a packet filter glitch or a protocol - botch), then the undelivered data will at that point be delivered - to the connection's analyzer. - - const tcp_reassembler_ports_orig: set[port] = {} &redef; - const tcp_reassembler_ports_resp: set[port] = {} &redef; - - Sets of ports for which, if a connection has the corresponding - originator/responder port, then the event engine will reassemble - the byte stream of the connection. - - Normally, the event engine reassembles byte streams for any - connection for which there's an analyzer, and otherwise doesn't. - These variables can be used to force reassembly for the originator - or responder side (respectively) of connections for which there - isn't otherwise an analyzer. This is useful when doing signature - matching on reassembled byte streams, for protocols that are - not otherwise analyzed by the event engine. - - const table_expire_interval = 1 min &redef; - - How often to check table entries to see whether they've expired - (see &read_expire, etc., below). - - const requires_trace_commitment = F; - - If true, then when rewriting/anonymizing traces, nothing will - actually be written to the edited trace file unless you call: - - rewrite_commit_trace(c: connection, commit: bool, future: bool) - - If "future" is true, then future rewritten packets will be - automatically commited; otherwise, writing them to the trace - file requires another explicit rewrite_commit_trace() call. - - const inactivity_timeout = 0 secs &redef; - - As noted above, when a connection becomes inactive, time it out - after this interval. If 0 secs, then don't time it out. - -- An SSH analyzer extracts SSH client/server version information. See - ssh.bro for the related policy script. - -- There's now a (very) simple TFTP analyzer available in tftp.bro. - -- You can now set the global "frag_timeout" to an interval which controls - how long fragments are kept before discarding them (contributed by Ashley - Thomas). If you don't set the global, or set it to 0.0 sec, then fragments - are kept around indefinitely. - -- An implementation of an experimental anti-evasion technique, "active - mapping", has been written by Umesh Shankar. It is not yet ready for - general use, and isn't compiled in unless -DACTIVE_MAPPING. - -- Four new attributes can now be associated with tables (implemented - by Robin Sommer): &read_expire, &write_expire, and &create_expire - will delete table entries after a given interval has elapsed since - the table entry was last read, written, or created. For example: - - global a: table[addr] of count &default=0 &create_expire = 5 sec; - - will delete each entry added to it 5 seconds after the entry was added, - regardless of subsequent read/write activity to the element. - - &expire_func allows you to associate a function with the table such that - whenever an entry expires, the function is invoked. It's passed the - value of the table entry (not the index - perhaps this should be changed), - and returns an "interval" value. If the interval is <= 0.0 seconds, then - the table entry is immediately deleted. Otherwise, it is deleted after - the given interval has elapsed. - -- When listing multiple attributes, you no longer separate them with - commas. For example, if you used to have: - - global a: table[string] of count &default=0, &redef; - - you now need to use: - - global a: table[string] of count &default=0 &redef; - -- You can now construct records using - - [$field1 = , $field2 = , ...] - - Such record values can be assigned to other records providing that the - target value's type includes all of the fields (same name and type) - present in the record value, and that any missing fields have the - &optional or &default attribute (see next item). - - You can also include a record value inside the record constructor, and - all of its fields will be included in the constructed record value. - -- Record fields can now be annotated with &optional, to indicate - that the field needn't be present, or &default, which indicates - a default value to provide if the field is missing. - -- You can query as to whether a record has a value for a given field - using the new "?$" operator. So for example: - - type my_rec: record { - num: count &default = 0; - msg: string; # mandatory, since no &optional/&default - }; - - global r: my_rec; - - r = [$msg = "hello"]; - - print r?$num, r?$msg, r$num; - - will print "F, T, 0," because even though 'r' has a default value - for $num (which shows up when printing r$num), that field is missing, - hence r?$num is F. - -- An experimental scheme has been added (by Umesh Shankar) for managing - general attributes associated either with all values ("global attributes") - or particular particular values. This scheme is likely to change in - the near future, and hence isn't explained here further. - -- The DNS analysis now includes ADDL and AUTH records, and much richer - policy script analysis (per policy/dns.bro). - -- You can now "redef" a function or event handler to override its - previous definition. For a function, this looks like: - - redef log_hook = function(msg: string): bool - { - ... - } - - For an event handler, it's just the usual definition preceded by "redef. - For example, - - redef event ack_above_hole(c: connection) { } - - would replace the default ack_above_hole handler with one that does nothing. - -- HTTP server and HTTP proxy backdoor detectors have been added, - generating http_signature_found and http_proxy_signature_found, - respectively (contributed by Ruoming Pang). - -- A KaZaA backdoor detector has been added, which generates - kazaa_signature_found for likely KaZaA connections. - -- The new policy scripts flag-irc.bro and flag-warez.bro provide - hooks for defining site policies for detecting IRC and access - to warez. - -- portmapper.bro now tracks the services it sees, and the names are - used in connection summaries rather than generic services like port-656. - -- bro -C (or redef'ing the "ignore_checksums" global to T) instructs - Bro to ignore any checksum errors and go ahead and analyze such packets. - -- The (trivial) policy script print-globals.bro dumps out all of the policy - script global variables and the amount of memory they consume. - -- The policy script code-red.bro has been renamed worm.bro and generalized - to detect Nimda as well as Code Red 1 & 2. - -- A bunch of additional default sensitive URIs have been added to http.bro. - http.bro also now doesn't report worm-related URIs. - -- A bunch of less common portnames were removed from port-names.bro. - -- Empty regular expressions are now allowed. - -- The finger_request event now has a third parameter, the additional - text after the username. - -- More systematic handling of NULs and CRLF by the event engine. - -- Hex escape sequences now must have exactly two hexadecimal characters. - -- FYI - work has begun on significantly altering the way that policy - scripts generate alerts. - -- Work has begun (by Robin Sommer) on a general framework for tracking - client/server versions. See software.bro. - -- Work has begun on a NETBIOS analyzer (see NetbiosSSN.cc). Contributions - (e.g., finishing it :-) welcome. - -- Work has begun on migrating the internals to process IPv6 in addition - to IPv4. - -- A number of bug fixes, leaks, and memory allocation lint tweaks. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.7a90 Thu Sep 06 00:50:43 PDT 2001 - -- Better state management, especially for use in high-volume sites. There - are now a number of ways to track the resources consumed by Bro while it - runs, and to reduce the amount of memory Bro consumes. Adding - - @load print-resources - - will log a summary of the resources consumed by Bro when it exits, and - - @load reduce-memory - - will change a number of default values in order to significantly diminish - the amount of memory Bro requires. - - Other ways to lower the required memory discussed in the next three items. - -- The global "maintain_http_sessions" now controls whether http.bro tracks - multiple HTTP connections between the same hosts as a single HTTP session. - Doing so costs a considerable amount of state (so set to F for reducing - the memory impact of HTTP analysis). - -- The global "suppress_scan_checks", if true, turns off address and port - scan detection. (You can't achieve this by simply not @load'ing scan.bro, - because it's loaded by some of the default policy scripts.) Turning it - off can save a lot of memory. - -- Note, the ident.bro is also expensive in terms of state-holding. - -- New library functions: - - - resource_usage() returns a record detailing real time, - CPU time, memory, other getrusage info, and the current - and maximum number of TCP/UDP/ICMP connections, and - timers and fragments. - - - val_size() returns the size in bytes needed to represent - a given value (which can be a record, a table, or a - simple constant, for example). It's not fully accurate - but is in the right ballpark. - - - global_sizes() return a table mapping every global - variable to its size (per val_size()). Useful for - tracking which ones are growing large over time. - -- You can now control a number of timers related to Bro's connection - management. Setting them to lower values generally decreases the - amount of state Bro has to keep (see reduce-memory.bro), though - this can also make it easier for an attacker to evade detection: - - tcp_SYN_timeout: interval - Controls how long Bro waits after seeing the - beginning of a connection (whether due to a SYN - or not; the timer is misnamed) before checking - whether it elicited any reply. - - tcp_session_timer: interval - After a connection has closed, wait this long for - further activity before checking whether to time - out its state. - - tcp_connection_linger: interval - When checking a closed connection for further - activity, Bro should consider it inactive if there - hasn't been any for this long. It also complains - if the connection is reused before this much time - has elapsed. - - tcp_attempt_delay: interval - Bro waits this long upon seeing an initial SYN - before timing out the connection attempt. - - tcp_close_delay: interval - Upon seeing a normal connection close, Bro flushes - state after this much time. - - tcp_reset_delay: interval - Upon seeing a RST, Bro flushes state after this - much time. - - tcp_partial_close_delay: interval - Bro generates a connection_partial_close event this - much time after one half of a partial connection - closes, assuming there has been no subsequent - activity. - - non_analyzed_lifetime: interval - If a connection belongs to an application that you - aren't analyzing, Bro times it out after this - interval. If 0 secs (default), then it doesn't - time it out. - - dns_session_timeout: interval - ntp_session_timeout: interval - rpc_timeout: interval - Bro waits this long before timing out a DNS/NTP/RPC - request. - - max_timer_expires: count - The maximum number of timers to expire after - processing each new packet. The value trades off - spreading out the timer expiration load with - possibly having to hold state longer. A value of 0 - means "process all expired timers with each new - packet". - -- Two new flags control other facets of Bro's connection management, - and thus state-holding: - - partial_connection_ok: bool - Whether Bro should analyze connections for which - it doesn't see the beginning, only the middle. - This can be very expensive to do in the face of - stealth-scanning, which looks like a bunch of - partial connections. - - Note, the HTTP analyzer has been modified to - now always skip partial connections. This should - instead be user controllable. - - tcp_SYN_ack_ok: bool - If true, Bro instantiates connection state when - it sees a SYN ack but not the initial SYN (even - if partial_connection_ok is false). The intent - behind this knob (which is not well tested) is - to allow you to filter out initial SYNs and only - react to SYN acks. This keeps Bro from holding - state during SYN scans and SYN flooding, except - for when the destination responds. - -- Some other miscellaneous thresholds that you can now modify from your - policy script: - - heartbeat_interval: count - How often to generate net_stats_update() events. - This timer really isn't needed any more, since - you can use "schedule" to achieve the same effect. - - tcp_storm_thresh: count - If Bro sees this many FINs/RSTs in a row, it - flags them as a "storm". - - tcp_storm_interarrival_thresh: interval - The FINs/RSTs must come with this much time or less - between them. - -- The state management for line-oriented applications like HTTP requests - has been improved. - -- The HTTP analyzer now expands %hex sequences. If anyone has a Unicode - expander to contribute, that'd be terrific. - -- The Code Red detection is more robust (fewer false positives). - -- A new redefinable variable, skip_services, lists applications that should - not be analyzed for purposes of detecting scans. (Default: ident) - The point of having this (poorly named) hook is so that code-red.bro - can add "http" to it, to avoid reporting the zillions of Code Red scans - that a site can see. - -- Bro now matches regular expressions using lazy DFA evaluation. The upshot - of this is (1) it no longer maintains .bro-RE-cache.v1, (2) it starts - up fast regardless of whether you've added new regular expressions, (3) - you can afford to add lots of regular expressions, and (4) it's actually - a bit faster. - -- The list of "hot_ids" has been expanded with a number of other common - root-privileged accounts. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.7a61 Fri Apr 06 14:51:47 PDT 2001 - -- An NTP analyzer has been added. See policy/ntp.bro for a corresponding - policy file that detects the newly discovered NTP remote buffer overflow - attack. - -- example-attacks/ is a new directory that contains trace files of attacks. - Currently, there are just two to play with: - - bro -r example-attacks/ftp-site-exec.trace mt - - will run on a trace of a "site exec" overflow attack, and - - bro -r example-attacks/ntp-attack.trace mt ntp - - will run on an example of the NTP overflow. - -- The doc/ directory includes the postscript and HTML versions of the - first draft of the Bro manual. - -- A new policy file, icmp.bro, has preliminary (and only partially developed) - policy for analyzing ICMP. - -- The file libpcap.bufsize.patch includes the patch necessary on some systems - to increase the maximum libpcap buffer size. - -- You can now use anonymous functions in &default expressions, so for - example you can do: - - global foo: table[count] of string = { - [1] = "1st", [2] = "2nd", [3] = "3rd", - } &default = function(n: count): string { return fmt("%dth", n); }; - - and then referring to foo[5] will yield "5th". - -- There's now a "for" statement to iterate over the indices of a table - or the members of a set: - - for ( i in foo ) - - for the above "foo" will iterate with i assigned to 1, 2, and 3; *but - not in general in that order*. - -- The function contains_string() has been removed, and now you can instead - use an expression like - - "bar" in "foobar" - - which will yield T. - -- The scan detection now has a mechanism for attempting to detect SYN flooding - backscatter and flagging it as different from a stealth scan. - -- New event handlers: - - new_connection_contents() - like new_connection(), but reassembles the - stream so you can use set_content_files() to - write it to a file - - udp_session_done() - invoked when a UDP session (which is defined on - a per-protocol basis; currently only for NTP) - finishes. - - ntp_message() - invoked for each NTP message - -- UDP processing now does accounting for detecting scans. - -- UDP processing now tracks numbers of requests/replies for sessions that - support that notion. The connections are annotated by udp_session_done() - with "[m,n]" for "m" requests and "n" replies, providing either m or n > 1. - -- New variable accessible from policy: - - watchdog_interval - how often the watchdog should check for whether - Bro is making internal progress - -- A bunch of functions no longer have a first argument of the current time; - get it instead from network_time() if you need it: - - authentication_accepted - authentication_rejected - conn_weird - conn_weird_addl - flow_weird - net_weird - -- A bunch of functions now return bool rather than int values: - - set_contents_file - set_login_state - set_record_packets - skip_further_processing - -- The variable "hot_dests" has been renamed to "hot_dsts". - -- 111/tcp is now identified as "portmap" rather than "rpc". - -- Connections flagged as hot for some types of characteristics are now - annotated with the reason associated with the decision. (I think a lot - more of this is needed.) - -- Portmapper dumps are annotated with the results of the mapping. This - will be streamlined in the future. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.7a48 Wed Sep 13 14:37:30 PDT 2000 - -- Changes between this release and v0.6 missing :-( - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.6 Wed Jul 21 17:02:50 PDT 1999 - -- Support for regular expressions added. You specify lex-style regular - expressions between '/'s, for example "/\/etc\/(passwd|shadow)/" has - the type "pattern" and matches /etc/passwd or /etc/shadow (the slashes - in the pattern need to be escaped or else they'd delimit the end of the - pattern). Pattern-matching is via the "in" operator, so for example: - - if ( filename in /\/etc\/(passwd|shadow)/ ) - sensitive_file_access(filename); - - or - - const sensitive_files = /\/etc\/(passwd|shadow)/; - - ... - - if ( filename in sensitive_files ) - sensitive_file_access(filename); - - Presently the "in" operator requires that the entire left-hand side - be matched by the pattern. So, for example, if you want to find the - string "eggdrop" anywhere inside the string "line", you would use - - if ( line in /.*eggdrop.*/ ) - - If you leave off either of the .*'s, then eggdrop will only be matched - at the beginning or end of the line. - - In the future, there will be mechanisms for specifying whether you - want to match anywhere in a line, or anchored; accordingly, *the above - syntax is subject to change*. - - Bro compiles regular expressions into DFAs for fast matching. This can take - quite a bit of time for complicated patterns. Consequently, it maintains a - cache of compiled regular expressions in $HOME/.bro-RE-cache-v1. You can - always safely remove this file; Bro will recreate/repopulate it as needed. - It does not clean up unused entries from it, so if you change your patterns - frequently, you will accumulated lots of old ones and should delete the - file to garbage collect them. - -- An rlogin analysis module has been added and the telnet analysis - generalized to generic "login" analysis, with the following events: - - login_failure(c: connection, user: string, client_user: string, - password: string, line: string) - Generated on a failed attempt to log in. client_user is - the remote user name, if the login is via the rlogin - protocol. - login_success(c: connection, user: string, client_user: string, - password: string, line: string) - Generated on a successful attempt to log in. - - login_input_line(c: connection, line: string) - Generated per line of input typed by the user. - login_output_line(c: connection, line: string) - Generated per line of output generated by the server. - - login_confused(c: connection, msg: string, line: string) - Generated when a login dialog confuses the heuristic - analyzer. msg is a tag for the state mismatch that - was unexpected, line is the corresponding dialog text. - - login_confused_text(c: connection, line: string) - Once a connection is in the confused state, then this - is generated for each subsequent line. - - login_terminal(c: connection, terminal: string) - Generated if the terminal type associated with the - connection is seen. - - login_display(c: connection, display: string) - Generated if the display associated with the connection - is seen. - - excessive_line(c: connection) - Generated when the connection has produced an excessively - long line. - - login_input_line() and login_output_line() are very powerful for - detecting intrusions, when coupled with regular-expression matching. - - login_terminal() is used to detect backdoors that are triggered - by the terminal environment variable. - -- An ident analysis module has been added (port 113). It generates - ident_request, ident_reply, and ident_error events. Port 113 used - to be referred to as "auth"; now it's referred to as "ident". - -- A new type of scan detection has been added, which is triggered - by a remote host trying a large number of username/password - combinations. See the account_tried() function in scan.bro. - -- The default search path for .bro files is now - - .:priv-policy:policy:pub-policy:/usr/local/lib/bro - - where priv-policy/ is intended for private policy and pub-policy/ - for public policy. The Bro alpha distribution ships with a - sample set of pub-policy scripts. - -- New built-ins: - - system(s: string): int - executes the given shell command using system() - and returns its status. - - set_contents_file(c: conn_id, direction: count, f: file) - copies connection c's reassembled byte stream in - either the originator-to-responder direction (if - direction is CONTENTS_ORIG) or the responder-to- - originator direction (CONTENTS_RESP) to the file f. - - reading_live_traffic(): bool - returns true if Bro is running on live traffic (read - from a network interface), false if it's reading from - a save file. - - mkdir(f: string): bool - creates the given directory, returning true if it - was able to, false if not. - - get_orig_seq(c: conn_id): count; - returns the highest sequence number sent by the - originator of connection c. - get_resp_seq(c: conn_id): count; - same for c's responder. - -- Additional new events (other than those related to the new analyzers): - - new_connection(c: connection) - is generated whenever a new connection is seen. - - partial_connection(c: connection) - is generated whenever a new partial connection (one - that doesn't begin with a SYN handshake) is seen. - - pm_bad_port(r: connection, bad_p: count) - is generated when a portmapper response contains - a bad port number. - -- Functions, tables and sets can now be assigned. Assignment is - made by reference to the underlying object. - -- Bro no longer looks up identifiers using getservbyname() to see if they - should be interpreted as port numbers, since this led to portability - problems. Instead, a number of constants are defined in bro.init: - bgp, domain, finger, ftp, gopher, http, ident, rlogin, smtp, ssh and telnet, - -- Bro now supports an arbitrary number of open files (not bound by - the system's limit on file descriptors). - -- There's now a finger_reply event to go with finger_request. - -- A bunch more RPC service names have been added, thanks to Job de Haas - and others. - -- A bug has been fixed in the watchdog handling that caused it to - sometimes expire after a period of network inactivity. - -- The Bro paper in doc/ has been revised (it isn't quite up-to-date, - but considerably closer than the USENIX version). - -- There has been a large amount of reworking of the internals, both - to Bro itself and in the policy scripts. If you find something you're - wondering about, feel free to send me mail asking about it. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.5 Sun Oct 4 00:19:35 PDT 1998 - -- Added Linux support. - -- Major autoconf changes. - -- Some tweaks to suppress g++ warnings. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.4 Fri Sep 11 00:08:04 PDT 1998 - -- The new aux/ directory has some utility programs and scripts. - See aux/README for details. - -- Documentation (though exceedingly limited) describing the connection - log summaries generated by policy/tcp.bro now available in doc/conn-logs. - -- The Telnet username recognition code has been heavily reworked and is - now significantly more robust. - -- A new policy file, policy/scan.bro, detects port and address scanning. - -- Bro now detects the "Land" attack (a trivial addition to its spoof - detection). - -- The distribution now comes with BIND 8.1.2. - -- A new flavor of "weird" event has been added, flow_weird, for unusual - events associated with flows (i.e., a coarser - granularity than TCP or UDP connections). - -- Bro now reassembles fragments, checking for overlaps and consistency. - Associated with fragments are the following flow_weird's: - - excessively_large_fragment - excessively_small_fragment - fragment_inconsistency - fragment_overlap - fragment_protocol_inconsistency - fragment_size_inconsistency - fragment_with_DF - incompletely_captured_fragment - -- The TCP stream reassembly code was rewritten to share functionality - with the new fragment reassembly code. - -- If a handler for it is present, then Bro will generate "packet_contents" - events containing the contents of each packet it receives. This is just - for experimenting with, and, in particular, this event does *not* reflect - TCP stream reassembly. - -- The handling of "conn_weird" events in tcp.bro now has more options. - In the weird_action table, WEIRD_LOG_ALWAYS means that the weird event - should always be logged; WEIRD_LOG_PER_CONN means that it should be - logged once per connection; and WEIRD_LOG_PER_ORIG that it should be - launched once per originator IP address. - -- The example hot.bro now includes provisions for flagging sensitive - inbound and outbound services. - -- Bro now limits the number of events processed when draining the event - queue after processing a packet. This makes Bro less prone to packet - loss during high-speed scans. - -- The line-oriented TCP endpoint class was split into TCP_EndpointLine, - which supports the line-oriented functionality, and TCP_NVT (derived - from TCP_EndpointLine), which implements the Network Virtual Terminal - used by Telnet and FTP. - -- The TCP_NVT class now understands the Telnet Environment option. - -- Escape sequences are now '\' followed by 1 or more octal digits, - instead of excatly three octal digits (which is error prone). - -- If the watchdog timer expires, it now reports the number of events - processed in the current batch of packets, as well as other timing - information. - -- Bro now should not report packet drops that occur after it has begun - to exit (these can occur when draining the pending event queue takes - a while). - -- Bro now detects TCP acknowledgements that occur above a sequence - hole, generating an ack_above_hole event. Nominally, this indicates - packet filter drops, but in fact some buggy TCPs manage to do this :-(. - -- Fledgling HTTP support added. An http_request event is generated when - a new HTTP request is seen, and http_stats is generated when an HTTP - connection terminates, giving (uninteresting to most people) statistics - concerning the request(s). A lot more is needed: parsing persistent - connections and HTTP replies, for one. This is just a start. A stub - for http_reply exists but these events are not presently generated. - -- Ported to Linux (thanks to Pascal Bouchareine). - -- A bug in to_lower() and to_upper() was fixed. - -- The reporting for unexpected FTP connections now more directly - identifies the corresponding FTP session. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.3 Fri Feb 13 19:59:39 PST 1998 - -- Timers are now implemented using calendar queues rather than priority - queues (with thanks to Craig Leres). - -- The new byte_len() built-in returns the length of a string interpreted - as a set of bytes (including any NUL bytes, especially the final one - for a typical string). - -- The new sub_bytes() built-in extracts a subset of a string interpreted - as a set of bytes (i.e., immune to any embedded NULs). - -- Fixed bad interaction with the latest version of libpcap that on some - BPF systems would cause Bro to exit any time a little bit of time went by - without any traffic matching its filter. - -- A bug with constant-folding of the ?: operator has been fixed. - -- A new "conn_stats" event delivers statistical analysis (number of packets - transmitted, retransmitted, out-of-order, replicated) of each connection. - If you define a handler for it, then *no* other TCP processing is done. - This was added for off-line analysis of traces with large numbers of - connections in them. - -- Some minor portability tweaks. - --+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ - -v0.2 Sat Oct 25 11:22:37 PDT 1997 - -- Added autoconf support, thanks to Scott Denton. - -- Ported to FreeBSD, Solaris. - -- Fixed a bug in which partial connections were ignored even for - protocols (e.g., FTP) that can make use of partial dialogs. - -- Included BIND version 8 sources directly in the distribution. - -- Better usage() information (again thanks to Scott), -h and -v flags. - -- README, CHANGES files created. diff --git a/Makefile b/Makefile index adf45f84f4..cbc44c6780 100644 --- a/Makefile +++ b/Makefile @@ -44,13 +44,14 @@ broxygenclean: configured dist: @rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz @rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz - @mkdir $(VERSION_FULL) - @tar --exclude=$(VERSION_FULL)* --exclude=$(VERSION_MIN)* --exclude=.git -cf - . | ( cd $(VERSION_FULL) && tar -xpf - ) - @( cd $(VERSION_FULL) && cp -R ../.git . && git reset -q --hard HEAD && git clean -xdfq && rm -rf .git ) + @git clone --recursive . $(VERSION_FULL) >/dev/null 2>&1 + @find $(VERSION_FULL) -name .git\* | xargs rm -rf @tar -czf $(VERSION_FULL).tgz $(VERSION_FULL) && echo Package: $(VERSION_FULL).tgz && rm -rf $(VERSION_FULL) - @$(HAVE_MODULES) && mkdir $(VERSION_MIN) || exit 0 - @$(HAVE_MODULES) && tar --exclude=$(VERSION_FULL)* --exclude=$(VERSION_MIN)* --exclude=.git `git submodule | awk '{print "--exclude="$$2}' | grep -v cmake | tr '\n' ' '` -cf - . | ( cd $(VERSION_MIN) && tar -xpf - ) || exit 0 - @$(HAVE_MODULES) && ( cd $(VERSION_MIN) && cp -R ../.git . && git reset -q --hard HEAD && git clean -xdfq && rm -rf .git ) || exit 0 + @$(HAVE_MODULES) && git clone . $(VERSION_MIN) >/dev/null 2>&1 || exit 0 + @$(HAVE_MODULES) && (cd $(VERSION_MIN) && git submodule update --init cmake >/dev/null 2>&1) || exit 0 + @$(HAVE_MODULES) && (cd $(VERSION_MIN) && git submodule update --init src/3rdparty >/dev/null 2>&1) || exit 0 + @$(HAVE_MODULES) && (cd $(VERSION_MIN) && git submodule update --init magic >/dev/null 2>&1) || exit 0 + @$(HAVE_MODULES) && find $(VERSION_MIN) -name .git\* | xargs rm -rf || exit 0 @$(HAVE_MODULES) && tar -czf $(VERSION_MIN).tgz $(VERSION_MIN) && echo Package: $(VERSION_MIN).tgz && rm -rf $(VERSION_MIN) || exit 0 bindist: @@ -65,6 +66,7 @@ test: test-all: test test -d aux/broctl && ( cd aux/broctl && make test ) + test -d aux/btest && ( cd aux/btest && make test ) configured: @test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 ) diff --git a/NEWS b/NEWS index dba45c279d..e192c52380 100644 --- a/NEWS +++ b/NEWS @@ -1,53 +1,147 @@ This document summarizes the most important changes in the current Bro -release. For a complete list of changes, see the ``CHANGES`` file +release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as BroControl and Broccoli, come with -their own CHANGES.) +their own ``CHANGES``.) -Bro 2.2 (Work In Progress) -========================== +Bro 2.2 Beta +============ New Functionality ----------------- +- A new file analysis framework moves most of the processing of file + content from script-land into the core, where it belongs. See + ``doc/file-analysis.rst``, or the online documentation, for more + information. + + Much of this is an internal change, but the framework also comes + with the following user-visible functionality (some of that was + already available before but is done differently, and more + efficiently, now): + + - HTTP: + * Identify MIME type of messages. + * Extract messages to disk. + * Compute MD5 for messages. + + - SMTP: + * Identify MIME type of messages. + * Extract messages to disk. + * Compute MD5 for messages. + * Provide access to start of entity data. + + - FTP data transfers: + * Identify MIME types of data. + * Record to disk. + + - IRC DCC transfers: Record to disk. + + - Support for analyzing data transfered via HTTP range requests. + + - A binary input reader interfaces the input framework with the + file analysis, allowing to inject files on disk into Bro's + content processing. + +- A new framework for computing a wide array of summary statistics, + such as counters and thresholds checks, standard deviation and mean, + set cardinality, top K, and more. The framework operates in + real-time, independent of the underlying data, and can aggregate + information from many independent monitoring points (including + clusters). It provides a transparent, easy-to-use user interface, + and can optionally deploy a set of probabilistic data structures for + memory-efficient operation. The framework is located in + ``scripts/base/frameworks/sumstats``. + + A number of new applications now ship with Bro that are built on top + of the summary statistics framework: + + * Scan detection: Detectors for port and address scans. See + ``policy/misc/scan.bro`` (these scan detectors used to exist in + Bro versions <2.0; it's now back, but quite different). + + * Tracerouter detector: ``policy/misc/detect-traceroute.bro`` + + * Web application detection/measurement: + ``policy/misc/app-stats/*`` + + * FTP and SSH brute-forcing detector: + ``policy/protocols/ftp/detect-bruteforcing.bro``, + ``policy/protocols/ssh/detect-bruteforcing.bro`` + + * HTTP-based SQL injection detector: + ``policy/protocols/http/detect-sqli.bro`` (existed before, but + now ported to the new framework) + +- GridFTP support. This is an extension to the standard FTP analyzer + and includes: + + - An analyzer for the GSI mechanism of GSSAPI FTP AUTH method. + GSI authentication involves an encoded TLS/SSL handshake over + the FTP control session. For FTP sessions that attempt GSI + authentication, the ``service`` field of the connection log + will include ``gridftp`` (as well as also ``ftp`` and + ``ssl``). + + - An example of a GridFTP data channel detection script. It + relies on the heuristics of GridFTP data channels commonly + default to SSL mutual authentication with a NULL bulk cipher + and that they usually transfer large datasets (default + threshold of script is 1 GB). For identified GridFTP data + channels, the ``services`` fields of the connection log will + include ``gridftp-data``. + +- Modbus and DNP3 support. Script-level support is only basic at this + point but see ``src/analyzer/protocol/{modbus,dnp3}/events.bif``, or + the online documentation, for the events Bro generates. For Modbus, + there are also some example policies in + ``policy/protocols/modbus/*``. + +- The documentation now includes a new introduction to writing Bro + scripts. See ``doc/scripting/index.rst`` or, much better, the online + version. There's also the beginning of a chapter on "Using Bro" in + ``doc/using/index.rst``. + - GPRS Tunnelling Protocol (GTPv1) decapsulation. -- GridFTP support. TODO: Extend. +- The scripting language now provide "hooks", a new flavor of + functions that share characteristics of both standard functions and + events. They are like events in that multiple bodies can be defined + for the same hook identifier. They are more like functions in the + way they are invoked/called, because, unlike events, their execution + is immediate and they do not get scheduled through an event queue. + Also, a unique feature of a hook is that a given hook handler body + can short-circuit the execution of remaining hook handlers simply by + exiting from the body as a result of a ``break`` statement (as + opposed to a ``return`` or just reaching the end of the body). See + ``doc/scripts/builtins.rst``, or the online documentation, for more + informatin. -- Modbus support. TODO: Extend. +- Bro's language now has a working ``switch`` statement that generally + behaves like C-style switches (except that case labels can be + comprised of multiple literal constants delimited by commas). Only + atomic types are allowed for now. Case label bodies that don't + execute a ``return`` or ``break`` statement will fall through to + subsequent cases. A ``default`` case label is supported. -- DNP3 support. TODO: Extend. - -- ssl.log now also records the subject client and issuer certificates. - -- Hooks: TODO: Briefly summarize the documention from - doc/scripts/builtins.rst here. - -- The ASCII writer can now output CSV files on a per filter basis. - -- Bro's language now has a working "switch" statement that generally - behaves like C-style switches except case labels can be comprised of - multiple literal constants delimited by commas. Only atomic types - are allowed for now. Case label bodies that don't execute a - "return" or "break" statement will fall through to subsequent cases. - A default case label is allowed. - -- Bro's language now has a new set of types "opaque of X". Opaque +- Bro's language now has a new set of types ``opaque of X``. Opaque values can be passed around like other values but they can only be manipulated with BiF functions, not with other operators. Currently, - the following opaque types are supported: + the following opaque types are supported:: - - opaque of md5 - - opaque of sha1 - - opaque of sha256 - - opaquey of entropy. + opaque of md5 + opaque of sha1 + opaque of sha256 + opaque of cardinality + opaque of topk + opaque of bloomfilter - They go along with the corrsponding BiF functions md5_*, sha1_*, - sha256_*, and entropy_*, respectively. Note that these functions - have changed their signatures to work with opaques types rather - than global state as it was before. + These go along with the corrsponding BiF functions ``md5_*``, + ``sha1_*``, ``sha256_*``, ``entropy_*``, etc. . Note that where + these functions existed before, they have changed their signatures + to work with opaques types rather than global state. -- The scripting language now supports a constructing sets, tables, +- The scripting language now supports constructing sets, tables, vectors, and records by name:: type MyRecordType: record { @@ -61,57 +155,33 @@ New Functionality global s = MySet([$c=1], [$c=2]); - Strings now support the subscript operator to extract individual - characters and substrings (e.g., s[4], s[1,5]). The index expression - can take up to two indices for the start and end index of the - substring to return (e.g. "mystring[1,3]"). + characters and substrings (e.g., ``s[4]``, ``s[1,5]``). The index + expression can take up to two indices for the start and end index of + the substring to return (e.g. ``mystring[1,3]``). -- Functions now support default parameters, e.g.: +- Functions now support default parameters, e.g.:: - global foo: function(s: string, t: string &default="abc", u: count &default=0); + global foo: function(s: string, t: string &default="abc", u: count &default=0); -- Scripts can now use two new "magic constants" @DIR and @FILENAME - that expand to the directory path of the current script and just the - script file name without path, respectively. (Jon Siwek) +- Scripts can now use two new "magic constants" ``@DIR`` and + ``@FILENAME`` that expand to the directory path of the current + script and just the script file name without path, respectively. -- The new file analysis framework moves most of the processing of file - content from script-land into the core, where it belongs. See - doc/file-analysis.rst for more information. +- ``ssl.log`` now also records the subject client and issuer + certificates. - Much of this is an internal change, but the framework also comes - with the following user-visibible functionality (some of that was - already available before, but done differently): +- The ASCII writer can now output CSV files on a per filter basis. - [TODO: Update with changes from 984e9793db56.] +- New SQLite reader and writer plugins for the logging framework allow + to read/write persistent data from on disk SQLite databases. - - A binary input reader interfaces the input framework with file - analysis, allowing to inject files on disk into Bro's - processing. - - - Supports for analyzing data transfereed via HTTP range - requests. - - - HTTP: - * Identify MIME type of message. - * Extract message to disk. - * Compute MD5 for messages. - - - SMTP: - * Identify MIME type of message. - * Extract message to disk. - * Compute MD5 for messages. - * Provide access to start of entity data. - - - FTP data transfers: Identify MIME type; record to disk. - - - IRC DCC transfers: Record to disk. - -- New packet filter framework supports BPF-based load-balancing, +- A new packet filter framework supports BPF-based load-balancing, shunting, and sampling; plus plugin support to customize filters dynamically. - Bro now provides Bloom filters of two kinds: basic Bloom filters supporting membership tests, and counting Bloom filters that track - the frequency of elements. The corresponding functions are: + the frequency of elements. The corresponding functions are:: bloomfilter_basic_init(fp: double, capacity: count, name: string &default=""): opaque of bloomfilter bloomfilter_basic_init2(k: count, cells: count, name: string &default=""): opaque of bloomfilter @@ -121,10 +191,11 @@ New Functionality bloomfilter_merge(bf1: opaque of bloomfilter, bf2: opaque of bloomfilter): opaque of bloomfilter bloomfilter_clear(bf: opaque of bloomfilter) - See for full documentation. + See ``src/probabilistic/bloom-filter.bif``, or the online + documentation, for full documentation. - Bro now provides a probabilistic data structure for computing - "top k" elements. The corresponding functions are: + "top k" elements. The corresponding functions are:: topk_init(size: count): opaque of topk topk_add(handle: opaque of topk, value: any) @@ -136,73 +207,82 @@ New Functionality topk_merge(handle1: opaque of topk, handle2: opaque of topk) topk_merge_prune(handle1: opaque of topk, handle2: opaque of topk) - See for full documentation. + See ``src/probabilistic/top-k.bif``, or the online documentation, + for full documentation. -- base/utils/exec.bro provides a module to start external processes - asynchronously and retrieve their output on termination. - base/utils/dir.bro uses it to monitor a directory for changes, and - base/utils/active-http.bro for providing an interface for querying - remote web servers. +- Bro now provides a probabilistic data structure for computing set + cardinality, using the HyperLogLog algorithm. The corresponding + functions are:: -- Summary statistics framework. [Extend] + hll_cardinality_init(err: double, confidence: double): opaque of cardinality + hll_cardinality_add(handle: opaque of cardinality, elem: any): bool + hll_cardinality_merge_into(handle1: opaque of cardinality, handle2: opaque of cardinality): bool + hll_cardinality_estimate(handle: opaque of cardinality): double + hll_cardinality_copy(handle: opaque of cardinality): opaque of cardinality -- A number of new applications build on top of the summary statistics - framework: + See ``src/probabilistic/cardinality-counter.bif``, or the online + documentation, for full documentation. - * Scan detection: Detectors for port and address scans return. See - policy/misc/scan.bro. +- ``base/utils/exec.bro`` provides a module to start external + processes asynchronously and retrieve their output on termination. + ``base/utils/dir.bro`` uses it to monitor a directory for changes, + and ``base/utils/active-http.bro`` for providing an interface for + querying remote web servers. - * Tracerouter detector: policy/misc/detect-traceroute +- BroControl can now pin Bro processes to CPUs on supported platforms: + To use CPU pinning, a new per-node option ``pin_cpus`` can be + specified in node.cfg if the OS is either Linux or FreeBSD. - * Web application detection/measurement: policy/misc/app-metrics.bro +- BroControl comes with its own test-suite now. ``make test`` in + ``aux/broctl`` will run it. - * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro - - * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro - (existed before, but now ported to the new framework) - - * SSH brute-forcing detector feeding the intelligence framework: - policy/protocols/ssh/detect-bruteforcing.bro +In addition to these, Bro 2.2 comes with a large set of smaller +extensions, tweaks, and fixes across the whole code base, including +most submodules. Changed Functionality --------------------- -- We removed the following, already deprecated, functionality: +- The interface to extracting content from application-layer protocols + (including HTTP, SMTP, FTP) has changed significantly due to the + introduction of the new file analysis framework (see above). + +- Removed the following, already deprecated, functionality: * Scripting language: - - &disable_print_hook attribute. + - ``&disable_print_hook attribute``. * BiF functions: - - parse_dotted_addr(), dump_config(), - make_connection_persistent(), generate_idmef(), - split_complete() + - ``parse_dotted_addr()``, ``dump_config()``, + ``make_connection_persistent()``, ``generate_idmef()``, + ``split_complete()`` - - md5_*, sha1_*, sha256_*, and entropy_* have all changed - their signatures to work with opaque types (see above). + - ``md5_*``, ``sha1_*``, ``sha256_*``, and ``entropy_*`` have + all changed their signatures to work with opaque types (see + above). +- Removed a now unused argument from ``do_split`` helper function. -- Removed a now unused argument from "do_split" helper function. +- ``this`` is no longer a reserved keyword. -- "this" is no longer a reserved keyword. - -- The Input Framework's update_finished event has been renamed to - end_of_data. It will now not only fire after table-reads have been - completed, but also after the last event of a whole-file-read (or - whole-db-read, etc.). +- The Input Framework's ``update_finished`` event has been renamed to + ``end_of_data``. It will now not only fire after table-reads have + been completed, but also after the last event of a whole-file-read + (or whole-db-read, etc.). - Renamed the option defining the frequency of alarm summary mails to - 'Logging::default_alarm_mail_interval'. When using BroControl, the + ``Logging::default_alarm_mail_interval``. When using BroControl, the value can now be set with the new broctl.cfg option - "MailAlarmsInterval". + ``MailAlarmsInterval``. -- We have completely reworded the "notice_policy" mechanism. It now no - linger uses a record of policy items but a "hook", a new language - element that's roughly equivalent to a function with multiple - bodies. The documentation [TODO: insert link] describes how to use - the new notice policy. For existing code, the two main changes are: +- We have completely rewritten the ``notice_policy`` mechanism. It now + no longer uses a record of policy items but a ``hook``, a new + language element that's roughly equivalent to a function with + multiple bodies (see above). For existing code, the two main changes + are: - - What used to be a "redef" of "Notice::policy" now becomes a hook - implementation. Example: + - What used to be a ``redef`` of ``Notice::policy`` now becomes a + hook implementation. Example: Old:: @@ -221,9 +301,9 @@ Changed Functionality add n$actions[Notice::ACTION_EMAIL]; } - - notice() is now likewise a hook, no longer an event. If you have - handlers for that event, you'll likely just need to change the - type accordingly. Example: + - notice() is now likewise a hook, no longer an event. If you + have handlers for that event, you'll likely just need to change + the type accordingly. Example: Old:: @@ -233,17 +313,17 @@ Changed Functionality hook notice(n: Notice::Info) { ... } -- The notice_policy.log is gone. That's a result of the new notice +- The ``notice_policy.log`` is gone. That's a result of the new notice policy setup. -- Removed the byte_len() and length() bif functions. Use the ``|...|`` - operator instead. +- Removed the ``byte_len()`` and ``length()`` bif functions. Use the + ``|...|`` operator instead. -- The SSH::Login notice has been superseded by an corresponding - intelligence framework observation (SSH::SUCCESSFUL_LOGIN). +- The ``SSH::Login`` notice has been superseded by an corresponding + intelligence framework observation (``SSH::SUCCESSFUL_LOGIN``). -- PacketFilter::all_packets has been replaced with - PacketFilter::enable_auto_protocol_capture_filters. +- ``PacketFilter::all_packets`` has been replaced with + ``PacketFilter::enable_auto_protocol_capture_filters``. - We removed the BitTorrent DPD signatures pending further updates to that analyzer. diff --git a/VERSION b/VERSION index 421b3adf31..deed660f20 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.1-1368 +2.1-1387 diff --git a/aux/binpac b/aux/binpac index eeb19daacc..3c29b917e5 160000 --- a/aux/binpac +++ b/aux/binpac @@ -1 +1 @@ -Subproject commit eeb19daacc9f12bc4e7c885fa70e71f856a90b1f +Subproject commit 3c29b917e59e8d8200f669d3d9729d36c34b9245 diff --git a/aux/bro-aux b/aux/bro-aux index eb24e62864..ee2d64928e 160000 --- a/aux/bro-aux +++ b/aux/bro-aux @@ -1 +1 @@ -Subproject commit eb24e628648c7d7b931bdb57d38ab32c28296e72 +Subproject commit ee2d64928edc38b10e508bd577a22f52b024c992 diff --git a/aux/broccoli b/aux/broccoli index 5bcee43070..c0d5345bf2 160000 --- a/aux/broccoli +++ b/aux/broccoli @@ -1 +1 @@ -Subproject commit 5bcee430700f714b19a9e794de75cb42408c9ecf +Subproject commit c0d5345bf25d25f6965f3201048344687bacc860 diff --git a/aux/broctl b/aux/broctl index a4912816d7..3582f494de 160000 --- a/aux/broctl +++ b/aux/broctl @@ -1 +1 @@ -Subproject commit a4912816d7a50c882fa537dbeadac13449ca3716 +Subproject commit 3582f494de247784fc7634b319ddf99aef44b6e1 diff --git a/aux/btest b/aux/btest index 3918bd9f5f..6e940b7315 160000 --- a/aux/btest +++ b/aux/btest @@ -1 +1 @@ -Subproject commit 3918bd9f5f99863faec2501e5bc7839ffb17bdc9 +Subproject commit 6e940b73152a14ae63a4405f6a4bc23cf6cbeec1 diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst index 197180241e..9f98c43a5f 100644 --- a/doc/scripting/index.rst +++ b/doc/scripting/index.rst @@ -10,13 +10,6 @@ Writing Bro Scripts Understanding Bro Scripts ========================= -.. todo:: - - The MHR integration has changed significantly since the text was - written. We need to update it, however I'm actually not sure this - script is a good introductory example anymore unfortunately. - -Robin - Bro includes an event-driven scripting language that provides the primary means for an organization to extend and customize Bro's functionality. Virtually all of the output generated by Bro @@ -33,100 +26,113 @@ are invalid. This entire process is setup by telling Bro that should it see a server or client issue an SSL ``HELLO`` message, we want to know about the information about that connection. -It's often the easiest to understand Bro's scripting language by +It's often easiest to understand Bro's scripting language by looking at a complete script and breaking it down into its identifiable components. In this example, we'll take a look at how -Bro queries the `Team Cymru Malware hash registry -`_ for downloads via -HTTP. Part of the Team Cymru Malware Hash registry includes the -ability to do a host lookup on a domain with the format -``MALWARE_HASH.malware.hash.cymru.com`` where ``MALWARE_HASH`` is the MD5 or -SHA1 hash of a file. Team Cymru also populates the TXT record of -their DNS responses with both a "last seen" timestamp and a numerical -"detection rate". The important aspect to understand is Bro already -generates hashes for files it can parse from HTTP streams, but the -script ``detect-MHR.bro`` is responsible for generating the -appropriate DNS lookup and parsing the response. +Bro checks the SHA1 hash of various files extracted from network traffic +against the `Team Cymru Malware hash registry +`_. Part of the Team Cymru Malware +Hash registry includes the ability to do a host lookup on a domain with the format +``.malware.hash.cymru.com`` where ```` is the SHA1 hash of a file. +Team Cymru also populates the TXT record of their DNS responses with both a "first seen" +timestamp and a numerical "detection rate". The important aspect to understand is Bro already +generating hashes for files via the Files framework, but it is the +script ``detect-MHR.bro`` that is responsible for generating the +appropriate DNS lookup, parsing the response, and generating a notice if appropriate. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro -Visually, there are three distinct sections of the script. A base -level with no indentation followed by an indented and formatted -section explaining the custom variables being provided (``export``) and another -indented and formatted section describing the instructions for a -specific event (``event log_http``). Don't get discouraged if you don't +Visually, there are three distinct sections of the script. First, there is a base +level with no indentation where libraries are included in the script through ``@load`` +and a namespace is defined with ``module``. This is followed by an indented and formatted +section explaining the custom variables being provided (``export``) as part of the script's namespace. +Finally there is a second indented and formatted section describing the instructions to take for a +specific event (``event file_hash``). Don't get discouraged if you don't understand every section of the script; we'll cover the basics of the script and much more in following sections. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 7-11 + :lines: 4-6 Lines 7 and 8 of the script process the ``__load__.bro`` script in the respective directories being loaded. The ``@load`` directives are often considered good practice or even just good manners when writing -Bro scripts to make sure they can be -used on their own. While it's unlikely that in a +Bro scripts to make sure they can be used on their own. While it's unlikely that in a full production deployment of Bro these additional resources wouldn't already be loaded, it's not a bad habit to try to get into as you get more experienced with Bro scripting. If you're just starting out, -this level of granularity might not be entirely necessary though. +this level of granularity might not be entirely necessary. The ``@load`` directives +are ensuring the Files framework, the Notice framework and the script to hash all files has +been loaded by Bro. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 12-24 + :lines: 10-31 The export section redefines an enumerable constant that describes the -type of notice we will generate with the logging framework. Bro -allows for redefinable constants, which at first, might seem +type of notice we will generate with the Notice framework. Bro +allows for re-definable constants, which at first, might seem counter-intuitive. We'll get more in-depth with constants in a later chapter, for now, think of them as variables that can only be altered before Bro starts running. The notice type listed allows for the use of the :bro:id:`NOTICE` function to generate notices of type -``Malware_Hash_Registry_Match`` as done in the next section. Notices +``TeamCymruMalwareHashRegistry::Match`` as done in the next section. Notices allow Bro to generate some kind of extra notification beyond its default log types. Often times, this extra notification comes in the -form of an email generated and sent to a pre-configured address. +form of an email generated and sent to a preconfigured address, but can be altered +depending on the needs of the deployment. The export section is finished off with +the definition of two constants that list the kind of files we want to match against and +the minimum percentage of detection threshold in which we are interested. + +Up until this point, the script has merely done some basic setup. With the next section, +the script starts to define instructions to take in a given event. .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro - :lines: 26-44 + :lines: 33-57 The workhorse of the script is contained in the event handler for -``log_http``. The ``log_http`` event is defined as an event-hook in -the :doc:`/scripts/base/protocols/http/main` script and allows scripts -to handle a connection as it is being passed to the logging framework. -The event handler is passed an :bro:type:`HTTP::Info` data structure -which will be referred to as ``rec`` in body of the event handler. +``file_hash``. The ``file_hash`` event is defined in the +:doc:`/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro` script and allows scripts to access +the information associated with a file for which Bro's file analysis framework has +generated a hash. The event handler is passed the file itself as ``f``, the type of digest +algorithm used as ``kind`` and the hash generated as ``hash``. -An ``if`` statement is used to check for the existence of a data structure -named ``md5`` nested within the ``rec`` data structure. Bro uses the ``$`` as -a deference operator and as such, and it is employed in this script to -check if ``rec$md5`` is present by including the ``?`` operator within the -path. If the ``rec`` data structure includes a nested data structure -named ``md5``, the statement is processed as true and a local variable -named ``hash_domain`` is provisioned and given a format string based on -the contents of ``rec$md5`` to produce a valid DNS lookup. +On line 35, an ``if`` statement is used to check for the correct type of hash, in this case +a SHA1 hash. It also checks for a mime type we've defined as being of interest as defined in the +constant ``match_file_types``. The comparison is made against the expression ``f$mime_type``, which uses +the ``$`` dereference operator to check the value ``mime_type`` inside the variable ``f``. Once both +values resolve to true, a local variable is defined to hold a string comprised of the SHA1 hash concatenated +with ``.malware.hash.cymru.com``; this value will be the domain queried in the malware hash registry. The rest of the script is contained within a ``when`` block. In short, a ``when`` block is used when Bro needs to perform asynchronous -actions, such a DNS lookup, to ensure that performance isn't effected. +actions, such as a DNS lookup, to ensure that performance isn't effected. The ``when`` block performs a DNS TXT lookup and stores the result in the local variable ``MHR_result``. Effectively, processing for this event continues and upon receipt of the values returned by :bro:id:`lookup_hostname_txt`, the ``when`` block is executed. The -``when`` block splits the string returned into two seperate values and -checks to ensure an expected format. If the format is invalid, the -script assumes that the hash wasn't found in the respository and -processing is concluded. If the format is as expected and the -detection rate is above the threshold set by ``MHR_threshold``, two -new local variables are created and used in the notice issued by -:bro:id:`NOTICE`. +``when`` block splits the string returned into a portion for the date on which +the malware was first detected and the detection rate by splitting on an text space +and storing the values returned in a local table variable. In line 42, if the table +returned by ``split1`` has two entries, indicating a successful split, we store the detection +date in ``mhr_first_detect`` and the rate in ``mhr_detect_rate`` on lines 45 and 45 respectively +using the appropriate conversion functions. From this point on, Bro knows it has seen a file +transmitted which has a hash that has been seen by the Team Cymru Malware Hash Registry, the rest +of the script is dedicated to producing a notice. -In approximately 15 lines of actual code, Bro provides an amazing +On line 47, the detection time is processed into a string representation and stored in +``readable_first_detected``. The script then compares the detection rate against the +``notice_threshold`` that was defined on line 30. If the detection rate is high enough, the script +creates a concise description of the notice on line 50, a possible URL to check the sample against +virustotal.com's database, and makes the call to :bro:id:`NOTICE` to hand the relevant information +off to the Notice framework. + +In approximately 25 lines of code, Bro provides an amazing utility that would be incredibly difficult to implement and deploy -with other products. In truth, claiming that Bro does this in 15 +with other products. In truth, claiming that Bro does this in 25 lines is a misdirection; there is a truly massive number of things going on behind-the-scenes in Bro, but it is the inclusion of the scripting language that gives analysts access to those underlying -layers in a succinct and well defined manner. +layers in a succinct and well defined manner. The Event Queue and Event Handlers ================================== @@ -168,7 +174,7 @@ the event, and a concise explanation of the functions use. :lines: 29-54 Above is a segment of the documentation for the event -:bro:id:`dns_request` (and the preceeding link points to the +:bro:id:`dns_request` (and the preceding link points to the documentation generated out of that). It's organized such that the documentation, commentary, and list of arguments precede the actual event definition used by Bro. As Bro detects DNS requests being @@ -197,13 +203,8 @@ such, there are events defined for the primary parts of the connection life-cycle as you'll see from the small selection of connection-related events below. -.. todo:: - - Update the line numbers, this isn't pulling in the right events - anymore but I don't know which ones it were. - .. btest-include:: ${BRO_SRC_ROOT}/build/scripts/base/bif/event.bif.bro - :lines: 135-138,154,204-208,218,255-256,266,335-340,351 + :lines: 69-72,88,106-109,129,132-137,148 Of the events listed, the event that will give us the best insight into the connection record data type will be @@ -245,7 +246,7 @@ information gleaned from the analysis of a connection as a complete unit. To break down this collection of information, you will have to make use of use Bro's field delimiter ``$``. For example, the originating host is referenced by ``c$id$orig_h`` which if given a -narritive relates to ``orig_h`` which is a member of ``id`` which is +narrative relates to ``orig_h`` which is a member of ``id`` which is a member of the data structure referred to as ``c`` that was passed into the event handler." Given that the responder port (``c$id$resp_p``) is ``53/tcp``, it's likely that Bro's base DNS scripts @@ -343,7 +344,7 @@ Constants Bro also makes use of constants, which are denoted by the ``const`` keyword. Unlike globals, constants can only be set or altered at parse time if the ``&redef`` attribute has been used. Afterwards (in -runtime) the constants are unalterable. In most cases, redefinable +runtime) the constants are unalterable. In most cases, re-definable constants are used in Bro scripts as containers for configuration options. For example, the configuration option to log password decrypted from HTTP streams is stored in @@ -359,7 +360,7 @@ following line to our ``site/local.bro`` file before firing up Bro. .. btest-include:: ${DOC_ROOT}/scripting/data_type_const_simple.bro -While the idea of a redefinable constant might be odd, the constraint +While the idea of a re-definable constant might be odd, the constraint that constants can only be altered at parse-time remains even with the ``&redef`` attribute. In the code snippet below, a table of strings indexed by ports is declared as a constant before two values are added @@ -417,7 +418,7 @@ The table below shows the atomic types used in Bro, of which the first four should seem familiar if you have some scripting experience, while the remaining six are less common in other languages. It should come as no surprise that a scripting language for a Network Security -Monitoring platform has a fairly robust set of network centric data +Monitoring platform has a fairly robust set of network-centric data types and taking note of them here may well save you a late night of reinventing the wheel. @@ -479,7 +480,7 @@ the ``for`` loop, the next element is chosen. Since sets are not an ordered data type, you cannot guarantee the order of the elements as the ``for`` loop processes. -To test for membership in a set the ``in`` statment can be combined +To test for membership in a set the ``in`` statement can be combined with an ``if`` statement to return a true or false value. If the exact element in the condition is already in the set, the condition returns true and the body executes. The ``in`` statement can also be @@ -546,7 +547,7 @@ iterate over, say, the directors; we have to iterate with the exact format as the keys themselves. In this case, we need squared brackets surrounding four temporary variables to act as a collection for our iteration. While this is a contrived example, we could easily have -had keys containin IP addresses (``addr``), ports (``port``) and even a ``string`` +had keys containing IP addresses (``addr``), ports (``port``) and even a ``string`` calculated as the result of a reverse hostname lookup. .. btest-include:: ${DOC_ROOT}/scripting/data_struct_table_complex.bro @@ -647,7 +648,7 @@ subnet ~~~~~~ Bro has full support for CIDR notation subnets as a base data type. -There is no need to manage the IP and the subnet mask as two seperate +There is no need to manage the IP and the subnet mask as two separate entities when you can provide the same information in CIDR notation in your scripts. The following example below uses a Bro script to determine if a series of IP addresses are within a set of subnets @@ -807,7 +808,7 @@ composite type. We have, in fact, already encountered a a complex example of the ``record`` data type in the earlier sections, the :bro:type:`connection` record passed to many events. Another one, :bro:type:`Conn::Info`, which corresponds to the fields logged into -``conn.log``, is shown by the exerpt below. +``conn.log``, is shown by the excerpt below. .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro :lines: 10-12,16,17,19,21,23,25,28,31,35,37,56,62,68,90,93,97,100,104,108,109,114 @@ -818,7 +819,7 @@ definition is within the confines of an export block, what is defined is, in fact, ``Conn::Info``. The formatting for a declaration of a record type in Bro includes the -descriptive name of the type being defined and the seperate fields +descriptive name of the type being defined and the separate fields that make up the record. The individual fields that make up the new record are not limited in type or number as long as the name for each field is unique. @@ -834,7 +835,7 @@ string, a set of ports, and a count to define a service type. Also included is a function to print each field of a record in a formatted fashion and a :bro:id:`bro_init` event handler to show some functionality of working with records. The definitions of the DNS and -HTTP services are both done inline using squared brackets before being +HTTP services are both done in-line using squared brackets before being passed to the ``print_service`` function. The ``print_service`` function makes use of the ``$`` dereference operator to access the fields within the newly defined Service record type. @@ -851,7 +852,7 @@ record. @TEST-EXEC: btest-rst-cmd bro ${DOC_ROOT}/scripting/data_struct_record_02.bro The example above includes a second record type in which a field is -used as the data type for a set. Records can be reapeatedly nested +used as the data type for a set. Records can be repeatedly nested within other records, their fields reachable through repeated chains of the ``$`` dereference operator. @@ -1128,7 +1129,7 @@ which we will cover shortly. +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+ | policy_items | set[count] | &log &optional | Policy items that have been applied | +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+ -| email_body_sections | vector | &optinal | Body of the email for email notices. | +| email_body_sections | vector | &optional | Body of the email for email notices. | +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+ | email_delay_tokens | set[string] | &optional | Delay functionality for email notices. | +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+ @@ -1142,7 +1143,7 @@ has been heuristically detected and the originating hostname is one that would raise suspicion. Effectively, the script attempts to define a list of hosts from which you would never want to see SSH traffic originating, like DNS servers, mail servers, etc. To -accomplish this, the script adhere's to the seperation of detection +accomplish this, the script adheres to the separation of detection and reporting by detecting a behavior and raising a notice. Whether or not that notice is acted upon is decided by the local Notice Policy, but the script attempts to supply as much information as @@ -1226,7 +1227,7 @@ Bro. In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script which identifies when SSL certificates are set to expire and raises -notices when it crosses a pre-defined threshold, the call to +notices when it crosses a predefined threshold, the call to ``NOTICE`` above also sets the ``$identifier`` entry by concatenating the responder IP, port, and the hash of the certificate. The selection of responder IP, port and certificate hash fits perfectly @@ -1262,7 +1263,7 @@ In short, there will be notice policy considerations where a broad decision can be made based on the ``Notice::Type`` alone. To facilitate these types of decisions, the Notice Framework supports Notice Policy shortcuts. These shortcuts are implemented through the -means of a group of data structures that map specific, pre-defined +means of a group of data structures that map specific, predefined details and actions to the effective name of a notice. Primarily implemented as a set or table of enumerables of :bro:type:`Notice::Type`, Notice Policy shortcuts can be placed as a single directive in your @@ -1308,5 +1309,3 @@ Notice::emailed_types set while the shortcut below alters the length of time for which those notices will be suppressed. .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_shortcuts_02.bro - - diff --git a/src/Expr.cc b/src/Expr.cc index 5447999dd9..e78a7145e6 100644 --- a/src/Expr.cc +++ b/src/Expr.cc @@ -2442,7 +2442,7 @@ RefExpr::RefExpr(Expr* arg_op) : UnaryExpr(EXPR_REF, arg_op) if ( IsError() ) return; - if ( ! is_assignable(op->Type()) ) + if ( ! ::is_assignable(op->Type()) ) ExprError("illegal assignment target"); else SetType(op->Type()->Ref()); diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc index 64c29d632c..7b3f0595b5 100644 --- a/src/input/readers/Raw.cc +++ b/src/input/readers/Raw.cc @@ -73,15 +73,15 @@ void Raw::DoClose() if ( execute && childpid > 0 && kill(childpid, 0) == 0 ) { - // kill child process - kill(childpid, SIGTERM); + // Kill child process group. + kill(-childpid, SIGTERM); if ( forcekill ) { usleep(200); // 200 msecs should be enough for anyone ;) if ( kill(childpid, 0) == 0 ) // perhaps it is already gone - kill(childpid, SIGKILL); + kill(-childpid, SIGKILL); } } } @@ -146,6 +146,11 @@ bool Raw::Execute() else if ( childpid == 0 ) { // we are the child. + + // Obtain a process group w/ child's PID. + if ( setpgid(0, 0) == -1 ) + _exit(251); + close(pipes[stdout_in]); if ( dup2(pipes[stdout_out], stdout_fileno) == -1 ) _exit(252); @@ -180,6 +185,15 @@ bool Raw::Execute() else { // we are the parent + + // Parent also sets child process group immediately to avoid a race. + if ( setpgid(childpid, childpid) == -1 ) + { + char buf[256]; + strerror_r(errno, buf, sizeof(buf)); + Warning(Fmt("Could not set child process group: %s", buf)); + } + if ( ! UnlockForkMutex() ) return false; diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output index fc14c86399..0d981e7fc0 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output @@ -2,24 +2,20 @@ -- event.bif.bro +## Generated for every new connection. This event is raised with the first +## packet of a previously unknown connection. Bro uses a flow-based definition +## of "connection" here that includes not only TCP sessions but also UDP and +## ICMP flows. +global new_connection: event(c: connection ); +## Generated when a TCP connection timed out. This event is raised when +## no activity was seen for an interval of at least +## :bro:id:`tcp_connection_linger`, and either one endpoint has already +## closed the connection or one side never became active. +global connection_timeout: event(c: connection ); +## Generated when a connection's internal state is about to be removed from +## memory. Bro generates this event reliably once for every connection when it +## is about to delete the internal state. As such, the event is well-suited for ## script-level cleanup that needs to be performed for every connection. This ## event is generated not only for TCP sessions but also for UDP and ICMP ## flows. -## -## -global connection_external: event(c: connection , tag: string ); - - -## Generated when a UDP session for a supported protocol has finished. Some of -## Bro's application-layer UDP analyzers flag the end of a session by raising -## Generated when a connection is seen that is marked as being expected. - - -global ipv6_ext_headers: event(c: connection , p: pkt_hdr ); -## their specifics differ slightly. Often, however, both will be raised for -## the same connection if some of its data is missing. We should eventually -## merge the two. -global ack_above_hole: event(c: connection ); - - -## +global connection_state_remove: event(c: connection ); diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output index 8d26caba6c..656abf3ba1 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output @@ -2,8 +2,6 @@ -- detect-MHR.bro - -module TeamCymruMalwareHashRegistry; - -export { - redef enum Notice::Type += { +@load base/frameworks/files +@load base/frameworks/notice +@load frameworks/files/hash-all-files diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output index ace6e79c5e..75d94f6990 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output @@ -2,6 +2,8 @@ -- detect-MHR.bro +export { + redef enum Notice::Type += { ## The hash value of a file transferred over HTTP matched in the ## malware hash registry. Match @@ -15,3 +17,10 @@ /application\/x-java-applet/ | /application\/jar/ | /video\/mp4/ &redef; + + ## The malware hash registry runs each malware sample through several A/V engines. + ## Team Cymru returns a percentage to indicate how many A/V engines flagged the + ## sample as malicious. This threshold allows you to require a minimum detection + ## rate. + const notice_threshold = 10 &redef; +} diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output index 6f4cda878c..7e3e1b0a11 100644 --- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output +++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output @@ -2,13 +2,6 @@ -- detect-MHR.bro - ## The malware hash registry runs each malware sample through several A/V engines. - ## Team Cymru returns a percentage to indicate how many A/V engines flagged the - ## sample as malicious. This threshold allows you to require a minimum detection - ## rate. - const notice_threshold = 10 &redef; -} - event file_hash(f: fa_file, kind: string, hash: string) { if ( kind=="sha1" && match_file_types in f$mime_type ) @@ -21,3 +14,16 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( |MHR_answer| == 2 ) { local mhr_first_detected = double_to_time(to_double(MHR_answer[1])); + local mhr_detect_rate = to_count(MHR_answer[2]); + + local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected); + if ( mhr_detect_rate >= notice_threshold ) + { + local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); + local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); + } + } + } + } + } diff --git a/testing/btest/doc/sphinx/include-scripts_base_bif_event_bif_bro.btest b/testing/btest/doc/sphinx/include-scripts_base_bif_event_bif_bro.btest index fc14c86399..0d981e7fc0 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_bif_event_bif_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_base_bif_event_bif_bro.btest @@ -2,24 +2,20 @@ -- event.bif.bro +## Generated for every new connection. This event is raised with the first +## packet of a previously unknown connection. Bro uses a flow-based definition +## of "connection" here that includes not only TCP sessions but also UDP and +## ICMP flows. +global new_connection: event(c: connection ); +## Generated when a TCP connection timed out. This event is raised when +## no activity was seen for an interval of at least +## :bro:id:`tcp_connection_linger`, and either one endpoint has already +## closed the connection or one side never became active. +global connection_timeout: event(c: connection ); +## Generated when a connection's internal state is about to be removed from +## memory. Bro generates this event reliably once for every connection when it +## is about to delete the internal state. As such, the event is well-suited for ## script-level cleanup that needs to be performed for every connection. This ## event is generated not only for TCP sessions but also for UDP and ICMP ## flows. -## -## -global connection_external: event(c: connection , tag: string ); - - -## Generated when a UDP session for a supported protocol has finished. Some of -## Bro's application-layer UDP analyzers flag the end of a session by raising -## Generated when a connection is seen that is marked as being expected. - - -global ipv6_ext_headers: event(c: connection , p: pkt_hdr ); -## their specifics differ slightly. Often, however, both will be raised for -## the same connection if some of its data is missing. We should eventually -## merge the two. -global ack_above_hole: event(c: connection ); - - -## +global connection_state_remove: event(c: connection ); diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@2.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@2.btest index 8d26caba6c..656abf3ba1 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@2.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@2.btest @@ -2,8 +2,6 @@ -- detect-MHR.bro - -module TeamCymruMalwareHashRegistry; - -export { - redef enum Notice::Type += { +@load base/frameworks/files +@load base/frameworks/notice +@load frameworks/files/hash-all-files diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest index ace6e79c5e..75d94f6990 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@3.btest @@ -2,6 +2,8 @@ -- detect-MHR.bro +export { + redef enum Notice::Type += { ## The hash value of a file transferred over HTTP matched in the ## malware hash registry. Match @@ -15,3 +17,10 @@ /application\/x-java-applet/ | /application\/jar/ | /video\/mp4/ &redef; + + ## The malware hash registry runs each malware sample through several A/V engines. + ## Team Cymru returns a percentage to indicate how many A/V engines flagged the + ## sample as malicious. This threshold allows you to require a minimum detection + ## rate. + const notice_threshold = 10 &redef; +} diff --git a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest index 6f4cda878c..7e3e1b0a11 100644 --- a/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest +++ b/testing/btest/doc/sphinx/include-scripts_policy_frameworks_files_detect-MHR_bro@4.btest @@ -2,13 +2,6 @@ -- detect-MHR.bro - ## The malware hash registry runs each malware sample through several A/V engines. - ## Team Cymru returns a percentage to indicate how many A/V engines flagged the - ## sample as malicious. This threshold allows you to require a minimum detection - ## rate. - const notice_threshold = 10 &redef; -} - event file_hash(f: fa_file, kind: string, hash: string) { if ( kind=="sha1" && match_file_types in f$mime_type ) @@ -21,3 +14,16 @@ event file_hash(f: fa_file, kind: string, hash: string) if ( |MHR_answer| == 2 ) { local mhr_first_detected = double_to_time(to_double(MHR_answer[1])); + local mhr_detect_rate = to_count(MHR_answer[2]); + + local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected); + if ( mhr_detect_rate >= notice_threshold ) + { + local message = fmt("Malware Hash Registry Detection rate: %d%% Last seen: %s", mhr_detect_rate, readable_first_detected); + local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash); + NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]); + } + } + } + } + }