diff --git a/.gitmodules b/.gitmodules index 95053091cf..4998cc6b80 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,18 +1,21 @@ [submodule "aux/bro-aux"] path = aux/bro-aux - url = git://git.bro-ids.org/bro-aux + url = git://git.bro.org/bro-aux [submodule "aux/binpac"] path = aux/binpac - url = git://git.bro-ids.org/binpac + url = git://git.bro.org/binpac [submodule "aux/broccoli"] path = aux/broccoli - url = git://git.bro-ids.org/broccoli + url = git://git.bro.org/broccoli [submodule "aux/broctl"] path = aux/broctl - url = git://git.bro-ids.org/broctl + url = git://git.bro.org/broctl [submodule "aux/btest"] path = aux/btest - url = git://git.bro-ids.org/btest + url = git://git.bro.org/btest [submodule "cmake"] path = cmake - url = git://git.bro-ids.org/cmake + url = git://git.bro.org/cmake +[submodule "src/3rdparty"] + path = src/3rdparty + url = git://git.bro.org/bro-3rdparty diff --git a/CHANGES b/CHANGES index 912d7d301f..36a41ee656 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,1685 @@ +2.3-beta-18 | 2014-06-06 13:11:50 -0700 + + * Add two more SSL events, one triggered for each handshake message + and one triggered for the tls change cipherspec message. (Bernhard + Amann) + + * Small SSL bug fix. In case SSL::disable_analyzer_after_detection + was set to false, the ssl_established event would fire after each + data packet once the session is established. (Bernhard Amann) + +2.3-beta-16 | 2014-06-06 13:05:44 -0700 + + * Re-activate notice suppression for expiring certificates. + (Bernhard Amann) + +2.3-beta-14 | 2014-06-05 14:43:33 -0700 + + * Add new TLS extension type numbers from IANA (Bernhard Amann) + + * Switch to double hashing for Bloomfilters for better performance. + (Matthias Vallentin) + + * Bugfix to use full digest length instead of just one byte for + Bloomfilter's universal hash function. Addresses BIT-1140. + (Matthias Vallentin) + + * Make buffer for X509 certificate subjects larger. Addresses + BIT-1195 (Bernhard Amann) + +2.3-beta-5 | 2014-05-29 15:34:42 -0500 + + * Fix misc/load-balancing.bro's reference to + PacketFilter::sampling_filter (Jon Siwek) + +2.3-beta-4 | 2014-05-28 14:55:24 -0500 + + * Fix potential mem leak in remote function/event unserialization. + (Jon Siwek) + + * Fix reference counting bug in table coercion expressions (Jon Siwek) + + * Fix an "unused value" warning. (Jon Siwek) + + * Remove a duplicate unit test baseline dir. (Jon Siwek) + +2.3-beta | 2014-05-19 16:36:50 -0500 + + * Release 2.3-beta + + * Clean up OpenSSL data structures on exit. (Bernhard Amann) + + * Fixes for OCSP & x509 analysis memory leak issues. (Bernhard Amann) + + * Remove remaining references to BROMAGIC (Daniel Thayer) + + * Fix typos and formatting in event and BiF documentation (Daniel Thayer) + + * Update intel framework plugin for ssl server_name extension API + changes. (Bernhard Amann, Justin Azoff) + + * Fix expression errors in SSL/x509 scripts when unparseable data + is in certificate chain. (Bernhard Amann) + +2.2-478 | 2014-05-19 15:31:33 -0500 + + * Change record ctors to only allow record-field-assignment + expressions. (Jon Siwek) + +2.2-477 | 2014-05-19 14:13:00 -0500 + + * Fix X509::Result record's "result" field to be set internally as type int instead of type count. (Bernhard Amann) + + * Fix a couple of doc build warnings (Daniel Thayer) + +2.2-470 | 2014-05-16 15:16:32 -0700 + + * Add a new section "Cluster Configuration" to the docs that is + intended as a how-to for configuring a Bro cluster. Most of this + content was moved here from the BroControl doc (which is now + intended as more of a reference guide for more experienced users) + and the load balancing FAQ on the website. (Daniel Thayer) + + * Update some doc tests and line numbers (Daniel Thayer) + +2.2-457 | 2014-05-16 14:38:31 -0700 + + * New script policy/protocols/ssl/validate-ocsp.bro that adds OSCP + validation to ssl.log. The work is done by a new bif + x509_ocsp_verify(). (Bernhard Amann) + + * STARTTLS support for POP3 and SMTP. The SSL analyzer takes over + when seen. smtp.log now logs when a connection switches to SSL. + (Bernhard Amann) + + * Replace errors when parsing x509 certs with weirds. (Bernhard + Amann) + + * Improved Heartbleed attack/scan detection. (Bernhard Amann) + + * Let TLS analyzer fail better when no longer in sync with the data + stream. (Bernhard Amann) + +2.2-444 | 2014-05-16 14:10:32 -0500 + + * Disable all default AppStat plugins except facebook. (Jon Siwek) + + * Update for the active http test to force it to use ipv4. (Seth Hall) + +2.2-441 | 2014-05-15 11:29:56 -0700 + + * A new RADIUS analyzer. (Vlad Grigorescu) + + It produces a radius.log and generates two events: + + event radius_message(c: connection, result: RADIUS::Message); + event radius_attribute(c: connection, attr_type: count, value: string); + +2.2-427 | 2014-05-15 13:37:23 -0400 + + * Fix dynamic SumStats update on clusters (Bernhard Amann) + +2.2-425 | 2014-05-08 16:34:44 -0700 + + * Fix reassembly of data w/ sizes beyond 32-bit capacities. (Jon Siwek) + + Reassembly code (e.g. for TCP) now uses int64/uint64 (signedness + is situational) data types in place of int types in order to + support delivering data to analyzers that pass 2GB thresholds. + There's also changes in logic that accompany the change in data + types, e.g. to fix TCP sequence space arithmetic inconsistencies. + + Another significant change is in the Analyzer API: the *Packet and + *Undelivered methods now use a uint64 in place of an int for the + relative sequence space offset parameter. + + Addresses BIT-348. + + * Fixing compiler warnings. (Robin Sommer) + + * Update SNMP analyzer's DeliverPacket method signature. (Jon Siwek) + +2.2-417 | 2014-05-07 10:59:22 -0500 + + * Change handling of atypical OpenSSL error case in x509 verification. (Jon Siwek) + + * Fix memory leaks in X509 certificate parsing/verification. (Jon Siwek) + + * Fix new []/delete mismatch in input::reader::Raw::DoClose(). (Jon Siwek) + + * Fix buffer over-reads in file_analysis::Manager::Terminate() (Jon Siwek) + + * Fix buffer overlows in IP address masking logic. (Jon Siwek) + + That could occur either in taking a zero-length mask on an IPv6 address + (e.g. [fe80::]/0) or a reverse mask of length 128 on any address (e.g. + via the remask_addr BuiltIn Function). + + * Fix new []/delete mismatch in ~Base64Converter. (Jon Siwek) + +2.2-410 | 2014-05-02 12:49:53 -0500 + + * Replace an unneeded OPENSSL_malloc call. (Jon Siwek) + +2.2-409 | 2014-05-02 12:09:06 -0500 + + * Clean up and documentation for base SNMP script. (Jon Siwek) + + * Update base SNMP script to now produce a snmp.log. (Seth Hall) + + * Add DH support to SSL analyzer. When using DHE or DH-Anon, sever + key parameters are now available in scriptland. Also add script to + alert on weak certificate keys or weak dh-params. (Bernhard Amann) + + * Add a few more ciphers Bro did not know at all so far. (Bernhard Amann) + + * Log chosen curve when using ec cipher suite in TLS. (Bernhard Amann) + +2.2-397 | 2014-05-01 20:29:20 -0700 + + * Fix reference counting for lookup_ID() usages. (Jon Siwek) + +2.2-395 | 2014-05-01 20:25:48 -0700 + + * Fix missing "irc-dcc-data" service field from IRC DCC connections. + (Jon Siwek) + + * Correct a notice for heartbleed. The notice is thrown correctly, + just the message conteined wrong values. (Bernhard Amann) + + * Improve/standardize some malloc/realloc return value checks. (Jon + Siwek) + + * Improve file analysis manager shutdown/cleanup. (Jon Siwek) + +2.2-388 | 2014-04-24 18:38:07 -0700 + + * Fix decoding of MIME quoted-printable. (Mareq) + +2.2-386 | 2014-04-24 18:22:29 -0700 + + * Do a Intel::ADDR lookup for host field if we find an IP address + there. (jshlbrd) + +2.2-381 | 2014-04-24 17:08:45 -0700 + + * Add Java version to software framework. (Brian Little) + +2.2-379 | 2014-04-24 17:06:21 -0700 + + * Remove unused Val::attribs member. (Jon Siwek) + +2.2-377 | 2014-04-24 16:57:54 -0700 + + * A larger set of SSL improvements and extensions. Addresses + BIT-1178. (Bernhard Amann) + + - Fixes TLS protocol version detection. It also should + bail-out correctly on non-tls-connections now + + - Adds support for a few TLS extensions, including + server_name, alpn, and ec-curves. + + - Adds support for the heartbeat events. + + - Add Heartbleed detector script. + + - Adds basic support for OCSP stapling. + + * Fix parsing of DNS TXT RRs w/ multiple character-strings. + Addresses BIT-1156. (Jon Siwek) + +2.2-353 | 2014-04-24 16:12:30 -0700 + + * Adapt HTTP partial content to cache file analysis IDs. (Jon Siwek) + + * Adapt SSL analyzer to generate file analysis handles itself. (Jon + Siwek) + + * Adapt more of HTTP analyzer to use cached file analysis IDs. (Jon + Siwek) + + * Adapt IRC/FTP analyzers to cache file analysis IDs. (Jon Siwek) + + * Refactor regex/signature AcceptingSet data structure and usages. + (Jon Siwek) + + * Enforce data size limit when checking files for MIME matches. (Jon + Siwek) + + * Refactor file analysis file ID lookup. (Jon Siwek) + +2.2-344 | 2014-04-22 20:13:30 -0700 + + * Refactor various hex escaping code. (Jon Siwek) + +2.2-341 | 2014-04-17 18:01:41 -0500 + + * Fix duplicate DNS log entries. (Robin Sommer) + +2.2-341 | 2014-04-17 18:01:01 -0500 + + * Refactor initialization of ASCII log writer options. (Jon Siwek) + + * Fix a memory leak in ASCII log writer. (Jon Siwek) + +2.2-338 | 2014-04-17 17:48:17 -0500 + + * Disable input/logging threads setting their names on every + heartbeat. (Jon Siwek) + + * Fix bug when clearing Bloom filter contents. Reported by + @colonelxc. (Matthias Vallentin) + +2.2-335 | 2014-04-10 15:04:57 -0700 + + * Small logic fix for main SSL script. (Bernhard Amann) + + * Update DPD signatures for detecting TLS 1.2. (Bernhard Amann) + + * Remove unused data member of SMTP_Analyzer to silence a Coverity + warning. (Jon Siwek) + + * Fix missing @load dependencies in some scripts. Also update the + unit test which is supposed to catch such errors. (Jon Siwek) + +2.2-326 | 2014-04-08 15:21:51 -0700 + + * Add SNMP datagram parsing support.This supports parsing of SNMPv1 + (RFC 1157), SNMPv2 (RFC 1901/3416), and SNMPv2 (RFC 3412). An + event is raised for each SNMP PDU type, though there's not + currently any event handlers for them and not a default snmp.log + either. However, simple presence of SNMP is currently visible now + in conn.log service field and known_services.log. (Jon Siwek) + +2.2-319 | 2014-04-03 15:53:25 -0700 + + * Improve __load__.bro creation for .bif.bro stubs. (Jon Siwek) + +2.2-317 | 2014-04-03 10:51:31 -0400 + + * Add a uid field to the signatures.log. Addresses BIT-1171 + (Anthony Verez) + +2.2-315 | 2014-04-01 16:50:01 -0700 + + * Change logging's "#types" description of sets to "set". Addresses + BIT-1163 (Bernhard Amann) + +2.2-313 | 2014-04-01 16:40:19 -0700 + + * Fix a couple nits reported by Coverity.(Jon Siwek) + + * Fix potential memory leak in IP frag reassembly reported by + Coverity. (Jon Siwek) + +2.2-310 | 2014-03-31 18:52:22 -0700 + + * Fix memory leak and unchecked dynamic cast reported by Coverity. + (Jon Siwek) + + * Fix potential memory leak in x509 parser reported by Coverity. + (Bernhard Amann) + +2.2-304 | 2014-03-30 23:05:54 +0200 + + * Replace libmagic w/ Bro signatures for file MIME type + identification. Addresses BIT-1143. (Jon Siwek) + + Includes: + + - libmagic is no longer used at all. All MIME type detection is + done through new Bro signatures, and there's no longer a means + to get verbose file type descriptions. The majority of the + default file magic signatures are derived from the default magic + database of libmagic ~5.17. + + - File magic signatures consist of two new constructs in the + signature rule parsing grammar: "file-magic" gives a regular + expression to match against, and "file-mime" gives the MIME type + string of content that matches the magic and an optional strength + value for the match. + + - Modified signature/rule syntax for identifiers: they can no + longer start with a '-', which made for ambiguous syntax when + doing negative strength values in "file-mime". Also brought + syntax for Bro script identifiers in line with reality (they + can't start with numbers or include '-' at all). + + - A new built-in function, "file_magic", can be used to get all + file magic matches and their corresponding strength against a + given chunk of data. + + - The second parameter of the "identify_data" built-in function + can no longer be used to get verbose file type descriptions, + though it can still be used to get the strongest matching file + magic signature. + + - The "file_transferred" event's "descr" parameter no longer + contains verbose file type descriptions. + + - The BROMAGIC environment variable no longer changes any behavior + in Bro as magic databases are no longer used/installed. + + - Removed "binary" and "octet-stream" mime type detections. They + don' provide any more information than an uninitialized + mime_type field which implicitly means no magic signature + matches and so the media type is unknown to Bro. + + - The "fa_file" record now contains a "mime_types" field that + contains all magic signatures that matched the file content + (where the "mime_type" field is just a shortcut for the + strongest match). + + - Reverted back to minimum requirement of CMake 2.6.3 from 2.8.0. + + * The logic for adding file ids to {orig,resp}_fuids fields of the + http.log incorrectly depended on the state of + {orig,resp}_mime_types fields, so sometimes not all file ids + associated w/ the session were logged. (Jon Siwek) + + * Fix MHR script's use of fa_file$mime_type before checking if it's + initialized. (Jon Siwek) + +2.2-294 | 2014-03-30 22:08:25 +0200 + + * Rework and move X509 certificate processing from the SSL protocol + analyzer to a dedicated file analyzer. This will allow us to + examine X509 certificates from sources other than SSL in the + future. Furthermore, Bro now parses more fields and extensions + from the certificates (e.g. elliptic curve information, subject + alternative names, basic constraints). Certificate validation also + was improved, should be easier to use and exposes information like + the full verified certificate chain. (Bernhard Amann) + + This update changes the format of ssl.log and adds a new x509.log + with certificate information. Furthermore all x509 events and + handling functions have changed. + +2.2-271 | 2014-03-30 20:25:17 +0200 + + * Add unit tests covering vector/set/table ctors/inits. (Jon Siwek) + + * Fix parsing of "local" named table constructors. (Jon Siwek) + + * Improve type checking of records. Addresses BIT-1159. (Jon Siwek) + +2.2-267 | 2014-03-30 20:21:43 +0200 + + * Improve documentation of Bro clusters. Addresses BIT-1160. + (Daniel Thayer) + +2.2-263 | 2014-03-30 20:19:05 +0200 + + * Don't include locations into serialization when cloning values. + (Robin Sommer) + +2.2-262 | 2014-03-30 20:12:47 +0200 + + * Refactor SerializationFormat::EndWrite and ChunkedIO::Chunk memory + management. (Jon Siwek) + + * Improve SerializationFormat's write buffer growth strategy. (Jon + Siwek) + + * Add --parse-only option to exit after parsing scripts. May be + useful for syntax-checking tools. (Jon Siwek) + +2.2-256 | 2014-03-30 19:57:28 +0200 + + * For the summary statistics framewirk, change all &create_expire + attributes to &read_expire in the cluster part. (Bernhard Amann) + +2.2-254 | 2014-03-30 19:55:22 +0200 + + * Update instructions on how to build Bro docs. (Daniel Thayer) + +2.2-251 | 2014-03-28 08:37:37 -0400 + + * Quick fix to the ElasticSearch writer. (Seth Hall) + +2.2-250 | 2014-03-19 17:20:55 -0400 + + * Improve performance of MHR script by reducing cloned Vals in + a "when" scope. (Jon Siwek) + +2.2-248 | 2014-03-19 14:47:40 -0400 + + * Make SumStats work incrementally and non-blocking in non-cluster + mode, but force it to operate by blocking if Bro is shutting + down. (Seth Hall) + +2.2-244 | 2014-03-17 08:24:17 -0700 + + * Fix compile errror on FreeBSD caused by wrong include file order. + (Bernhard Amann) + +2.2-240 | 2014-03-14 10:23:54 -0700 + + * Derive results of DNS lookups from from input when in BRO_DNS_FAKE + mode. Addresses BIT-1134. (Jon Siwek) + + * Fixing a few cases of undefined behaviour introduced by recent + formatter work. + + * Fixing compiler error. (Robin Sommer) + + * Fixing (very unlikely) double delete in HTTP analyzer when + decapsulating CONNECTs. (Robin Sommer) + +2.2-235 | 2014-03-13 16:21:19 -0700 + + * The Ascii writer has a new option LogAscii::use_json for writing + out logs as JSON. (Seth Hall) + + * Ascii input reader now supports all config options as per-input + stream "config" values. (Seth Hall) + + * Refactored formatters and updated the the writers a bit. (Seth + Hall) + +2.2-229 | 2014-03-13 14:58:30 -0700 + + * Refactoring analyzer manager code to reuse + ApplyScheduledAnalyzers(). (Robin Sommer) + +2.2-228 | 2014-03-13 14:25:53 -0700 + + * Teach async DNS lookup builtin-functions about BRO_DNS_FAKE. + Addresses BIT-1134. (Jon Siwek) + + * Enable fake DNS mode for test suites. + + * Improve analysis of TCP SYN/SYN-ACK reversal situations. (Jon + Siwek) + + - Since it's just the handshake packets out of order, they're no + longer treated as partial connections, which some protocol analyzers + immediately refuse to look at. + + - The TCP_Reassembler "is_orig" state failed to change, which led to + protocol analyzers sometimes using the wrong value for that. + + - Add a unit test which exercises the Connection::FlipRoles() code + path (i.e. the SYN/SYN-ACK reversal situation). + + Addresses BIT-1148. + + * Fix bug in Connection::FlipRoles. It didn't swap address values + right and also didn't consider that analyzers might be scheduled + for the new connection tuple. Reported by Kevin McMahon. Addresses + BIT-1148. (Jon Siwek) + +2.2-221 | 2014-03-12 17:23:18 -0700 + + * Teach configure script --enable-jemalloc, --with-jemalloc. + Addresses BIT-1128. (Jon Siwek) + +2.2-218 | 2014-03-12 17:19:45 -0700 + + * Improve DBG_LOG macro (perf. improvement for --enable-debug mode). + (Jon Siwek) + + * Silences some documentation warnings from Sphinx. (Jon Siwek) + +2.2-215 | 2014-03-10 11:10:15 -0700 + + * Fix non-deterministic logging of unmatched DNS msgs. Addresses + BIT-1153 (Jon Siwek) + +2.2-213 | 2014-03-09 08:57:37 -0700 + + * No longer accidentally attempting to parse NBSTAT RRs as SRV RRs + in DNS analyzer. (Seth Hall) + + * Fix DNS SRV responses and a small issue with NBNS queries and + label length. (Seth Hall) + + - DNS SRV responses never had the code written to actually + generate the dns_SRV_reply event. Adding this required + extending the event a bit to add extra information. SRV responses + now appear in the dns.log file correctly. + + - Fixed an issue where some Microsoft NetBIOS Name Service lookups + would exceed the max label length for DNS and cause an incorrect + "DNS_label_too_long" weird. + +2.2-210 | 2014-03-06 22:52:36 -0500 + + * Improve SSL logging so that connections are logged even when the + ssl_established event is not generated as well as other small SSL + fixes. (Bernhard Amann) + +2.2-206 | 2014-03-03 16:52:28 -0800 + + * HTTP CONNECT proxy support. The HTTP analyzer now supports + handling HTTP CONNECT proxies. (Seth Hall) + + * Expanding the HTTP methods used in the DPD signature to detect + HTTP traffic. (Seth Hall) + + * Fixing removal of support analyzers. (Robin Sommer) + +2.2-199 | 2014-03-03 16:34:20 -0800 + + * Allow iterating over bif functions with result type vector of any. + This changes the internal type that is used to signal that a + vector is unspecified from any to void. Addresses BIT-1144 + (Bernhard Amann) + +2.2-197 | 2014-02-28 15:36:58 -0800 + + * Remove test code. (Robin Sommer) + +2.2-194 | 2014-02-28 14:50:53 -0800 + + * Remove packet sorter. Addresses BIT-700. (Bernhard Amann) + +2.2-192 | 2014-02-28 09:46:43 -0800 + + * Update Mozilla root bundle. (Bernhard Amann) + +2.2-190 | 2014-02-27 07:34:44 -0800 + + * Adjust timings of a few leak tests. (Bernhard Amann) + +2.2-187 | 2014-02-25 07:24:42 -0800 + + * More Google TLS extensions that are being actively used. (Bernhard + Amann) + + * Remove unused, and potentially unsafe, function + ListVal::IncludedInString. (Bernhard Amann) + +2.2-184 | 2014-02-24 07:28:18 -0800 + + * New TLS constants from + https://tools.ietf.org/html/draft-bmoeller-tls-downgrade-scsv-01. + (Bernhard Amann) + +2.2-180 | 2014-02-20 17:29:14 -0800 + + * New SSL alert descriptions from + https://tools.ietf.org/html/draft-ietf-tls-applayerprotoneg-04. + (Bernhard Amann) + + * Update SQLite. (Bernhard Amann) + +2.2-177 | 2014-02-20 17:27:46 -0800 + + * Update to libmagic version 5.17. Addresses BIT-1136. (Jon Siwek) + +2.2-174 | 2014-02-14 12:07:04 -0800 + + * Support for MPLS over VLAN. (Chris Kanich) + +2.2-173 | 2014-02-14 10:50:15 -0800 + + * Fix misidentification of SOCKS traffic that in particiular seemed + to happen a lot with DCE/RPC traffic. (Vlad Grigorescu) + +2.2-170 | 2014-02-13 16:42:07 -0800 + + * Refactor DNS script's state management to improve performance. + (Jon Siwek) + + * Revert "Expanding the HTTP methods used in the signature to detect + HTTP traffic." (Robin Sommer) + +2.2-167 | 2014-02-12 20:17:39 -0800 + + * Increase timeouts of some unit tests. (Jon Siwek) + + * Fix memory leak in modbus analyzer. Would happen if there's a + 'modbus_read_fifo_queue_response' event handler. (Jon Siwek) + + * Add channel_id TLS extension number. This number is not IANA + defined, but we see it being actively used. (Bernhard Amann) + + * Test baseline updates for DNS change. (Robin Sommer) + +2.2-158 | 2014-02-09 23:45:39 -0500 + + * Change dns.log to include only standard DNS queries. (Jon Siwek) + + * Improve DNS analysis. (Jon Siwek) + + - Fix parsing of empty question sections (when QDCOUNT == 0). In this + case, the DNS parser would extract two 2-byte fields for use in either + "dns_query_reply" or "dns_rejected" events (dependent on value of + RCODE) as qclass and qtype parameters. This is not correct, because + such fields don't actually exist in the DNS message format when + QDCOUNT is 0. As a result, these events are no longer raised when + there's an empty question section. Scripts that depends on checking + for an empty question section can do that in the "dns_message" event. + + - Add a new "dns_unknown_reply" event, for when Bro does not know how + to fully parse a particular resource record type. This helps fix a + problem in the default DNS scripts where the logic to complete + request-reply pair matching doesn't work because it's waiting on more + RR events to complete the reply. i.e. it expects ANCOUNT number of + dns_*_reply events and will wait until it gets that many before + completing a request-reply pair and logging it to dns.log. This could + cause bogus replies to match a previous request if they happen to + share a DNS transaction ID. (Jon Siwek) + + - The previous method of matching queries with replies was still + unreliable in cases where the reply contains no answers. The new code + also takes extra measures to avoid pending state growing too large in + cases where the condition to match a query with a corresponding reply is + never met, but yet DNS messages continue to be exchanged over the same + connection 5-tuple (preventing cleanup of the pending state). (Jon Siwek) + + * Updates to httpmonitor and mimestats documentation. (Jeannette Dopheide) + + * Updates to Logs and Cluster documentation (Jeannette Dopheide) + +2.2-147 | 2014-02-07 08:06:53 -0800 + + * Fix x509-extension test sometimes failing. (Bernhard Amann) + +2.2-144 | 2014-02-06 20:31:18 -0800 + + * Fixing bug in POP3 analyzer. With certain input the analyzer could + end up trying to write to non-writable memory. (Robin Sommer) + +2.2-140 | 2014-02-06 17:58:04 -0800 + + * Fixing memory leaks in input framework. (Robin Sommer) + + * Add script to detect filtered TCP traces. Addresses BIT-1119. (Jon + Siwek) + +2.2-137 | 2014-02-04 09:09:55 -0800 + + * Minor unified2 script documentation fix. (Jon Siwek) + +2.2-135 | 2014-01-31 11:09:36 -0800 + + * Added some grammar and spelling corrections to Installation and + Quick Start Guide. (Jeannette Dopheide) + +2.2-131 | 2014-01-30 16:11:11 -0800 + + * Extend file analysis API to allow file ID caching. This allows an + analyzer to either provide file IDs associated with some file + content or to cache a file ID that was already determined by + script-layer logic so that subsequent calls to the file analysis + interface can bypass costly detours through script-layer. This + can yield a decent performance improvement for analyzers that are + able to take advantage of it and deal with streaming content (like + HTTP, which has been adapted accordingly). (Jon Siwek) + +2.2-128 | 2014-01-30 15:58:47 -0800 + + * Add leak test for Exec module. (Bernhard Amann) + + * Fix file_over_new_connection event to trigger when entire file is + missed. (Jon Siwek) + + * Improve TCP connection size reporting for half-open connections. + (Jon Siwek) + + * Improve gap reporting in TCP connections that never see data. We + no longer accomodate SYN/FIN/RST-filtered traces by not reporting + missing data. The behavior can be reverted by redef'ing + "detect_filtered_trace". (Jon Siwek) + + * Improve TCP FIN retransmission handling. (Jon Siwek) + +2.2-120 | 2014-01-28 10:25:23 -0800 + + * Fix and extend x509_extension() event, which now actually returns + the extension. (Bernhard Amann) + + New event signauture: + + event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info) + +2.2-117 | 2014-01-23 14:18:19 -0800 + + * Fixing initialization context in anonymous functions. (Robin + Sommer) + +2.2-115 | 2014-01-22 12:11:18 -0800 + + * Add unit tests for new Bro Manual docs. (Jon Siwek) + + * New content for the "Using Bro" section of the manual. (Rafael + Bonilla/Jon Siwek) + +2.2-105 | 2014-01-20 12:16:48 -0800 + + * Support GRE tunnel decapsulation, including enhanced GRE headers. + GRE tunnels are treated just like IP-in-IP tunnels by parsing past + the GRE header in between the delivery and payload IP packets. + Addresses BIT-867. (Jon Siwek) + + * Simplify FragReassembler memory management. (Jon Siwek) + +2.2-102 | 2014-01-20 12:00:29 -0800 + + * Include file information (MIME type and description) into notice + emails if available. (Justin Azoff) + +2.2-100 | 2014-01-20 11:54:58 -0800 + + * Fix caching of recently validated SSL certifcates. (Justin Azoff) + +2.2-98 | 2014-01-20 11:50:32 -0800 + + * For notice suppresion, instead of storing the entire notice in + Notice::suppressing, just store the time the notice should be + suppressed until. This saves significant memory but can no longer + raise end_suppression, which has been removed. (Justin Azoff) + +2.2-96 | 2014-01-20 11:41:07 -0800 + + * Integrate libmagic 5.16. Bro now now always relies on + builtin/shipped magic library/database. (Jon Siwek) + + * Bro now requires a CMake 2.8.x, but no longer a pre-installed + libmagic. (Jon Siwek) + +2.2-93 | 2014-01-13 09:16:51 -0800 + + * Fixing compile problems with some versions of libc++. Reported by + Craig Leres. (Robin Sommer) + +2.2-91 | 2014-01-13 01:33:28 -0800 + + * Improve GeoIP City database support. When trying to open a city + database, it now considers both the "REV0" and "REV1" versions of + the city database instead of just the former. (Jon Siwek) + + * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek) + + - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set. + + - Fix failure to locate bro binary if invoking from a relative + path and '.' isn't in PATH. + + * Fix for packet writing to make it use the global snap length. + (Seth Hall) + + * Fix for traffic with TCP segmentation offloading with IP header + len field being set to zero. (Seth Hall) + + * Canonify output of a unit test. (Jon Siwek) + + * A set of documentation updates. (Daniel Thayer) + + - Fix typo in Bro 2.2 NEWS on string indexing. + - Fix typo in the Quick Start Guide, and clarified the + instructions about modifying crontab. + - Add/fix documentation for missing/misnamed event parameters. + - Fix typos in BIF documentation of hexstr_to_bytestring. + - Update the documentation of types and attributes. + - Documented the new substring extraction functionality. + - Clarified the description of "&priority" and "void". + +2.2-75 | 2013-12-18 08:36:50 -0800 + + * Fixing segfault with mismatching set &default in record fields. + (Robin Sommer) + +2.2-74 | 2013-12-16 08:49:55 -0800 + + * Improve warnings emitted from raw/execute input reader. (Jon + Siwek) + + * Further improve core.when-interpreter-exceptions unit test. (Jon + Siwek) + +2.2-72 | 2013-12-12 07:12:47 -0800 + + * Improve the core.when-interpreter-exceptions unit test to prevent + it from occasionally timing out. (Jon Siwek) + +2.2-70 | 2013-12-10 15:02:50 -0800 + + * Fix (harmless) uninitialized field in basename/dirname util + wrapper. (Jon Siwek) + +2.2-68 | 2013-12-09 15:19:37 -0800 + + * Several improvements to input framework error handling for more + robustness and more helpful error messages. Includes tests for + many cases. (Bernhard Amann) + +2.2-66 | 2013-12-09 13:54:16 -0800 + + * Fix table &default reference counting for record ctor expressions. + (Jon Siwek) + + * Close signature files after done parsing. (Jon Siwek) + + * Fix unlikely null ptr deref in broxygen::Manager. (Jon Siwek) + + * FreeBSD build fix addendum: unintended variable shadowing. (Jon + Siwek) + + * Fix build on FreeBSD. basename(3)/dirname(3) const-ness may vary + w/ platform. (Jon Siwek) + + * Updated software framework to support parsing IE11 user-agent + strings. (Seth Hall) + + * Fix the irc_reply event for several server message types. (Seth + Hall) + + * Fix memory leak in input framework. If the input framework was + used to read event streams and those streams contained records + with more than one field, not all elements of the threading Values + were cleaned up. Addresses BIT-1103. (Bernhard Amann) + + * Minor Broxygen improvements. Addresses BIT-1098. (Jon Siwek) + +2.2-51 | 2013-12-05 07:53:37 -0800 + + * Improve a unit test involving 'when' conditionals. (Jon Siwek) + +2.2-48 | 2013-12-04 13:45:47 -0800 + + * Support omission of string slice low/high indices, BIT-1097. + + Omission of the low index defaults to 0: + + s = "12345"; s[:3] == "123" + + Omission of the high index defaults to length of the string: + + s = "12345"; s[3:] == "45" (Jon Siwek) + + * Tweak to SMTP script to adjust for new string slicing behaviour. + (Robin Sommer) + + * Test updates. (Robin Sommer) + +2.2-44 | 2013-12-04 12:41:51 -0800 + + * Fix string slice notation. Addresses BIT-1097. (Jon Siwek) + + Slice ranges were not correctly determined for negative indices + and also off by one in general (included one more element at the + end of the substring than what actually matched the index range). + It's now equivalent to Python slice notation. Accessing a string + at a single index is also the same as Python except that an + out-of-range index returns an empty string instead of throwing an + expection. + +2.2-41 | 2013-12-04 12:40:51 -0800 + + * Updating tests. (Robin Sommer) + +2.2-40 | 2013-12-04 12:16:38 -0800 + + * ssl_client_hello() now receives a vector of ciphers, instead of a + set, to preserve their order. (Bernhard Amann) + +2.2-38 | 2013-12-04 12:10:54 -0800 + + * New script misc/dump-events.bro, along with core support, that + dumps events Bro is raising in an easily readable form for + debugging. (Robin Sommer) + + * Prettyfing Describe() for record types. If a record type has a + name and ODesc is set to short, we now print the name instead of + the full field list. (Robin Sommer) + +2.2-35 | 2013-12-04 10:10:32 -0800 + + * Rework the automated script-reference documentation generation + process, broxygen. Addresses BIT-701 and BIT-751. (Jon Siwek) + + Highlights: + + - Remove --doc-scripts and -Z options to toggle documentation + mode. The parser is now always instrumented to gather + documentation from comments of the form "##", "##!", or + "##<". + + - Raw comments are available at runtime through several BIF + functions: get_*_comments; + + - Add --broxygen and -X options to toggle generating + reST-format documentation output, driven by a config file + argument. + + - Add a "broxygen" Sphinx extension domain, allowing certain + pieces of documentation to be generated on-the-fly via + invoking a Bro process. Re-organized/cleaned up the Sphinx + source tree in doc/ to use this in some places. + +2.2-11 | 2013-12-03 10:56:28 -0800 + + * Unit test for broccoli vector support. (Jon Siwek) + + * Changed ordering of Bro type tag enum, which was out of sync. (Jon + Siwek) + +2.2-9 | 2013-11-18 14:03:21 -0800 + + * Update local.bro for Bro >= 2.2. The commented out Notice::policy + example didn't work anymore. (Daniel Thayer) + +2.2-6 | 2013-11-15 07:05:15 -0800 + + * Make "install-example-configs" target use DESTDIR. (Jon Siwek) + +2.2-5 | 2013-11-11 13:47:54 -0800 + + * Fix the irc_reply event for certain server message types. (Seth + Hall) + + * Fixed Segmentation fault in SQLite Writer. (Jon Crussell) + +2.2 | 2013-11-07 10:25:50 -0800 + + * Release 2.2. + + * Removing location information from ssh.log in external tests. + (Robin Sommer) + +2.2-beta-199 | 2013-11-07 00:36:46 -0800 + + * Fixing warnings during doc build. (Robin Sommer) + +2.2-beta-198 | 2013-11-06 22:54:30 -0800 + + * Update docs and tests for a recent change to detect-MHR.bro + (Daniel Thayer) + + * Update tests and baselines for sumstats docs. (Daniel Thayer) + +2.2-beta-194 | 2013-11-06 14:39:50 -0500 + + * Remove resp_size from the ssh log. Refactor when we write out to + the log a bit. Geodata now works reliably. (Vlad Grigorescu) + + * Update VirusTotal URL to work with changes to their website and + changed it to a redef. (Vlad Grigorescu) + + * Added a document for the SumStats framework. (Seth Hall) + +2.2-beta-184 | 2013-11-03 22:53:42 -0800 + + * Remove swig-ruby from required packages section of install doc. + (Daniel Thayer) + +2.2-beta-182 | 2013-11-01 05:26:05 -0700 + + * Adding source and original copyright statement to Mozilla cert + list. (Robin Sommer) + + * Canonfying an intel test to not depend on output order. (Robin + Sommer) + +2.2-beta-177 | 2013-10-30 04:54:54 -0700 + + * Fix thread processing/termination conditions. (Jon Siwek) + +2.2-beta-175 | 2013-10-29 09:30:09 -0700 + + * Return the Dir module to file name tracking instead of inode + tracking to avoid missing files that reuse a formerly seen inode. + (Seth Hall) + + * Deprecate Broccoli Ruby bindings and no longer build them by + default; use --enable-ruby to do so. (Jon Siwek) + +2.2-beta-167 | 2013-10-29 06:02:38 -0700 + + * Change percent_lost in capture-loss from a string to a double. + (Vlad Grigorescu) + + * New version of the threading queue deadlock fix. (Robin Sommer) + + * Updating README with download/git information. (Robin Sommer) + +2.2-beta-161 | 2013-10-25 15:48:15 -0700 + + * Add curl to list of optional dependencies. It's used by the + active-http.bro script. (Daniel Thayer) + + * Update test and baseline for a recent doc test fix. (Daniel + Thayer) + +2.2-beta-158 | 2013-10-25 15:05:08 -0700 + + * Updating README with download/git information. (Robin Sommer) + +2.2-beta-157 | 2013-10-25 11:11:17 -0700 + + * Extend the documentation of the SQLite reader/writer framework. + (Bernhard Amann) + + * Fix inclusion of wrong example file in scripting tutorial. + Reported by Michael Auger @LM4K. (Bernhard Amann) + + * Alternative fix for the thrading deadlock issue to avoid potential + performance impact. (Bernhard Amann) + +2.2-beta-152 | 2013-10-24 18:16:49 -0700 + + * Fix for input readers occasionally dead-locking. (Robin Sommer) + +2.2-beta-151 | 2013-10-24 16:52:26 -0700 + + * Updating submodule(s). + +2.2-beta-150 | 2013-10-24 16:32:14 -0700 + + * Change temporary ASCII reader workaround for getline() on + Mavericks to permanent fix. (Bernhard Amann) + +2.2-beta-148 | 2013-10-24 14:34:35 -0700 + + * Add gawk to list of optional packages. (Daniel Thayer) + + * Add more script package README files. (Daniel Thayer) + + * Add NEWS about new features of BroControl and upgrade info. + (Daniel Thayer) + + * Intel framework notes added to NEWS. (Seth Hall) + + * Temporary OSX Mavericks libc++ issue workaround for getline() + problem in ASCII reader. (Bernhard Amann) + + * Change test of identify_data BIF to ignore charset as it may vary + with libmagic version. (Jon Siwek) + + * Ensure that the starting BPF filter is logged on clusters. (Seth + Hall) + + * Add UDP support to the checksum offload detection script. (Seth + Hall) + +2.2-beta-133 | 2013-10-23 09:50:16 -0700 + + * Fix record coercion tolerance of optional fields. (Jon Siwek) + + * Add NEWS about incompatible local.bro changes, addresses BIT-1047. + (Jon Siwek) + + * Fix minor formatting problem in NEWS. (Jon Siwek) + +2.2-beta-129 | 2013-10-23 09:47:29 -0700 + + * Another batch of documentation fixes and updates. (Daniel Thayer) + +2.2-beta-114 | 2013-10-18 14:17:57 -0700 + + * Moving the SQLite examples into separate Bro files to turn them + into sphinx-btest tests. (Robin Sommer) + +2.2-beta-112 | 2013-10-18 13:47:13 -0700 + + * A larger chunk of documentation fixes and cleanup. (Daniel Thayer) + + Apart from many smaller improves this includes in particular: + + * Add README files for most Bro frameworks and base/protocols. + * Add README files for base/protocols. + * Update installation instructions. + * Improvements to file analysis docs and conversion to using + btest sphinx. + +2.2-beta-80 | 2013-10-18 13:18:05 -0700 + + * SQLite reader/writer documentation. (Bernhard Amann) + + * Check that the SQLite reader is only used in MANUAL reading mode. + (Bernhard Amann) + + * Rename the SQLite writer "dbname" configuration option to + "tablename". (Bernhard Amann) + + * Remove the "dbname" configuration option from the SQLite reader as + it wasn't used there. (Bernhard Amann) + +2.2-beta-73 | 2013-10-14 14:28:25 -0700 + + * Fix misc. Coverity-reported issues (leaks, potential null pointer + deref, dead code, uninitialized values, + time-of-check-time-of-use). (Jon Siwek) + + * Add check for sqlite3 command to tests that require it. (Daniel + Thayer) + +2.2-beta-68 | 2013-10-14 09:26:09 -0700 + + * Add check for curl command to active-http.test. (Daniel Thayer) + +2.2-beta-64 | 2013-10-14 09:20:04 -0700 + + * Review usage of Reporter::InternalError, addresses BIT-1045. + + Replaced some with InternalWarning or AnalyzerError, the later + being a new method which signals the analyzer to not process + further input. (Jon Siwek) + + * Add new event for TCP content file write failures: + "contents_file_write_failure". (Jon Siwek) + +2.2-beta-57 | 2013-10-11 17:23:25 -0700 + + * Improve Broxygen end-of-sentence detection. (Jon Siwek) + +2.2-beta-55 | 2013-10-10 13:36:38 -0700 + + * A couple of new TLS extension numbers. (Bernhard Amann) + + * Suport for three more new TLS ciphers. (Bernhard Amann) + + * Removing ICSI notary from default site config. (Robin Sommer) + +2.2-beta-51 | 2013-10-07 17:33:56 -0700 + + * Polishing the reference and scripting sections of the manual. + (Robin Sommer) + + * Fixing the historical CHANGES record. (Robin Sommer) + + * Updating copyright notice. (Robin Sommer) + +2.2-beta-38 | 2013-10-02 11:03:29 -0700 + + * Fix uninitialized (or unused) fields. (Jon Siwek) + + * Remove logically dead code. (Jon Siwek) + + * Remove dead/unfinished code in unary not expression. (Jon Siwek) + + * Fix logic for failed DNS TXT lookups. (Jon Siwek) + + * A couple null ptr checks. (Jon Siwek) + + * Improve return value checking and error handling. (Jon Siwek) + + * Remove unused variable assignments. (Jon Siwek) + + * Prevent division/modulo by zero in scripts. (Jon Siwek) + + * Fix unintentional always-false condition. (Jon Siwek) + + * Fix invalidated iterator usage. (Jon Siwek) + + * Fix DNS_Mgr iterator mismatch. (Jon Siwek) + + * Set safe umask when creating script profiler tmp files. (Jon Siwek) + + * Fix nesting/indent level whitespace mismatch. (Jon Siwek) + + * Add checks to avoid improper negative values use. (Jon Siwek) + +2.2-beta-18 | 2013-10-02 10:28:17 -0700 + + * Add support for further TLS cipher suites. (Bernhard Amann) + +2.2-beta-13 | 2013-10-01 11:31:55 -0700 + + * Updating bifcl usage message. (Robin Sommer) + + * Fix bifcl getopt() usage. (Jon Siwek) + +2.2-beta-8 | 2013-09-28 11:16:29 -0700 + + * Fix a "make doc" warning. (Daniel Thayer) + +2.2-beta-4 | 2013-09-24 13:23:30 -0700 + + * Fix for setting REPO in Makefile. (Robin Sommer) + + * Whitespace fix. (Robin Sommer) + + * Removing :doc: roles so that we can render this with docutils + directly. (Robin Sommer) + +2.2-beta | 2013-09-23 20:57:48 -0700 + + * Update 'make dist' target. (Jon Siwek) + +2.1-1387 | 2013-09-23 11:54:48 -0700 + + * Change submodules to fixed URL. (Jon Siwek) + + * Updating NEWS. (Robin Sommer) + + * Fixing an always false condition. (Robin Sommer) + + * Fix required for compiling with clang 3.3. (Robin Sommer) + +2.1-1377 | 2013-09-20 14:38:15 -0700 + + * Updates to the scripting introduction. (Scott Runnels) + + * Kill raw input reader's child by process group to reliably clean + it up. (Jon Siwek) + +2.1-1368 | 2013-09-19 20:07:57 -0700 + + * Add more links in the GeoLocation document (Daniel Thayer) + +2.1-1364 | 2013-09-19 15:12:08 -0700 + + * Add links to Intelligence Framework documentation. (Daniel Thayer) + + * Update Mozilla root CA list. (Bernhard Amann, Jon Siwek) + + * Update documentation of required packages. (Daniel Thayer) + +2.1-1359 | 2013-09-18 15:01:50 -0700 + + * Make client and server random available on script-level. Addresses + BIT-950. (Eric Wustrow) + +2.1-1357 | 2013-09-18 14:58:52 -0700 + + * Update HLL API and its documentation. (Bernhard Amann) + + * Fix case in HLL where hll_error_margin could be undefined. + (Bernhard Amann) + +2.1-1352 | 2013-09-18 14:42:28 -0700 + + * Fix a number of compiler warnings. (Daniel Thayer) + + * Fix cmake warning about ENABLE_PERFTOOLS not being used. (Daniel + Thayer) + +2.1-1344 | 2013-09-16 16:20:55 -0500 + + * Refactor Analyzer::AddChildAnalyzer and usages. (Jon Siwek) + + * Minor refactor to SSL BinPAC grammer. (Jon Siwek) + + * Minor refactor to Broxygen enum comments. (Jon Siwek) + + * Fix possible (unlikely) use of uninitialized value. (Jon Siwek) + + * Fix/improve dereference-before-null-checks. (Jon Siwek) + + * Fix out-of-bounds memory accesses, and remove a + variable-length-array usage. (Jon Siwek) + + * Fix potential mem leak. (Jon Siwek) + + * Fix double-free and deallocator mismatch. (Jon Siwek) + + * Fix another function val reference counting bug. (Jon Siwek) + +2.1-1335 | 2013-09-12 16:13:53 -0500 + + * Documentation fixes (Daniel Thayer, Jon Siwek) + + * Fix various potential memory leaks. (Jon Siwek) + + * Fix significant memory leak in function unserialization. (Jon Siwek) + + * Fix use-after-free and invalid/mismatch deallocator bugs. (Jon Siwek) + + * Fixed an issue with the HLL_UNIQUE SumStats plugin that caused a reporter error. (Seth Hall) + + * Make the notice $actions field have a default empty set to avoid having to check for it's presence. (Seth Hall) + + * Fix signatures that use identifiers of type table. (Jon Siwek) + + * Fix memory leak if a DNS request fails to be made. (Jon Siwek) + + * Fix memory leak in DNS TXT lookups. (Jon Siwek) + + * Fix raw execution input reader's signal blocking which resulted in lingering processes. (Jon Siwek) + +2.1-1306 | 2013-08-31 16:06:05 -0700 + + * Reorganized and signifcantly extended documentation. This includes + two new chapters contributed by Scott Runnels. + +2.1-1216 | 2013-08-31 10:39:40 -0700 + + + * Support for probabilistic set cardinality, using the HyperLogLog + algorithm. (Bernhard Amann, Soumya Basu) + + Bro now provides the following BiFs: + + hll_cardinality_init(err: double, confidence: double): opaque of cardinality + hll_cardinality_add(handle: opaque of cardinality, elem: any): bool + hll_cardinality_merge_into(handle1: opaque of cardinality, handle2: opaque of cardinality): bool + hll_cardinality_estimate(handle: opaque of cardinality): double + hll_cardinality_copy(handle: opaque of cardinality): opaque of cardinality + +2.1-1154 | 2013-08-30 08:27:45 -0700 + + * Fix global opaque val segfault. Addresses BIT-1071. (Jon Siwek) + + * Fix malloc/delete mismatch. (Jon Siwek) + + * Fix invalid pointer dereference in AsciiFormatter. (Jon Siwek) + +2.1-1150 | 2013-08-29 13:43:01 -0700 + + * Fix input framework memory leaks. (Jon Siwek) + + * Fix memory leak in SOCKS analyzer for bad addr types. (Jon Siwek) + + * Fix Bloom filter memory leaks. (Jon Siwek) + +2.1-1144 | 2013-08-28 18:51:06 -0700 + + * Add bits_per_uid unit test. Addresses BIT-1016. (Jon Siwek) + + * UID optimizations. Addresses BIT-1016. (Jon Siwek) + + * Added a $unique_max field to Reducers for the SumStats::UNIQUE + calculation, and using the new option in scan.bro and the FTP + bruteforce detection. (Seth Hall) + +2.1-1137 | 2013-08-27 13:26:44 -0700 + + * Add BiF hexstr_to_bytestring() that does exactly the opposite of + bytestring_to_hexstr(). (Bernhard Amann) + +2.1-1135 | 2013-08-27 12:16:26 -0700 + + * More SumStats fixes. (Seth Hall) + + * Increase UIDs to 96 bits. (Jon Siwek) + + - The bit-length is adjustable via redef'ing bits_per_uid. + + - Prefix 'C' is added to connection UIDS (including IP tunnels) + and 'F' to files. + + Addresses BIT-1016. + +2.1-1128 | 2013-08-24 10:27:29 -0700 + + * Remove code relict in input framework. (Jon Siwek) + + * Fix documentation for mkdir BIF. (Jon Siwek) + + * File extraction tweaks. (Jon Siwek) + + - Default extraction limit of 100MB now provided via a tuning + script loaded in local.bro so that command-line Bro is unlimited + by default. + + - Extraction directory is now created on request of file + extraction rather than unconditionally in bro_init(). (Jon + Siwek) + +2.1-1124 | 2013-08-23 16:33:52 -0700 + + * Fixed a number of object bugs DNP3 analyzer. (Hui Lin) + +2.1-1122 | 2013-08-22 16:52:27 -0700 + + * Use macros to create file analyzer plugin classes. (Jon Siwek) + + * Add options to limit extracted file sizes w/ 100MB default. (Jon + Siwek) + +2.1-1117 | 2013-08-22 08:44:12 -0700 + + * A number of input framework fixes and corresponding test stability + improvements. (Jon Siwek) + + * Make memory leak tests able to time out. (Jon Siwek) + + * Fix a compiler warning regarding strncat misuse. (Jon Siwek) + +2.1-1103 | 2013-08-21 19:11:34 -0400 + + * A number of sumstats fixes. (Seth Hall, Vlad Grigorescu) + + * Fix memory leak w/ when statements. Addresses BIT-1058. (Jon + Siwek) + + * Switching to relative submodule paths (Robin Sommer) + +2.1-1089 | 2013-08-19 11:25:11 -0700 + + * Fix bloom filters' dependence on size_t. (Jon Siwek, Matthias + Vallentin). + +2.1-1081 | 2013-08-19 11:19:33 -0700 + + * New BiF levenshtein_distance() to compute the Levenshtein distance + between two strings. (Anthony Kasza) + +2.1-1078 | 2013-08-19 09:29:30 -0700 + + * Moving sqlite code into new external 3rdparty submodule. (Bernhard + Amann) + +2.1-1074 | 2013-08-14 10:29:54 -0700 + + * Fix timer type enum and timer name array mismatch. (Jon Siwek) + +2.1-1072 | 2013-08-14 10:28:51 -0700 + + * Adding the unified2 analyzer that reads unified2 files from disk, + turning them into events. (Seth Hall) + + * Fixing intel framework tests. (Seth Hall) + +2.1-1059 | 2013-08-13 23:52:41 -0400 + + * Add file name support to intel framework. (Seth Hall) + + * Add file support to intel framework and slightly restructure + intel http handling. (Seth Hall) + +2.1-1052 | 2013-08-12 14:38:14 -0700 + + * Fixing bug in DNP3 analyzer flagged by compiler warning. (Robin + Sommer) + +2.1-1050 | 2013-08-12 11:37:44 -0700 + + * Experimental DNP3 analyzer. This includes only very basic + script-level support at the moment, but quite a number of events + are provided. (Hui Lin, Robin Sommer) + +2.1-1041 | 2013-08-09 15:32:22 -0700 + + * Update coverage baselines for canonical load order of scripts. + (Jon Siwek) + +2.1-1039 | 2013-08-09 15:30:15 -0700 + + * Fix mem leak in DHCP analyzer. (Jon Siwek) + + * Fix a unit test outdated by recent sumstats changes. (Jon Siwek) + +2.1-1036 | 2013-08-05 17:29:11 -0400 + + * Fix the SSL infinite loop I just created. (Seth Hall) + +2.1-1035 | 2013-08-05 16:44:50 -0400 + + * Change to SSL log delay to cause the log to write even if delay times out. (Seth Hall) + +2.1-1034 | 2013-08-03 20:27:43 -0700 + + * A set of DHCP extensions. (Vlad Grigorescu) + + - Leases are logged to dhcp.log as they are seen. + - scripts/policy/protocols/dhcp/known-devices-and-hostnames.bro + - Added DPD sig. + +2.1-1027 | 2013-08-03 01:57:37 -0400 + + * Fix a major memory issue in the SumStats framework. + +2.1-1026 | 2013-08-02 22:35:09 -0400 + + * Fix the SumStats top-k plugin and test. (Seth Hall) + + * Rework of SumStats API to reduce high instantaneous memory + use on clusters. (Seth Hall) + + * Large update for the SumStats framework. + + - On-demand access to sumstats results through "return from" + functions named SumStats::request and Sumstats::request_key. + Both functions are tested in standalone and clustered modes. + + - $name field has returned to SumStats which simplifies cluster + code and makes the on-demand access stuff possible. + + - Clustered results can only be collected for 1 minute from their + time of creation now instead of time of last read. + + - Thresholds use doubles instead of counts everywhere now. + + - Calculation dependency resolution occurs at start up time now + instead of doing it at observation time which provide a minor + cpu performance improvement. A new plugin registration mechanism + was created to support this change. + + - AppStats now has a minimal doc string and is broken into hook-based + plugins. + + - AppStats and traceroute detection added to local.bro (Seth Hall) + +2.1-1009 | 2013-08-02 17:19:08 -0700 + + * A number of exec module and raw input reader fixes. (Jon Siwek) + +2.1-1007 | 2013-08-01 15:41:54 -0700 + + * More function documentation. (Bernhard Amann) + +2.1-1004 | 2013-08-01 14:37:43 -0700 + + * Adding a probabilistic data structure for computing "top k" + elements. (Bernhard Amann) + + The corresponding functions are: + + topk_init(size: count): opaque of topk + topk_add(handle: opaque of topk, value: any) + topk_get_top(handle: opaque of topk, k: count) + topk_count(handle: opaque of topk, value: any): count + topk_epsilon(handle: opaque of topk, value: any): count + topk_size(handle: opaque of topk): count + topk_sum(handle: opaque of topk): count + topk_merge(handle1: opaque of topk, handle2: opaque of topk) + topk_merge_prune(handle1: opaque of topk, handle2: opaque of topk) + +2.1-971 | 2013-08-01 13:28:32 -0700 + + * Fix some build errors. (Jon Siwek) + + * Internal refactoring of how plugin components are tagged/managed. + (Jon Siwek) + + * Fix various documentation, mostly related to file analysis. (Jon + Siwek) + + * Changing the Bloom filter hashing so that it's independent of + CompositeHash. (Robin Sommer) + +2.1-951 | 2013-08-01 11:19:23 -0400 + + * Small fix to deal with a bug in the SSL log delay mechanism. + +2.1-948 | 2013-07-31 20:08:28 -0700 + + * Fix segfault caused by merging an empty bloom-filter with a + bloom-filter already containing values. (Bernhard Amann) + +2.1-945 | 2013-07-30 10:05:10 -0700 + + * Make hashers serializable. (Matthias Vallentin) + + * Add docs and use default value for hasher names. (Matthias + Vallentin) + +2.1-939 | 2013-07-29 15:42:38 -0700 + + * Added Exec, Dir, and ActiveHTTP modules. (Seth Hall) + + base/utils/exec.bro provides a module to start external processes + asynchronously and retrieve their output on termination. + base/utils/dir.bro uses it to monitor a directory for changes, and + base/utils/active-http.bro for providing an interface for querying + remote web servers. + +2.1-930 | 2013-07-29 15:06:07 -0700 + + * Major file analysis overhaul in naming and appearance, along with + fixes and test updates. (Seth Hall and Jon Siwek) + + Includes: + + * Added protocol description functions that provide a super + compressed log representation. (Seth Hall) + + * Added mime types to http.log (Seth Hall) + + * Add jar files to the default MHR lookups. (Seth Hall) + + * Adding CAB files for MHR checking. (Seth Hall) + + * Improve malware hash registry script. + + - Include a link to a virustotal search in the notice sub message field. + - Give all information returned from Team Cymru in the notice message. + - Add more file types to match on to the default set. + + * Make the custom libmagic database a git submodule. + + * Add an is_orig parameter to file_over_new_connection event. + + * Recorrected the module name to Files. + + * Added Files::analyzer_name to get a more readable name for a + file analyzer. + + * Improved and just overall better handled multipart mime + transfers in HTTP and SMTP. HTTP now has orig_fuids and + resp_fuids log fields since multiple "files" can be transferred + with multipart mime in a single request/response pair. SMTP has + an fuids field which has file unique IDs for all parts + transferred. FTP and IRC have a log field named fuid added + because only a single file can be transferred per irc and ftp + log line. + +2.1-895 | 2013-07-29 14:07:35 -0700 + + * Adding a test for a DNSKEY RR. (Robin Sommer) + +2.1-894 | 2013-07-29 16:44:41 -0400 + + * Updates for the Intel Framework. (Seth Hall) + + - policy/frameworks/intel/seen is the new location for the + scripts that push data into the intel framework for checking. + + - The new policy/frameworks/intel/do_notice script adds an + example mechanism for data driven notices. + + - Remove the Intel insertion after heuristically detecting SSH + bruteforcing. + + - Intel importing format has changed (refer to docs). + + - All string matching is now case insensitive. + + - SMTP intel script has been updated to extract email + addresses correctly. + + - Small fix sneaking into the smtp base script to actually + extract individual email addresses in the To: field + correctly. + + 2.1-888 | 2013-07-25 12:02:41 -0700 * Protection about broken traces with empty pcap headers. (Matt @@ -47,7 +1728,7 @@ make it deterministic. (Robin Sommer) * Small raw reader tweaks that got left our earlier. (Robin Sommer) - + 2.1-814 | 2013-07-15 18:18:20 -0700 * Fixing raw reader crash when accessing nonexistant file, and @@ -173,12 +1854,12 @@ input data on to the file analysis framework. (Jon Siwek) * File analysis framework interface simplifications. (Jon Siwek) - + - Remove script-layer data input interface (will be managed directly by input framework later). - Only track files internally by file id hash. Chance of collision - too small to justify also tracking unique file string. + too small to justify also tracking unique file string. 2.1-741 | 2013-06-07 17:28:50 -0700 @@ -229,14 +1910,14 @@ 2.1-659 | 2013-05-24 17:24:18 -0700 * Fix broken/missing documentation. (Jon Siwek) - + * Fixing test that would fail without ES/curl support. (Robin Sommer) 2.1-656 | 2013-05-17 15:58:07 -0700 * Fix mutex lock problem for writers. (Bernhard Amann) - + 2.1-654 | 2013-05-17 13:49:52 -0700 * Tweaks to sqlite3 configuration to address threading issues. @@ -254,9 +1935,9 @@ 2.1-647 | 2013-05-17 07:47:14 -0700 * Fixing Broxygen generation to have BROMAGIC set. (Robin Sommer) - + * Fix for 'fchmod undeclared here' on FreeBSD. (Robin Sommer) - + * CMake policy fix to avoid errors with older versions. (Robin Sommer) @@ -369,7 +2050,7 @@ 2.1-386 | 2013-03-22 12:41:50 -0700 * Added reverse() function to strings.bif. (Yun Zheng Hu) - + 2.1-384 | 2013-03-22 12:10:14 -0700 * Fix record constructors in table initializer indices. Addresses @@ -378,16 +2059,16 @@ 2.1-382 | 2013-03-22 12:01:34 -0700 * Add support for 802.1ah (Q-in-Q). Addresses #641. (Seth Hall) - + 2.1-380 | 2013-03-18 12:18:10 -0700 * Fix gcc compile warnings in base64 encoder and benchmark reader. (Bernhard Amann) - + 2.1-377 | 2013-03-17 17:36:09 -0700 * Fixing potential leak in DNS error case. (Vlad Grigorescu) - + 2.1-375 | 2013-03-17 13:14:26 -0700 * Add base64 encoding functionality, including new BiFs @@ -399,14 +2080,14 @@ * Adding a test for extract-certs-pem.pem. (Robin Sommer) * Renaming Base64Decoder to Base64Converter. (Robin Sommer) - + 2.1-366 | 2013-03-17 12:35:59 -0700 * Correctly handle DNS lookups for software version ranges. (Seth Hall) * Improvements to vulnerable software detection. (Seth Hall) - + - Add a DNS based updating method. This needs to be tested still. @@ -440,9 +2121,9 @@ 2.1-351 | 2013-03-07 13:27:29 -0800 * Fix new/delete mismatch. Addresses #958. (Jacob Baines) - + * Fix compiler warnings. (Jon Siwek) - + 2.1-347 | 2013-03-06 16:48:44 -0800 * Remove unused parameter from vector assignment method. (Bernhard Amann) @@ -491,9 +2172,9 @@ 2.1-328 | 2013-02-05 01:34:29 -0500 - * New script to query the ICSI Certificate Notary + * New script to query the ICSI Certificate Notary (http://notary.icsi.berkeley.edu/) over DNS and add information - to the SSL log at runtime. (Matthias Vallentin) + to the SSL log at runtime. (Matthias Vallentin) * Add delayed logging to SSL base scripts. (Matthias Vallentin) @@ -544,7 +2225,7 @@ * Changing test=suite's btest call to use "-j" instead of "-j 5". (Robin Sommer) - + * Require "case" blocks to end with either "break", "return", or a new "fallthrough" statement that passes control on to the subsequent case. This gives us the best mix of safety, @@ -560,7 +2241,7 @@ ElasticSearch writer. (Gilbert Clark) * Removing unused class member. (Robin Sommer) - + * Add opaque type-ignoring for the accept_unsupported_types input framework option. (Bernhard Amann) @@ -605,7 +2286,7 @@ sha256_*, and entropy_*, respectively. Note that these functions have changed their signatures to work with opaques types rather than global state as it was before. - + 2.1-240 | 2012-12-20 15:21:07 -0800 * Improve error for invalid use of types as values. Addresses #923. @@ -730,7 +2411,7 @@ 2.1-195 | 2012-12-03 14:50:33 -0800 * Catching out-of-memory in patricia tree code. (Bill Parker) - + 2.1-194 | 2012-12-03 14:36:26 -0800 * Renaming ASCII writer filter option 'only_single_header_row' to @@ -791,7 +2472,7 @@ Hall) * Adding NEWS placeholder for hooks and CSV mode. (Robin Sommer) - + 2.1-178 | 2012-11-23 19:35:32 -0800 * The ASCII writer now supports a new filter config option @@ -846,7 +2527,7 @@ 2.1-112 | 2012-11-05 13:58:20 -0800 - * New base script for detecting cases of checksum offloading. + * New base script for detecting cases of checksum offloading. Reporter messages will now tell if one has bad checksums. (Seth Hall) @@ -856,9 +2537,9 @@ 2.1-109 | 2012-11-05 13:39:34 -0800 * Add detection rate threshold for MHR. (Vlad Grigorescu) - + * lookup_hostname_txt fixes. (Vlad Grigorescu) - + 2.1-104 | 2012-11-01 10:37:50 -0700 * A new built-in function lookup_hostname_txt() provides support for @@ -983,7 +2664,7 @@ Addresses #877. (Jon Siwek) * Add --with-curl option to ./configure. Addresses #877. (Jon Siwek) - + 2.1-61 | 2012-10-12 09:32:48 -0700 * Fix bug in the input framework: the config table did not work. @@ -1026,7 +2707,7 @@ * Remove deprecated script functionality (see NEWS for details). (Daniel Thayer) - + 2.1-39 | 2012-09-29 14:09:16 -0700 * Reliability adjustments to istate tests with network @@ -1038,7 +2719,7 @@ an error. (Daniel Thayer) * Fix parsing of large integers on 32-bit systems. (Daniel Thayer) - + * Serialize language.when unit test with the "comm" group. (Jon Siwek) @@ -1049,7 +2730,7 @@ 2.1-26 | 2012-09-23 08:46:03 -0700 * Add an item to FAQ page about broctl options. (Daniel Thayer) - + * Add more language tests. We now have tests of all built-in Bro data types (including different representations of constant values, and max./min. values), keywords, and operators (including @@ -1072,7 +2753,7 @@ * Adjusting some unit tests that do cluster communication. (Jon Siwek) * Small change to non-blocking DNS initialization. (Jon Siwek) - + * Reorder a few statements in scan.l to make 1.5msecs etc work. Adresses #872. (Bernhard Amann) @@ -1104,9 +2785,9 @@ Siwek) * Parse 64-bit consts in Bro scripts correctly. (Bernhard Amann) - + * Output 64-bit counts correctly on 32-bit machines (Bernhard Amann) - + * Input framework fixes, including: (Bernhard Amann) - One of the change events got the wrong parameters. @@ -1120,7 +2801,7 @@ - Hashing of lines just containing zero-length-strings was broken. - Make set_separators different from , work for input framework. - + - Input framework was not handling counts and ints out of 32-bit-range correctly. @@ -1128,20 +2809,20 @@ the line, log it, and continue. * Update documentation for builtin types. (Daniel Thayer) - + - Add missing description of interval "msec" unit. - + - Improved description of pattern by clarifying the issue of operand order and difference between exact and embedded matching. * Documentation fixes for signature 'eval' conditions. (Jon Siwek) - + * Remove orphaned 1.5 unit tests. (Jon Siwek) * Add type checking for signature 'eval' condition functions. (Jon Siwek) - + * Adding an identifier to the SMTP blocklist notices for duplicate suppression. (Seth Hall) @@ -1172,7 +2853,7 @@ 2.1-beta-31 | 2012-08-21 15:46:05 -0700 * Tweak to rotate-custom.bro unit test. (Jon Siwek) - + * Ignore small mem leak every rotation interval for dataseries logs. (Jon Siwek) @@ -1227,13 +2908,13 @@ 2.1-beta-6 | 2012-08-10 12:22:52 -0700 * Fix bug in input framework with an edge case. (Bernhard Amann) - + * Fix small bug in input framework test script. (Bernhard Amann) - + 2.1-beta-3 | 2012-08-03 10:46:49 -0700 * Merge branch 'master' of ssh://git.bro-ids.org/bro (Robin Sommer) - + * Fix configure script to exit with non-zero status on error (Jon Siwek) @@ -1284,7 +2965,7 @@ * Input framework: Make want_record=T the default for events (Bernhard Amann) - + * Changing the start/end markers in logs to open/close now reflecting wall clock. (Robin Sommer) @@ -1295,16 +2976,16 @@ * New test for input framework that fails to find a file. (Robin Sommer) - + * Improving error handling for threads. (Robin Sommer) - + * Tweaking the custom-rotate test to produce stable output. (Robin Sommer) 2.0-884 | 2012-07-26 14:33:21 -0700 * Add comprehensive error handling for close() calls. (Jon Siwek) - + * Add more test cases for input framework. (Bernhard Amann) * Input framework: make error output for non-matching event types @@ -1313,14 +2994,14 @@ 2.0-877 | 2012-07-25 17:20:34 -0700 * Fix double close() in FilerSerializer class. (Jon Siwek) - + * Fix build warnings. (Daniel Thayer) * Fixes to ElasticSearch plugin to make libcurl handle http responses correctly. (Seth Hall) * Fixing FreeBSD compiler error. (Robin Sommer) - + * Silencing compiler warnings. (Robin Sommer) 2.0-871 | 2012-07-25 13:08:00 -0700 @@ -1339,7 +3020,7 @@ 2.0-866 | 2012-07-24 16:02:07 -0700 * Correct a typo in usage message. (Daniel Thayer) - + * Fix file permissions of log files (which were created with execute permissions after a recent change). (Daniel Thayer) @@ -4001,30 +5682,67 @@ - ClamAV support has been removed, which has been non-functional for a while already. -1.5.2.7 Sun Sep 12 19:39:49 PDT 2010 +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ -- Addressed a number of lint nits (Vern Paxson). +1.5.3 Thu Mar 3 08:55:11 PST 2011 +- Removing aux/broctl/policy/cluster-addrs.hot.bro from the + distribution. The script is no longer needed and could in fact break + an installation because it redefines an old variable that has went + away. (Robin Sommer) -1.5.2.6 Sun Sep 12 17:00:13 PDT 2010 +- Smarter way to increase the communication module's pipe's socket + buffer size, resulting in a value closer to the allowed maximum. + (Craig Leres) + +- BroControl now also maintains links from the log archive to the + current set of logs when running in standalone mode. (Robin Sommer) + +- Bug fix for a file descriptor leak in the remote communication + module. (Scott Campbell) + +- Bug fix for BroControl to now activate trace-summary's sampling in + cluster mode, but not anymore in standalone mode. (Robin Sommer) + +- Broccoli updates: + + * Accept empty strings ("") as values in the configuration file. + (Craig Leres) + + * Support for specifying a separate host key for SSL-enabled + operation, with documentation update. (Craig Leres) + +1.5.2 Wed Jan 12 17:34:55 PST 2011 + +- Portability fixes for --enable-int64 (Vern Paxson). + +- Bug fix for Active Mapping support (Kevin Lo). + +- Broccoli compiler warning fixes (Kevin Lo). + +- Bug fixes for --enable-int64 and for avoiding bogus statistics / + bad memory references when generating profiling information upon + exit (Vern Paxson). + +- Bug fixes for terminating connections (Tyler Schoenke and Vern Paxson). + +- Removed now-quite-stale SSHv1 overflow detection, as it's more prone + to false positives than useful detection (Vern Paxson). - The SWIG file now explicitly lists those pieces from broccoli.h which it wants to wrap, rather than just including all of broccoli.h (Robin Sommer). - This fixes the problem that the SWIG bindings depend on what configure - finds out about the availability of libpcap even though the corresponding - functions don't need to be wrapped anyway. -- http-header.bro now includes a global include_header: set[string] - (Robin Sommer). If it contains any strings, then only those headers - will be processed. If left empty, then you continue to get the current - behavior of processing all headers. +- http-header.bro now includes a global "include_header: set[string]" If it + contains any strings, then only those headers will be processed. If left + empty, then you continue to get the current behavior of processing all + headers. (Robin Sommer). - Several changes to drop.bro (Robin Sommer): - * If true, the new flag Drop::dont_drop_locals indicates that + * If True, the new flag Drop::dont_drop_locals indicates that local hosts should never be dropped. On by default. - * If true, the new flag Drop::debugging activates extensive debugging + * If True, the new flag Drop::debugging activates extensive debugging output for the catch-and-release logic. Off by default. * The timeout for tracking dropping information is now 1 day @@ -4038,14 +5756,14 @@ Sommer). - The HTTP analyzer no longer attempts to track Server/User-Agent - versions, as these are hugely voluminous (Seth Hall). Ideally this - would still be available as an option for someone who truly wants - the full set. + versions, as these are hugely voluminous (Seth Hall). - HTTP and SMTP no longer have extra-short inactivity timeouts, as these were too often leading to premature expiration of a connection (Robin Sommer). +- Tracking of HTTP refer[r]er's by setting log_referrer. (Vern Paxson). + - The "rst" tool (aux/rst/) now takes an optional "-I " argument that instructs it to inject as payload rather than sending a RST packet (Vern Paxson). must be NUL-terminated, and the NUL is not @@ -4054,10040 +5772,113 @@ - Bug fix for crashes in the DNS analyzer when processing replies for which no request was seen (Robin Sommer). - -1.5.2.5 Mon Jul 19 16:20:58 PDT 2010 - -- Removed now-quite-stale SSHv1 overflow detection, as it's more prone - to false positives than useful detection (Vern Paxson). - - -1.5.2.4 Fri Jun 4 16:02:11 PDT 2010 - -- Bug fixes for terminating connections (Tyler Schoenke and Vern Paxson). - - -1.5.2.3 Wed Mar 24 18:23:57 PDT 2010 - -- Bug fixes for --enable-int64 and for avoiding bogus statistics / - bad memory references when generating profiling information upon - exit (Vern Paxson). - - -1.5.2.2 Tue Jan 12 12:33:42 PST 2010 - -- Broccoli compiler warning fixes (Kevin Lo). - - -1.5.2.1 Sun Jan 10 16:59:01 PST 2010 - -- Bug fix for Active Mapping support (Kevin Lo). - - -1.5.2 Sat Dec 26 18:38:37 PST 2009 - -- Portability fixes for --enable-int64 (Vern Paxson). - - -1.5.1 Fri Dec 18 15:17:12 PST 2009 - -- Due to a Python configuration problem, the original 1.5 distribution - did not include the BroControl component, which also introduced a - portability problem for CentOS. These issues have now been fixed (Robin - Sommer and Vern Paxson). - - -1.5 Wed Dec 16 21:28:47 PST 2009 - -- Bro now comes with a new framework, BroControl, for managing an - operational Bro setup, including support for installation, configuration, - and maintainance tasks such a log archival and mail notification. The - framework transparently supports both traditional standalone setups as - well as cluster installations in which multiple Bro boxes coordinate to - analyze a high-volume network link. - - See aux/broctl/README for more information about BroControl. - - Note, BroControl supersedes the older BroLite system, which is no longer - supported and has been deprecated for a while now. - -- Numerous adjustments to DPD = dynamic protocol detection (Robin Sommer): - - o The Analyzer::ProtocolViolation?() method can now be passed the - offending data (which POP3, SMTP, and FTP now do). This information - is added to the "reason" string passed to the script level. - - o SMTP now more accurately reports violations. - - o FTP stops processing when client & server successfully negotiate - an AUTH scheme (leading to subsequent encryption). - - o Analyzer::ProtocolViolation() is virtual, and - TCP_ApplicationAnalyzer() overrides it to not report violations - for any partial connections, because very likely these arise just - due to the analyzer getting confused. - - o TCP::IsPartial() returns true if any side did not start with - a SYN packet (used to be just be for the originator). - - o The connection_state_remove handler in conn.bro now has a higher - &priority so that other handlers for the same event can use - determine_service() and see any changes it performs. - - o DynDisable:max_volume specifies a volume limit (default 10K). - Once a connection exceeds this limit, further protocol - limitations will neither raise ProtocolViolation notices nor - cause the analyzer to be disabled. - - o The event engine no longer raises protocol_violation events for - TCP connections which had gaps, as these have proven too unreliable. - (Note that, ideally, the *analyzers* should avoid reporting - protocol_violations when they can't reliably parse a connection - anymore after a gap; but many don't.) - -- A set of new script functions provide support for incrementally computing - MD5 checksums (Seth Hall). - - md5_hash_init(index: any): bool - Initializes an incremental hashing instance. "index" is - a value of arbitrary type, used to identify this particular - instance (you can have multiple concurrent instances by - using different index values). Returns T on success, - F on failure (such as the index is already in use). - - md5_hash_update(index: any, data: string): bool - For the given hashing instance, updates the hash - based on the given data. Returns T on success, F on - failure (such as the index has not been initialized). - - md5_hash_finish(index: any): string - Returns the MD5-printable hash for the given index - and terminates the instance, or the string "" if the - index was not active. - -- Bro now supports a believed-to-be-robust mechanism for estimating the - proportion of traffic that it failed to capture ("measurement drops"), - which can arise due to overload in either Bro itself, the kernel's - packet filter, or problems with the link tapping mechanism (Vern Paxson). - The event engine can generate estimates for either live traffic or what - was previously recorded in a trace file, though traces subject to some - forms of selective omission (such as skipping over parts of a connection - to reduce storage) can lead to erroneous values. - - The estimates are based on observing gaps in TCP data streams, and - come in two forms: the rate at which such gaps appear, and the relative - volume of data missing due to the gaps. (We've found however that the - volume-based estimator is not robust due to occasional packets with - incorrect sequence numbers, so this estimator is off by default.) - - The easy way to get the estimates is to load capture-loss.bro. - By default, it generates a CaptureLossSummary notice upon Bro's exit, - which can look like: - - 1130222759.344066 CaptureLossSummary estimated rate = 0.00089124 / 0.000970997 (events/bytes) - - If the estimated loss is none, however, it suppresses this notice, - unless you redef CaptureLoss::summary_if_none to T. - - You can also get finer-grained access by defining a "gap_report" - event handler and redef'ing gap_report_freq to a non-zero interval - (such as "10 sec"). This event allows you to pinpoint regions in - time that exhibit significant capture loss. See capture-loss.bro - for an example of a handler for this event. - - Finally, these changes include a number of fixes to Bro's - ack_above_hole/content_gap analysis, which is now significantly - more robust. - -- GeoIP support now supports ASN lookups via the built-in - function lookup_asn(a: addr): count (Scott Campbell and Seth Hall). - -- The GeoIP built-in's lookup_location() and lookup_asn() now - support IPv6 (Seth Hall). Note, the current GeoIP distribution - doesn't include any IPv6 databases, so for now these won't succeed, - but the hooks are in place for when databases become available. - -- lookup_location() now falls back back to the country database if - the city database isn't available (Seth Hall). - -- The new SuccessfulPasswordGuessing Notice is generated when a host - has been seen attempting password guessing (currently only for FTP - sessions) and then successfully logs in (Royal Chan). You can control the - threshold for such reports in terms of how many attempts the host must - have made by redef'ing the variable password_guessing_success_threshhold, - which defaults to 20. - -- The new script http-detect-passwd.bro analyzes the Web items returned - for fetches that appear to be accessing the passwd file (Akhil Dhar). - It generates a PasswordFullFetch Notice if it appears that the item - includes a full password file, and PasswordShadowFetch if it looks like - a shadowed password file. - -- The new built-in - - system_env(cmd: string, env: table[string] of string) - - works like system(), but puts the table entries into the environment - before invoking the command (Robin Sommer). Each in the table - creates an environment variable of the form "BRO_ARG_", whose - value is the corresponding table entry. - -- The new script function - - execute_with_notice(cmd: string, notice_info) - - executes "cmd" with an environment containing the fields of the - notice_info, i.e., the information associated with a Notice (Robin Sommer). - Per the new system_env() function above, the environment variables appear - as "BRO_ARG_", where is the field tag as it appears in - notice.log when you enable use_tagging. - -- The new built-in enable_raw_output(file) acts the same as - the attribute &raw_output (Seth Hall). - -- The new built-in file_opened(f: file) event is generated any time Bro - opens a script-level file (Justin Azoff). You can use this, for example, - if you want to ensure that a given file has a prelude in it such as - human-readable headers, even when the file is rotated. - -- The notice_info record has a new field - - aux: table[string] of string &optional - - which you can use for information specific to a given type of notice - (Robin Sommer). Entries in $aux appear as "aux_" tags in notice.log. - -- Another new notice_info record field is the boolean do_alarm (default=T), - which, if set to F, overides a notice action otherwise specifying to - generate an alarm (Robin Sommer). In other words, if do_alarm is F, no - alarm will be generated independent of the notice action. - - This is a work-around for the fact that we can't specify more than one - action. In particular, we couldn't NOTICE_DROP but then *not* alarm, - which we now can by returning NOTICE_DROP yet setting do_alarm to F. - -- The notice_info record field $dropped now appears in the tagged output - format if true (Robin Sommer). - -- NOTICEs relating to scan detection now no longer include the connection - that triggered the notice, as it really doesn't contain any useful - information, given that the particular trigger simply depends on the - detection algorithm and its parameters (Robin Sommer). However, we do - explicitly set $p (port number) in the notice, and also $n with the - number of attempts. - -- drop.bro now hardwires a Catch-and-Release redrop after seeing one - connection from a previously-dropped-but-already-released host - (Robin Sommer). - -- drop.bro now provides some new hooks (Robin Sommer): - - event address_dropped(a: addr) - Generated when an address has been dropped. - - event address_restored(a: addr) - Generated when connectivity to an address has been restored, - such as using the Catch-and-Release mechanism. - - event address_cleared(a: addr) - Generated when an address that was dropped in the past is - no longer being monitored looking for new connections - (as part of the Catch-and-Release mechanism). - -- The new built-in function - - hexdump(data_str: string) : string - - returns a hex dump representation of the given input data (Christian - Kreibich). The dump renders 16 bytes per line, with hex on the left and - ASCII (where printable) on the right. - -- Bro's notion of when a TCP connection begins now dastes to the first - instance of an initial SYN packet seen, rather than the last (Gregor Maier). - -- The Time Machine script tm-contents.bro now generates - - event contents_saved: event(c: connection, orig_file: string, - resp_file: string) - - when the content of a connection has been completely saved to disk - (Robin Sommer). - -- The mime.bro script now exports the MIME header callback table, and also - marks it as &redef'able so you can modify its entries (Matthias Vallentin). - The mime_log file is also now exported. - -- A new signature file, policy/sigs/http-bots.sig, contains signatures - to detect some of the current HTTP based controlled bot families (Seth Hall). - -- The signature engine's HTTP pattern matching has been fixed (Seth Hall) - to align with the documentation at: - - http://www.bro-ids.org/wiki/index.php/Reference_Manual:_Signatures#Content_conditions - - In particular, the content condition "http" is now referred to as - "http-request" (though "http" still works for backward compatibility), - "http-request-header" and "http-reply-header" now provide access to - headers seen in only one direction, and similarly for "http-request-body" - and "http-reply-body". (This latter is still accessible as "http-body" - for backwards compatibility.) - -- The new script variable max_remote_events_processed: count (default 10) - sets a limit on the number of remote events processed in each round, - before tending to other inputs (Robin Sommer). - -- If you set the new script variable dump_used_event_handlers to T, - then on startup Bro dumps out all of the event handlers that the - loaded set of scripts can invoke (Matthias Vallenti). - -- Summaries for DNS PTR scanning now use a separate Notice, - DNS_PTR_Scan_Summary, rather than overloading DNS_PTR_Scan (Robin Sommer). - -- scan.bro now provides a table skip_dest_server_ports: set[addr, port] - which lists servers (defined as an address and a port) excluded from - scan detection computations (Craig Leres and Jay Krous). - -- When redefining values on the command line directly (using var=value), - quotation marks are now implicit only if "var" is a variable of type - string (Christian Kreibich). This allows other string-like values - (such as enum's) to be passed as well. - -- scan.bro now explicitly loads conn.bro so that it can itself - be loaded independently (Robin Sommer). - -- login.bro depends on scan.bro (because of tracking authentication - "scans"), so now it explicitly loads it (Vern Paxson). - -- UDP_datagram_length_mismatch is now by default flagged just once per - originating host rather than once per connection, as it can generate - tons of messages (Vern Paxson). - -- Removed now-long-boring flagging of access to Solaris "listen" - service as "hot" (Vern Paxson). - -- Removal of libedit, since libreadline provides similar functionality - (Christian Kreibich). - -- Added scripts missing from distribution: dce.bro, ncp.bro, and smb.bro - (Vern Paxson). - -- ssh.bro now exports ssh_ports (Seth Hall) - -- A number of improvements to inter-Bro communication (Robin Sommer). - - (1) Remote communication now no longer includes location information for - serialized objects; that removes quite a bit of redundacy from the network - traffic. - - (2) The new option 'remote_check_sync_consistency" disables the cross-check - on the receiving side of &synchronized state of whether the current value - of a variable has the value expected by the sender. Transmitting the - original values in addition to the updates generates quite a bit CPU & - network load in some cases (in particular, a table of tables). The default - for remote_check_sync_consistency is off, and so far that in particular - seems to reduce the proxy's load quite a bit. - - (3) Complete overhaul of the internal caching of serialized objects. The - objective of the caching is avoid retransmitting already sent values over - and over again. It turns out, however, that some objects are very stable - and hardly change or get replaced (e.g., Bro types); while other change - all the time and are hardly reused some time later (e.g., Vals). Now - we maintain *two* caches independently for these types of objects; one - with a low turn-over one and another with a high one. This should reduce - CPU load on both sender and receiver sides. - - The new scheme is only used if both communicating Bros support it; with - older Bros, as well as with Broccoli, we continue using the old scheme. - -- Some reworking of remote printing (Robin Sommer), as follows. Bro now - uses a new interprocess message rather than print_hook events, to better - manage buffering and associated load (these can produce failures depending - on system configuration; see remote.log). A number of timeouts and - buffer sizes have been tuned. Internally, EINTR errors are now treated - separately from EAGAIN. Finally, even with remote_check_sync_consistency=F, - one type of consistency check was still being done; this is no longer - the case. - -- The DNS analyzer now generates events (dns_query_reply/dns_rejected) - for replies with zero questions (Robin Sommer). - -- Perftools support for incompatible changes in the 1.0 API (Robin Sommer). - -- Rearranged (generally reducing, though not always) some state timeouts - associated with scan detection (Robin Sommer). In addition, when a - scanning address crosses ignore_scanners_threshold (meaning that it will - be ignored from now on anyway), it gets discarded from all state-tracking - tables. Finally, the ignore_scanners_threshold now applies all kinds - of scans, not just address scans. - -- Substantial Broccoli updates, including a new initialization requirement - that breaks backward compatibility, support for enqueueing serialized - event data for transmission, and OpenSSL threadsafe initialization. - See aux/broccoli/ChangeLog for details (Christian Kreibich, Robin - Sommer, and Matthias Vallentin). - -- Broccoli hashtable optimisation. See aux/broccoli/ChangeLog for - details (Christian Kreibich & Matthias Vallentin). - -- Broccoli memory leak fixed, see aux/broccoli/ChangeLog for details - (Christian Kreibich). - -- Broccoli: updates to bropipe tool (Steve Chan and Robin Sommer). - -- Bug fixes for Broccoli Python bindings (Robin Sommer and Matthias Vallentin). - -- Fixed nasty bug due to module scoping that completely kept stepping-stone - detection from working (Vern Paxson). - -- A serious bug in the packet sorter has been fixed (Robin Sommer). - -- Bug fix for extra NULs getting embedded in escaped strings (Seth Hall). - -- Bug fix for HTTP messages that use "Connection: close" rather than length - headers, which yielded erroneous reassembled messages with \r\n's when - only \n's were present (Bernhard Ager). - -- Fix for reporting on ICMP flows that are expired from the flow table - (Vern Paxson). Previously there was a race condition if the flow - was flushed prior to its summary timer expiring. - -- The -l option (list the scripts that Bro loads) now correctly prints - scripts loaded by the prefix mechanism, and uses indentation to indicate - the load hierarchy (Robin Sommer). - -- A bug has been fixed (really, worked around) in drop.bro that prevented - dropped addresses from being properly restored (Robin Sommer). - -- Fixes for deadlocking problems in the Broccoli protocol. See - aux/broccoli/ChangeLog for details (Christian Kreibich & Robin Sommer). - -- Bug fix for DNS analyzer on 64-bit machines (Gregor Maier). - -- Bug fix for asynchronous DNS lookups to prevent some successful lookups - being reported as timed out (Robin Sommer). - -- Bug fix for tracking line numbers associated with compound statements - (Po-Ching Lin). - -- Fix for a rare condition in which the main Bro process couldn't kill - its child process (Robin Sommer). - -- Fix for file rotation when the underlying file is deleted before the - timer expires (Robin Sommer). - -- Fix for potential crash when communication connections break down, - and also for releasing cached objects (Robin Sommer). - -- Fix for default table entries computed by function invocation to not - cache previous results (Robin Sommer). - -- Fix for Bro's internal DNS resolution (Scott Campbell and Robin Sommer). - -- Portability fix for DAG packet capture (Gregor Maier). - -- Portability fix for --enable-brov6 (Robin Sommer). - -- Portability fixes for FreeBSD (Vern Paxson). - -- A work around for new_packet() crashing on IPv6 packets (Vern Paxson). - For now, IPv6 packets are skipped. Also, for fragments the event handler - is now only called for the fully reassembled packet. - -- The new configuration option --disable-nbdns supports disabling non-blocking - DNS at configure time (Sean McCreary). Note, there are some known problems - with it in some environments. - -- A number of configuration fixes and enhancements (Christian Kreibich - and Robin Sommer). - -- Consistency nit for the configuration process (Seth Hall). - -- A number of reference-counting and other memory management fixes - (Robin Sommer). - -- Bug fix for inter-Bro communication lockup (Seth Hall and Robin Sommer). - -- Bug fix for computing TCP payload length in new_packet event (Lothar Braun). - -- Bug fix for sending boolean True values via Broccoli (Seth Hall). - -- make distcheck fix to clean up .bif.bro files (Christian Kreibich). - -- Bug fix for DPD's recognition of SSLv2 connections (Seth Hall). - -- Bug fix for &default for tables indexed by subnets (Seth Hall). - -- A bug has been fixed that could crash Bro when you called get_event_peer() - after a remote connection had already disppeared (Robin Sommer). - -- Introduced a work-around for crashes that occur when Bro exits - due to handling a signal (Robin Sommer). - -- Bug fix for checkpoint.bro - don't schedule timers for times that - aren't actually in the future (Robin Sommer). - -- Hostname formatting fix for anon.bro (Fabian Schneider). - -- Bug fix for redundant .log extension in Time Machine log file - (reported by CS Lee). - -- Removed now-outdated special-casing of Linux reporting of packet filter - statistics (Peter Wurzinger and Robin Sommer). - -- A number of memory leaks fixed (Robin Sommer). - -- Addressed warnings from newer versions of g++ (Robin Sommer and Vern Paxson). - -- Fixed an invocation issue in the ca-create script that prevented it from - working with recent OpenSSL versions (Craig Leres & Christian Kreibich). - -- Comment fixed in drop-adapt (Justin Azoff). - -- Duplicate code removed from Val (Seth Hall). - - -1.4 Fri Oct 17 11:08:52 PDT 2008 - -- We are no longer supporting a previous Bro release as the "stable" - version. Rather, the model now is that the current public release will - aim for increasing stability (occasionally updated with fixes), and those - who wish to use a "bleeding-edge" snapshot can do so via access to the - public SVN source code repository, as explained at - - http://bro-ids.org/wiki/index.php/Subversion#Public_Access - - Note that all previous releases remain available from the download page; - what is changing is that we no longer commit to support for the most - recent of these. - -- We have clarified the copyright statement that covers most of the - code to remove the "advertising clause" that derived from older - BSD licenses, and we have removed copyright wording from most source - code files. See COPYING for the current wording and a list of - files that retain their own copyright notices. - -- Bro now supports analyzing NetFlow v5 data, i.e., from Cisco routers - (Bernhard Ager). NetFlow can be useful for intrusion detection as it - allows analysis of traffic from many different points in the network. - Bro can now read NetFlow data from a UDP socket, as well as (mostly - for debugging purposes) from a file in a specialized format. You can - create these files with the programs given in aux/nftools. - - Command line switches: - - -Y|--netflow :[=] | read flow from socket - - This is the usual way of getting NetFlow data into Bro by - opening a UDP socket on : and reading all incoming - packets. Setting the to 0.0.0.0 should work on most - platforms. Optionally you may set an identifier for the - source - useful if there are many different sources you want - to analyze in parallel. This might also be necessary if you - want to use this feature with a clustered Bro. - - Examples: - bro -Y 0.0.0.0:5555 netflow - bro -i eth0 -Y 10.0.0.1:1234=src1 brolite netflow - - -y|--flowfile [=] - - Used to read from a file. You can optionally include an - identifier for the source. - - Examples: - bro -y myflowfile netflow - bro -y myflowfile=src1 otherflowfile=src2 netflow - - Netflow Events: - - event netflow_v5_header(h: nf_v5_header) - - Generated upon reading a new NetFlow PDU, as summarized in the - argument. The field h_id gives the flow source identifier and - a serial number. You can use this field to associate subsequent - netflow_v5_record events with their header. - - event netflow_v5_record (r: nf_v5_record) - - Every record within a NFv5 PDU generates a corresponding - netflow_v5_record() event. The relatively complex timestamp - format of NFv5 is already converted to Bro's time type, and - the TCP header flags are separated into bools. - - The distribution includes an example analysis script, netflow.bro. - It simply dumps received NetFlow records. If netflow_restitch is T - (the default), then Bro performs flow restitching as well, and two - script variables become relevant: - - global netflow_finished_conn_expire = 310 sec &redef; - - specifies how long to wait for additional flow records after - a RST or FIN for - - const netflow_table_expire = 31 min; - - Its setting only affects table declarations, and therefore - cannot be usefully redef'd. - - Auxiliary programs: - - Bro uses a custom format for flow data stored in files, - to enable preserving timestamps of the PDU arrivals and the - exporter's IP address. The tools nfcollector and ftwire2bro - in aux/nftools/ provide ways to manipulate the Bro NF file - format. The first dumps NetFlow data from a UDP socket to - stdout or to a file in Bro format. The second converts NetFlow - data in "wire" format to Bro format, and, while doing so, - fakes up the exporter's IP address and timestamp. You can get - "wire" format from normal flow-tools files, e.g., by using - 'flow-export -f 4'. Please note that the Bro format is just - a hack to allow for easier debugging. Therefore the format - is not in fact platform independent, and not suitable for data - storage. - -- A new DHCP analyzer generates the following events (Po-Ching Lin): - - event dhcp_discover(c: connection, msg: dhcp_msg, req_addr: addr) - event dhcp_offer(c: connection, msg: dhcp_msg, mask: addr, - event dhcp_request(c: connection, msg: dhcp_msg, - event dhcp_decline(c: connection, msg: dhcp_msg) - event dhcp_ack(c: connection, msg: dhcp_msg, mask: addr, - event dhcp_nak(c: connection, msg: dhcp_msg) - event dhcp_release(c: connection, msg: dhcp_msg) - event dhcp_inform(c: connection, msg: dhcp_msg) - - where dhcp_msg values look like: - - type dhcp_msg: record { - op: count; # 1 = BOOTREQUEST, 2 = BOOTREPLY - m_type: count; # the type of DHCP message - xid: count; # transaction ID of a DHCP session - h_addr: string; # hardware address of the client - ciaddr: addr; # original IP address of the client - yiaddr: addr; # IP address assigned to the client - }; - - See dhcp.bro for the corresponding analysis script (which could - probably use some refinements). - - Note, this analyzer is implemented using BinPAC, so you will need - to specify --use-binpac to activate it. - -- A BitTorrent analyzer is now available (Nadi Sarrar). See the policy - scripts bittorrent.bro and bt-tracker.bro for the events generated for - analyzing transfers and tracker dialogs, respectively. - -- The "Bro Lite" configuration is now deprecated and will not in - general be supported (Robin Sommer & Vern Paxson). - -- "make install" now only installs a core set of files (Robin Sommer). - Policy files are now installed in /share/bro/* (or whatever - configure determines $datadir to be), which is now in Bro's default - search path. It creates a directory /share/bro/site for local - policy files, and the default BROPATH is extended to include this. The - default path no longer includes policy/local. You can install the - additional files used by the (now deprecated) "Bro Lite" configuration - using "make install-brolite". - -- Substantial updates to Broccoli, including support for container - types (tables and sets) as well as a new metadata structure for event - callbacks, facilitating truly generic event handler implementations - (Christian Kreibich, Seth Hall and Robin Sommer). See aux/broccoli/ChangeLog - for details. - -- Extensive changes to allow Bro to process packets captured in the - past intermingled with those captured in real-time (Matthias Vallentin - and Robin Sommer). This operation reflects combining Bro with use of - "Time Machine" functionality for packet capture. - -- We have unfortunately had to disable support for configuring Bro - to use ClamAV, since it turns out that the key interface we need - for processing blocks of memory directly rather than whole files - is no longer supported by the package, and in fact was buggy even - when it was (Robin Sommer). - -- The new signature option "http-body //" matches - on the body data of HTTP entities (Robin Sommer). The matching is - done after decompressing the body, if necessary. - -- The new built-in function identify_data(data: string, return_mime: bool) - analyzes the string "data" and returns its type according to libmagic, - if installed (Seth Hall). The second argument controls whether it should - be returned as a MIME-type or just an identifying string. For example, - identify_data("MZpofigu", F) returns the string "MS-DOS executable", and - print identify_data("MZpofigu", T) returns "application/x-dosexec". - -- The new analysis script http-identified-files.bro identifies the - type of items returned by Web servers using libMagic (if available) - and generates notices for interesting types and mismatches between - URLs and types (Seth Hall). - - You configure it using two variables. watched_mime_types is a pattern - (default /application\/x-dosexec/ | /application\/x-executable/ ) for - which any MIME type matching the pattern generates a HTTP_WatchedMIMEType - notice. - - mime_types_extensions is a table mapping strings to patterns specifying - how URLs for the given MIME type should appear. (Ideally, this would - be a table mapping patterns to patterns, but Bro doesn't currently support - that.) It defaults to: - - ["application/x-dosexec"] = /\.([eE][xX][eE]|[dD][lL][lL])/ - - i.e., do Windows executables end in .exe or .dll. - - You can also redef the pattern ignored_urls to specify URLs that should - not generate complaints. It defaults to matching Windows Update. - -- The new script http-extract-items.bro extracts the items from HTTP - traffic into individual files (Vern Paxson). Files are named: - - .._._. - - where is a redef'able prefix (default: "http-item"), is a - number uniquely identifying the item, the next four are describe the - connection tuple, and is "orig" if the item was transferred - from the originator to the responder, "resp" otherwise. - -- The workings of how Bro interfaces to external programs for dropping/ - restoring connectivity of misbehaving hosts has been significantly - reworked (Brian Tierney and Robin Sommer). - - First, dropping decisions used to be made directly by analyzer scripts, - such as scan.bro directly calling drop_address(). Now instead the - scripts generate Notices and then the notice policy can have an - action of NOTICE_DROP to codify that the response to the given Notice - is to drop the source. The new notice_action_filter of drop_source - drops the source of notices, and drop_source_and_terminate both - drops the source and terminates the corresponding connection. - - So, to drop all sources triggering a specific notice, one can now, e.g., - write: - - redef notice_action_filters += { [Hot::SSH_Overflow] = drop_source }; - - Related to this change, notice_info has a new field $dropped, set to - true if the Notice triggered a (successful) drop. - - Second, by redef'ing Drop::use_catch_release to T (default F) you can - activate "catch-and-release" logic. You use this mode when you need to - manage a limited number of possible blocks, or to build in automatic - "forgiveness" in situations where blocked sources might become benign - (such as due to dynamic IP addresses). If a source has been idle for - Drop::drop_time, then it is unblocked. However, if it is again seen as - block-worthy, then it is blocked for an interval of Drop::long_drop_time. - - Third, ICMP scanning is now reported by its own notice, ICMPAddressScan, - rather than Scan::AddressScan. - -- Google's perftools have replaced mpatrol for leak-checking and - heap-profiling (Robin Sommer). If Bro is compiled with --enable-perftools - and configure finds the perftools, there are two command-line options - available: - - -m turns on leak checking of the main packet loop, with some - uninteresting leaks are suppressed. Currently, with one - exception (the RPC analyzer; problem not yet found), it reports - no leaks when running the test suite. - - -M turns on heap profiling: Bro will take a snapshot of the heap - before starting the main packet loop and another one when - finished. These snapshots can then be analyzed with pprof. - - For more information about the perftools see - - http://code.google.com/p/google-perftools - -- Notice tags are now generated in a pseudo-unique fashion that, with high - probability, ensures that tags generated by separate Bro processes don't - clash when logged to a common location, such as for a Bro cluster (Robin - Sommer). Tags are now string's rather than count's, and are associated - with all notices, not just that are connection-related. You can however - redef the string notice_tag_prefix or the function new_notice_tag to - further control how such tags are generated. - -- Four new built-ins for type conversion (Robin Sommer): - - function double_to_interval(d: double): interval - function addr_to_count(a: addr): count - function port_to_count(p: port): count - function count_to_port(c: count, t: transport_proto): port - -- Many policy scripts have been modified to use modules & scoping - (Robin Sommer and Matthias Vallentin), which may require updates to - existing scripts/refinements. - -- The new script variable dpd_conn_logs (default F), if true, changes the - semantics of the service field in connection logs written to conn.log, - as follows (Robin Sommer). It becomes a comma-separated list of analyzers - confirmed by DPD to parse the connection's payload. If no analyzer could - confirm its protocol, but the connection uses a well-known port, the - service is the name of the port with "?" appended (e.g., "http?"), as - long as the corresponding analyzer has not declined the connection. - In addition, ftp-data sessions are labeled "ftp-data" and portmapper - connections are labeled with the specific method-call (just as before). - - dpd_conn_logs defaults to F because the change in semantics may break - scripts that parse conn.logs; but it will likely change to the default - in the future. With dpd_conn_logs turned off, conn logs are generated - as they used to be, with a few rare exceptions (with previous versions, - the service field was sometimes determined while the connection was still - alive; now it's always determined at the time when the conn.log entry - is written out). - -- The SSL analyzer has been rewritten using BinPAC, with a number of - robustness improvements (Tobias Kiesling). It currently is only used - if you execute with --use-binpac. - -- Python bindings for Broccoli are now available in - aux/broccoli/bindings/python/ (Robin Sommer). See README/README.html - in that director for details. - -- The new "auth" option in remote.bro indicates whether a given side is - considered "authoritative" for shared state, in which case it sends its - initial state to &sync'ed peers (Robin Sommer). When two peers synchronize - their state, one side sends its current set of state to the other as - soon as the remote connection is established. The one sending the state - used to be the one who has been running longer; now it can also be - explicitly set via the "auth" flag in the Remote::Destination. - -- Two new tuning parameters for scan.bro (Robin Sommer): - - ignore_scanners_threshold (default 0): - - If a host has scanned more than this many hosts, it is completely - excluded from further scan detection. 0 disables. - - addr_scan_trigger (default 0): - - A host is only tracked for address scanning once it has contacted - this many different hosts. Primarily intended for using a two-stage - scan detection with a Bro cluster: first, each node searches locally - for scanners by looking for hosts contacting more than - addr_scan_trigger destinations. Those hosts which do are then - globally tracked throughout the cluster by &synchronizing the scan - detector tables. - -- When Bro serializes functions, it now does so by default using only - their name, rather than their full value (Robin Sommer). This prevents - propagation of expiration functions associated with tables and sets. - Note, currently there is no mechanism provided to switch from the - default behavior, but the internal hooks are in place to do so. - -- The new built-in variable trace_output_file gives the name of the -w - output trace file (Robin Sommer). - -- Bro no longer installs new file rotation timers when shutting down - (Robin Sommer). - -- The new policy scripts remote-print-id{,-reply}.bro support convenient - access to printing the identifiers of a remote Bro (Robin Sommer). - You use the script remote-print-id.bro to request and receive the - printing; the remote Bro must have loaded remote-print-id-reply.bro - in order to process the request. - - Example use: - - bro -e 'redef PrintID::dst="" PrintID::id=""' - remote-print-id - -- scan.bro has been heavily modified to better support distributed scan - analysis (Matthias Vallentin and Robin Sommer). - -- The check for unused event handlers is now turned off by default - (Robin Sommer). To enable, use "redef check_for_unused_event_handlers = T". - -- The new script drop.bro has been split off from scan.bro to isolate - the logic concerning dropping addresses to block scans (Robin Sommer). - -- The new -l flag lists each script as it is loaded (Robin Sommer). - -- Textual descriptions of identifiers now include their attributes - (Robin Sommer). - -- The new predefined function prefixed_id() returns a session identifier with - its peer-ID prepended if it's associated with a remote Bro (Robin Sommer). - This is now used when generating writing log files. - -- remote.bro now assigns a priority of -10 to its bro_init() event handler - to allow others a chance to modify destinations (Robin Sommer). - -- A large number of BinPAC updates (Ruoming Pang and Robin Sommer). - -- The new built-in type_name(v): string returns the name of the type - of the value v (Vern Paxson). For example, "typename(5.2)" returns - "double". This function is mainly for internal debugging (i.e., - finding mismatches between values generated by the event engine - versus how their type is expected by the script layer). - -- The new built-in str_shell_escape() does some basic escaping on strings - that will be passed to system() (Christian Kreibich). Note, this function - isn't ready (robust enough) for routine use, however. - -- The new built-in disable_print_hook(file) acts the same as - the attribute &disable_print_hook (Robin Sommer). - -- The new script terminate-connection.bro factors out the terminate_connection() - functionality that used to be in conn.bro (Robin Sommer). - -- The new attribute &group= can be associated with event handlers - to group them together into a set that can be manipulated as a whole - (Robin Sommer). is a string reflecting the name given to the group. - - The built-in enable_event_group(group: string) turns on all the analyzers - in a given group, and disable_event_group(group: string) deactivates them. - -- The new attribute &raw_output applies to variables of type file, disabling - escaping of non-printable characters (Seth Hall). - -- You can now iterate over the characters in a string value using - a "for" loop, e.g., "for ( c in str ) ..." (Robin Sommer). - -- The new built-in - - function cat_sep%(sep: string, def: string, ...%): string - - works similarly to cat(), except that it (a) separates the values - by "sep" and (b) substitutes "def" for empty strings (Seth Hall). - -- The function string_escape() now takes a string of characters to escape - rather than a single character (Robin Sommer). Each character in the - string is preceded by '\' in the return value (also any embedded '\'s, - as before). - -- The new built-in function global_ids() returns a table of all global - identifiers along with associated information (Robin Sommer). The - return value has type table[string] of script_id, indexed by the name - of the identifier and yielding records with the following fields: - - type script_id: record { - type_name: string; - exported: bool; - constant: bool; - enum_constant: bool; - redefinable: bool; - value: any &optional; - }; - -- The new script function find_last(str: string, re: pattern) returns - the last occurrence of the given pattern in the given string, or - an empty string if no match (Robin Sommer). Note that this function - returns the match that starts at the largest index in the string, which - is not necessarily the longest match. For example, a pattern of /.*/ - will return just the final character in the string. - -- The new script variable record_all_packets, if redef'd to T (default F), - instructs Bro to record every packet it processes (Robin Sommer). - Prior to introducing this variable, Bro applied a few heuristics to - reduce recording volume. Setting this variable also causes packets - to be recorded very early in processing, which can be helpful for - debugging crashes. - -- If the new script flag ssl_log_ciphers is set to T (default), ssl.bro - logs the ciphers seen (Robin Sommer). - -- Much more expanded Time Machine support, now located in - policy/time-machine/ (Robin Sommer), - -- The new command line option --status-file (alias -U) specifies - the name of a file into which Bro will write an indicator of its current - processing status (Robin Sommer). Possible values include "INITIALIZING", - "RUNNING", "TERMINATING", "TERMINATED". - -- The new policy script targeted-scan.bro looks for repeated access from - the same source to the same server, to detect things like SSH - password-guessing attacks (Jim Mellander). - -- The "alternative" style for printing strings (i.e., a fmt() argument - of "%As") now renders the raw string, other than escape-expanding - embedded NULs (Vern Paxson). This change may be temporary, pending - development of more fine-grained control over string rendering. - -- For now we have removed the %S functionality for fmt() (Robin Sommer). - %S was meant to print "raw" strings, but later processing of such - printing still introduces artifacts. - -- GeoIP information now includes latitude and longitude (Seth Hall). - -- ssh.bro now supports the variable skip_processing_after_handshake - which directs the event engine to omit any further processing of an - SSH connection after its initial handshake (Seth Hall and Robin Sommer). - This can help with performance for large file transfers but precludes - some kinds of analyses (e.g., tracking connection size). This change - also adds a scope of "SSH". - -- Email notification of notices now allows for separate destinations - depending on notice type (in particular, a regular mail destination - versus a pager destination), and also escapes the notice to prevent - injection attacks (Seth Hall and Robin Sommer). - -- The new policy script conn-flood.bro is a simple connection-flooding - detector, mainly meant as a demonstration (Robin Sommer). - -- A large number of additions to the TLS/SSL known-ciphers suite (Seth Hall). - -- Serialization now uses 64-bit IDs to cache items rather than 32-bit, - for robustness during long-running execution (Robin Sommer). - -- The new script variable tcp_max_initial_window specifies, for flows - for which ACKs have never been seen, the maximum volume of initial - data after which Bro will assume that it is seeing only one side - of the connection and will not buffer data for consistency checking - awaiting the later arrival of ACKs (Robin Sommer). It defaults to 4 KB. - (Note, this used to be an internal value, so the behavior is not new.) - Set to 0 to turn off this functionality and have Bro attempt to - track all such flows. - -- The new script variable tcp_max_above_hole_without_any_acks specifies, - for flows for which ACKs have never been seen, the maximum volume of - data above a sequence hole that Bro will tolerate for a connection - before giving up on tracking the flow (Robin Sommer). It defaults to 4 KB. - (Note, this differs from tcp_max_initial_window in that this threshold - applies to sequence holes rather than the beginning of flows. Like - tcp_max_initial_window this used to be an internal value.) Set to 0 to - turn off this functionality. - -- The new script variable tcp_excessive_data_without_further_acks specifies - a threshold similar to tcp_max_above_hole_without_any_acks, but for - flows for which Bro has seen ACKs (Robin Sommer). It defaults to 10 MB. - Set to 0 to turn off the functionality. - -- Equal signs ("=") in text for notices are now escaped when using the - tagged format to keep them unambiguous from the "=" delimiters - (Robin Sommer). - -- The final tallies for notices are now processed as NoticeTally - NOTICE's rather than directly alarm'd (Robin Sommer). - -- WeirdActivity notices now include an associated connection when appropriate - (Robin Sommer). - -- Support for large (> 2^32 bytes) pcap trace files (Po-Ching Lin). - -- Scoped names ("...::...") are now allowed in signature "eval" - constructs (Christian Kreibich). - -- scan.bro is now decoupled from conn.bro, i.e., you can @load the - latter without getting the former (Vern Paxson). As part of this - change, the logic to invoke TRW is now in scan.bro. - -- weird.bro has been updated with a number of missing Weird's (Vern Paxson). - -- If when using inter-Bro communication the child Bro process terminates, - it now also terminates the parent process (Robin Sommer). - -- BinPAC analyzers now interoperate with DPD (Robin Sommer). - -- Some http.bro processing options are now exported so they can be - accessed in other scripts (Robin Sommer). - -- SMTP analysis now applies to port 587/tcp as well as 25/tcp (Robin Sommer). - -- $conn is now set in ServerFound notices (Robin Sommer). - -- You can now create empty sets and tables using set() and table(), - i.e., the usual set/table constructors with no arguments (Vern Paxson). - By themselves, these have an unspecified type - you can't use them - directly other than to assign them. For example, - - local bad_guys: set[addr]; - ... - bad_guys = set(); # start over assuming no bad guys - -- A number of scripts have been (slightly) simplified to use the - new empty set()/table() constructors (Vern Paxson). Note that - these still aren't usable for field assignments in record constructors, - nor for attributes like &default = ... - -- Removed unused syntax for declaring sets based on a list of initial - values (Vern Paxson). - -- set() and table() can now be used as arguments to function calls - (Vern Paxson). - -- The vestigial &match attribute has been removed. - -- POP3 is now recognized using Dynamic Protocol Detection (Seth Hall). - -- The new event expected_connection_seen(c: connection, a: AnalyzerTag) - is generated whenever a connection is seen for which we have previously - scheduled an analyzer via expect_connection() (Robin Sommer). - -- The new built-in capture_state_updates logs all changes applied to - &synchronized variables, in a fashion similar to the capture_events() - built-in (Robin Sommer). An accompanying policy script, - capture-state-updates.bro, turns this on to the file state-updates.bst. - -- If the new script variable suppress_local_output is set (default: F), - Bro suppresses printing to local files if there's a receiver for - print_hook events (Robin Sommer). This option is however ignored - for files with a &disable_print_hook attribute. - -- The new notice action filter function file_if_remote specifies - that notices from sent from remote source addresses should - have an action NOTICE_FILE (Robin Sommer). - -- The new notice action filter function file_local_bro_notices specifies - that notices generated by the local Bro instance (as opposed to a - remote peer) should have an action NOTICE_FILE (Robin Sommer). - -- An arbitrary tag can now be past to post-processors for log rotation - (Robin Sommer). - -- Default inactivity timeouts for interactive services shortened to - 1 hour (Robin Sommer). - -- The scanning variables distinct_{peers,ports,low_ports} are now - redef'able (Robin Sommer). - -- The new -S (--summary-only) option for site-report.pl directs to - only generate connection summaries (Brian Tierney) - -- More useful default config file for edit-brorule.pl (Brian Tierney). - -- Bro now includes a test suite in testing/istate/ for its "independent - state" functionality (Robin Sommer). - -- Support for parallel builds via make -j (Christian Kreibich). - -- Bro's default search path now includes includes policy/sigs/ and - policy/time-machine/ (Robin Sommer). - -- Bro's internal processing of interprocess communication has been - significantly overhauled to prevent potentially fatal race conditions - (Robin Sommer). - -- Bro now checks calls to fmt() at compile-time to ensure that the - correct number of arguments are present (Vern Paxson). This is useful - in addition to Bro's run-time checking for arguments matching their - corresponding format-specifiers in the case of rarely-executed statements - that might not generate such run-time checks in routine testing. - -- The ports associated with Telnet and Rlogin are now redef'able (Robin Sommer). - -- MIME processing now removes leading whitespace from MIME headers - (Sanmeet Bhatia and Robin Sommer). - -- TCP "weird" events reported by the connection compressor now match - (other than a few rare corner-cases) those produced for normal TCP - processing (rmkml and Robin Sommer). - -- Added Scan::suppress_UDP_scan_checks to control false positives - on scan detection in environments with P2P protocols that use UDP - (Vern Paxson). - -- The internal analyzer interface now includes an EndOfData() method that - analyzers can use to report that all of a message has been delivered - (Robin Sommer). - -- Fix for a significant memory leak in processing UDP when using -w - (Robin Sommer). Note: this change turns off by default trace rewriting - for generic UDP traffic. - -- Two serious regular expression bugs fixed (Vern Paxson). In the - first, searching for a regular expression inside a string would - fail if the pattern occurred only after an embedded newline. In - the second, insufficient buffer was allocated when compiling regular - expressions, leading to memory corruption. - -- Base64 decoding bug fixes (Christian Kreibich and Ruoming Pang). - -- Automatic rotation of files is now disabled for contents files written - by the TCP reassembler, which otherwise leads to mangled files - (Robin Sommer). - -- Bro now ships with an updated version of libpcap (0.9.8), which hopefully - fixes problems managing trace files > 4 GB in size. - -- Significant bug fixes for gzip- and deflate-encoded Web items (Robin Sommer). - -- Bug fix for secondary-filter.bro (Vern Paxson). - -- Removed a naming ambiguity regarding TCP states (Vern Paxson). - -- Bug fix for signature scanner not matching all of its input (Vern Paxson). - -- Bug fix for using port values in signatures (Robin Sommer). - -- Minor policy script tweaks: state management for weird's, processing - of Notice tags associated with connections, and dependencies for - irc-bot.bro (Robin Sommer). - -- aux/ portability fixes (Vern Paxson). - -- Workarounds added for a BinPAC deficiency, which is that code in %cleanup - clauses can also be executed during recovery from exceptions when parsing - new data. This means that any delete's or Unref()'s need to also set the - corresponding pointer to nil (Vern Paxson). - -- Bug fix for crashes with the non-BinPAC SSL analyzer (Robin Sommer). - -- Tweak to peer-status.bro since Bro now requires events to be - declared prior to reference in a "schedule" statement (Robin Sommer). - -- The signature keyword "enable" now optionally accepts the syntax - "foo:bar" to specify "activate analyzer bar as a child of analyzer foo" - (Robin Sommer). This is used for example for an XML-over-HTTP analyzer - that's in the works. - -- irc-bot-syslog.bro now uses open_log_file() for its log file (including - the logging suffix) rather than a direct open (Vern Paxson). - -- Bug fix for tracking Blaster across a Bro Cluster (Robin Sommer). - -- Bug fix for the HTTP BinPAC analyzer chopping the trailing character - off of HTTP headers when generating the http_all_headers event (Gregor Maier). - -- Bug fix for HTTP chunked items for which the chunk size line was terminated - by CRLF but the CR and LF came in separate packets (Gregor Maier). - -- A bug has been fixed that would cause partial lines (for line-oriented - protocols) to fail to be processed when a connection terminated - (Robin Sommer). - -- Bro no longer treats a signal arriving before a previous signal has - been processed as fatal, nor does it attempt processing of a termination - signal if seemingly there are no race conditions to worry about - (Robin Sommer). Both of these changes are an attempt to improve - Bro's robustness. - -- Fix for attributes such as &encrypt not working in initial declarations - but only in later redef's (Seth Hall and Robin Sommer). - -- Fixes for memory leaks in SSL processing (Seth Hall and Robin Sommer). - -- Fix for POP3 analyzer to not treat lines like "." as message - terminators (Robin Sommer). - -- Bug fix for crashes arising from nil pointers in list expressions - (Seth Hall and Robin Sommer). - -- Bug fix: a signature's "enable" would activate the corresponding analyzer - even if no event handlers were defined for it (Robin Sommer). - -- Bug fixes to prevent crashes when mixing set_contents_file() with - subsequent explicit close(), and to ensure all data written to - file upon connection tear-down (Gert Doering and Robin Sommer). - -- Configuration support for MacPorts and Fink package management systems - (Christian Kreibich & Vern Paxson). - -- Communication-only Bro's now send out email alarms (Robin Sommer). - -- Writes to a file that fail due are now run-time errors rather than - fatal internal errors, since often these occur due to the disk - being full (Robin Sommer). - -- Byte-order bug fix for lookup_location() (Robin Sommer). - -- BinPAC portability fix for 64-bit machines (Bernhard Ager and Robin Sommer). - -- Portability fixes for newer versions of gcc (Jan Gerrit Goebel and - Robin Sommer). - -- Some support for porting to Solaris (Stephan Toggweiler). - -- Connection compressor bug fix for source and destination having the - same IP address, such as when monitoring loopback (Robin Sommer). - -- Connection compressor bug fix for connections with multiple SYNs - (Robin Sommer). - -- Bug fix for using already-declared local variables for looping - over vectors in a "for" loop (Robin Sommer & Vern Paxson). - -- Bug fix for not processing truncated UDP packets (Tom Kho and Robin Sommer). - -- Bounds-check added to BinPAC-generated code (Tom Kho and Robin Sommer). - -- Bug fix for checking whether an IPv6 address is part of a subnet - (Seth Hall). - -- Bug fixes for crashes relating to asynchronous DNS lookups performed - at start-up (Robin Sommer). These changes also lowered the timeout - before assuming failure from 20 seconds down to 5 seconds. - -- Portability and const-ness fixes (Kevin Lo and Robin Sommer). - -- Suppression of some content-gap complaints when running on traces - that have been filtered down to only TCP control packets (Robin Sommer). - -- Removed unnecessary dependency in notice-action-filters.bro - that led to errors when loading icmp.bro by itself (Vern Paxson). - -- Bug fix for potential infinite loop in client communiation (Robin Sommer). - -- Bug fix in reference counting that could eventually lead to roll-over - (Robin Sommer). - -- Bug fix in communication initialization (Robin Sommer). - -- Internal documentation fix: timers are specified using absolute time, - not relative (Robin Sommer). - -- Performance improvement for built-in find_all() function when running - on large strings (Robin Sommer). - -- Memory leak fixes (Robin Sommer, Bernhard Ager, Christian Kreibich). - -- Bug fix for error recovery when encountering an unknown link layer - (Bernhard Ager). - -- Bug fix for reversing client & server in a connection (Po-Ching Lin). - -- Bug fix for packet_contents when capture length exceeds the IP payload - length due to Ethernet frame padding (Christian Kreibich). - -- Bug fix for tcp_packet event erroneously including Ethernet padding - in its contents (Vern Paxson). - -- Bug fix for lookup_connection built-in (Seth Hall). - -- Portability nit for libedit tarball (Vern Paxson). - -- Broccoli portability fix for NetBSD (Christoph Leuzinger). - -- Type-checking for script-level event invocation was completedly broken - - now fixed (Vern Paxson). - -- Portability fixes for different versions of g++/STL (Nicholas Weaver - and Vern Paxson). - -- Fix for dynamic detection of SSL via DPD (Robin Sommer). - -- IPv6 portability fix for BinPAC-based DNS analyzer (Vern Paxson). - Note, more portability work is needed for it. - -- Bug fix for bifcl error messages (Vern Paxson). - -- Minor bug fix for remote communication, plus some improved communication - logging (Robin Sommer). - -- Bug fix for &printhook (Robin Sommer). - -- Bug fix for error message output (Robin Sommer). - -- Bug fix for termination cleanup (Robin Sommer). - -- Bug fix for some Rlogin corner cases (Robin Sommer & Vern Paxson). - -- Bug fix for bifcl generation of "interval" types (Vern Paxson). - -- Bug fix for getting connection memory statistics when Bro is - exiting (Robin Sommer). - -- Config fix: --enable-debug now turns off -O2 for gcc (Robin Sommer). - -- Bug fixes for "heavy" analysis (Vern Paxson). - -- Broccoli bug fixes for types net and port (Robin Sommer). - -- Bug fixes for Telnet environment options (Robin Sommer). - -- Bug fix for accessing remote peer description (Robin Sommer). - -- A fix for the connection compressor generating new_connection too - late (Robin Sommer). - -- Fixes for DAG support, including configuration and multiple - interfaces (Robin Sommer). - -- Bug fix for serializing time-stamps of table entries (Robin Sommer). - -- Bug fix for dealing with peer IDs for remote communication (Robin Sommer). - -- Bug fix to avoid installing timers when timers have already - been canceled (Robin Sommer). - -- Bug fix for interplay between serializing connections and - connection compressor (Robin Sommer). - -- Memory leak fix for enum's (Robin Sommer). - -- Bug fix for files being closed prior to bro_done() (Vern Paxson). - -- aux/broccoli/contrib was not included in distribution (Robin Sommer). - -- Auto-configuration bug fix for BinPAC (Craig Leres). - -- Bug fix for dynamic protocol detection (Robin Sommer). - -- A number of configuration fixes for installation and portability - (Christian Kreibich, Brian Tierney, Robin Sommer, Dan Kopecek). - - -1.3 Mon Jul 16 22:11:00 PDT 2007 - -- The Bro manual has been wikified at: - - http://www.bro-ids.org/wiki/index.php/User_Manual - - and this is the format in which it will evolve in the future - (Christian Kreibich). - -- Much more extensive support for SMB, NetBIOS and NCP (Chris Grier). - -- The new attribute &priority=n defines the order of execution for handlers - of the same event (Robin Sommer). Handlers with higher priority are - executed first. n is an integer expression that must evaluate to a - constant when the script is loaded. - - Example: - > cat foo.bro - event bro_init() &priority = -5 { print -5; } - event bro_init() &priority = 5 { print 5; } - event bro_init() { print 0; } # default priority=0 - > ./bro foo.bro - 5 - 0 - -5 - - The connection_state_remove() handler in conn.bro now has priority - -10 and therefore executes after all other handlers for this event. - This fixes a long-standing problem of sometimes $addl fields not showing - up in connection summaries. - -- The new expressions record(...), table(...), set(...) and vector(...) - are constructors for the corresponding aggregate types (Vern Paxson). - For example, - - record($foo = "hi", $bar = -6) - - is the same as the existing constructor - - [$foo = "hi", $bar = -6] - - For tables, sets, and vectors, the "..." values within the ()'s have - the same syntax as those that you can list in variable initializations. - For example, - - table([1, T] = "black", [4, F] = "red") - - returns a table of type "table[count, bool] of string". - - set(4, 3, -1) - - is a value of type "set[int]". - -- You can associate attributes with table() and set() constructors - (Robin Sommer). For example: - - local s = set(1.2.3.4) &read_expire = 5 secs; - - associates a 5-second read expiration with the set assigned to s. - -- Bro now explicitly supports port numbers reflecting a transport protocol - type of "unknown" (Christian Kreibich). Currently, this means "not TCP, - UDP or ICMP". The numerical value of such a port is the IP protocol, - so ranges from 0..255. For example: - - global p: port = 0/unknown; - - print fmt("%s", p); - print fmt("p is TCP? %s", get_port_transport_proto(p) == tcp); - print fmt("p is unknown? %s", - get_port_transport_proto(p) == unknown_transport); - - yields - - 0/unknown - p is TCP? F - p is unknown? T - - In comparisons of different protocol types, the following holds: - unknown < TCP < UDP < ICMP. - -- If your system supports "GeoIP" (see http://www.maxmind.com/app/geolitecity - for a corresponding city database), then the new script function - - lookup_location(a: addr): geo_location - - returns a record of geographic information associated with an address - (Seth Hall). The geo_location record has $country_code, $region and - $city fields. If no information is available, each of these will be - set to empty strings. - - If Bro hasn't been configured with GeoIP support, or if the address is - IPv6 that cannot be directly converted to IPv4, then Bro produces a - run-time error and likewise returns empty strings. - -- Signature-matching on HTTP components now processes the URI with - escape sequences expanded (Robin Sommer). Ideally, there would be - two signature keywords, one for decoded URIs (corresponding to this - case) and one that allows matching against the URI as originally - transmitted. - -- The connection compressor is no longer considered experimental, and - is used by default (Robin Sommer). - -- The new function lookup_hostname(host: string): addr_set asychronously - looks up the IPv4 address(es) of the given host via DNS (Robin Sommer). - Like lookup_addr(), this function can only be used within a "when" - statement. - -- The new built-in - - raw_bytes_to_v4_addr(s: string): addr - - takes a string that points to at least 4 bytes, and returns an address - corresponding to interpreting these as being an IPv4 address in network - order (Vern Paxson; suggested by Mike Dopheide). - -- Trace-rewriting support for DNS, SMB (Chris Grier). - -- The new script function find_all(str: string, re: pattern): string_set - returns a string_set giving all occurrences of the pattern "re" in - the string "str" (Robin Sommer). (Note that string_set's are unordered.) - -- The new policy script save-peer-status.bro generates a log - to peer_status.$BRO_LOG_SUFFIX of updates received from - communication peers (Robin Sommer). - -- The policy script print-filter.bro now includes two (scoped) variables, - terminate_bro and to_file, which control whether to exit after printing - the filter (default T) and whether to write to the log file - pcap_filter.$BRO_LOG_SUFFIX or (default) to stdout (Robin Sommer). - -- The new script variable check_for_unused_event_handlers controls whether - Bro checks for unused event handlers (Robin Sommer). It defaults to T, - which was the past behavior (always report). - -- Bro now terminates if the only pending activity is future timers - (Robin Sommer). It used to wait for those timers to expire, but this - can cause fundamental problems if the timers are associated with table - management (since these might never completely drain). - -- Tables and sets inside of records are now initialized to empty - values rather than uninitialized (Vern Paxson). - -- A new variable allow_services_from (in hot.bro) complements the - existing allow_service_to variable (Brian Tierney). It specifies - that access to the given service from the given originator is - allowed. - -- global_sizes() no longer reports internal variables (Robin Sommer). - -- The IRC analyzer is now activated if any of the (many) IRC event - handlers are defined (Robin Sommer). - -- The default value for tcp_close_delay is now 5 sec rather than 0 sec - (Robin Sommer). This prevents some spurious connection events. - -- Improved logic for dealing with "reversed" connections such - as backscatter (Vern Paxson). - -- You can now left-justify fields when using fmt() with "%-" like - in sprintf (Christian Kreibich). - -- Updates to DNS query types (Larry Leviton). - -- Added mechanism to http-header.bro to skip printing some HTTP headers - (Larry Leviton). - -- The IrcHotWord notice now sets the associated connection (Robin Sommer). - -- If a notice has a tag, it's no longer overridden (Robin Sommer). - -- ServerFound notices now set the port field (Robin Sommer). - -- The built-in lookup_ID() now returns the string "" if the - ID does not exist, rather than a run-time error (Robin Sommer). - -- The new tuning option ProtocolDetector::suppress_servers specifies a - set of analyzers for which Bro generates ServerFound notices, but not - ProtocolFound (Robin Sommer). This both reduces log file size and - conserves memory. - -- A new notice_action_filter, tally_notice_type_and_ignore, works the same - as tally_notice_type but returns IGNORE (Robin Sommer) - -- Setting summary_interval == 0 disables the creation of irc-bots.summary.log - (Robin Sommer). - -- If you @load foo and a directory "foo" is in your path, Bro no longer - tries to load it (Robin Sommer). - -- A number of BinPAC fixes and enhancements (Ruoming Pang, Chris Grier - and Vern Paxson). - -- BinPAC now resides in aux/binpac rather than src/binpac (Ruoming Pang - and Christian Kreibich). This reflects a decoupling of it from Bro so - that it can be used to generate protocol analyzers for other projects too. - -- Removed example Inktomi entries from skip_scan_sources initialization, - since they no longer exist (Vern Paxson). - -- The variable make notice_once_per_orig_tally_interval is now - redef'able (Brian Tierney). - -- SIGPROF to the communication child process now logs resource stats to - remote.log (Matthias Vallentin). - -- The new built-in getpid(): count returns Bro's process ID (Robin Sommer). - -- Patterns for detecting IRC-based bots updated (Robin Sommer). - -- irc-bot-syslog now logs just bots, not all IRC client/servers (Robin Sommer). - -- The new variable suppress_notice_actions in notice.bro suppresses - notice_actions events for selected notice types (Robin Sommer). - -- Files opened during operation now rotate just like those opened at - startup (Robin Sommer). - -- ResourceStats now also logs elapsed time and the reported number of - packets-on-the-link (Mark Dedlow). - -- Printing a "file" value now produces its name (Robin Sommer). - -- Removed deliberate truncation of payload in port 80 FIN packets - (Vern Paxson). - -- remote.log now includes received peer_descriptions (Robin Sommer). - -- Significant POP3 analyzer speed-ups (Vern Paxson). - -- Updated README (Vern Paxson). - -- Fix for "@load a" followed by "@load a.bro" not loading the same file - twice (Robin Sommer). - -- Bug fixes for propagating state operations to uninitialized variables - and for spurious state inconsistency messags (Robin Sommer). - -- Bug fix for sending final sync-points during pseudo-realtime mode - (Robin Sommer). - -- Fix for possible buffer overflow (Christian Kreibich). - -- Bug fix for spurious end-of-file's during inter-Bro communication - (Robin Sommer). - -- Bug fix for dpd_match_only_beginning=F (Robin Sommer). - -- Bug fix for updating timestamps (Christian Kreibich). - -- Bug fix for skipping ADU processing in adu.bro (Christian Kreibich - and Zhichun Li). - -- Fix for ICMPs that carry ICMP headers (or non-TCP/UDP/ICMP headers) - within them (Vern Paxson). - -- Fix for files being rotated after the timer queue has been deleted - (Vern Paxson). - -- Bug fix for signature-matching with IPv6 subnets (Vern Paxson). - -- Bug fix for connection compressor setting connection origin (Robin Sommer). - -- Bug fix for interconn.bro when processing peculiar connections (Vern Paxson). - -- Fix for off-by-one buffer in sscanf call (Christian Kreibich). - -- Fixed inefficiency/warning flagged by g++ (Vern Paxson). - -- Bug fix for NUL string termination in SMB processing (Zhichun Li). - -- Fix for over-ref'ing of file Val's (Vern Paxson). - -- Fixes for some g++ warnings (Christian Kreibich, Vern Paxson). - -- gcc 3.4.2 portability fixes (Robin Sommer). - -- Minor build fixes for Broccoli, including a version bump to match that - of Bro. See aux/broccoli/ChangeLog for details. - -- distcheck fixes (Christian Kreibich). - -- Configuration portability fixes (Matthias Vallentin, Jean-philippe Luiggi). - -- OpenBSD portability fixes (Jean-philippe Luiggi, Christian Kreibich). - - -1.2.1 Mon Dec 11 16:22:58 PST 2006 - -- Fixed delayed triggering of new_connection events when using the - connection compressor. - -- Fixed tracking of first packet in TCP analyzer. (Reported by Guohan Lu) - -- The syslog built-in got lost during some previous merge. - -- Fixed crash if local variable is given as timeout value for table. - (Reported by Mike Wood.) - -- Fixed using "time" values as table indices. - -- Added ssh to default brolite DPD configuration. - -- Fixed catching up to real-time in case of lull. - -- Fixed Broccoli "BRO_DATA_FORMAT_VERSION" to match version in Bro. - -- Fixed Makefile problem in doc directory. - -- Fixed Makefile dependency problem in binpac directory. - -- Added Linux tuning to brolite install script. - -- Modified Makefile to include broccoli/contrib. - -- Adding missing initialization to remote serializer. - -- Minor documentation updates for reference manual and Broccoli. - - -1.2 Tue Oct 17 12:09:49 PDT 2006 - -- Bro now supports DPD, dynamic protocol detection (Robin Sommer, Holger - Dreger, and Michael Mai). With DPD, Bro can analyze protocols regardless - of what port numbers they use: it infers the protocol based on which - application analyzers can parse it without error. Adding this functionality - involved extensive changes to Bro's internals, but also now enables - multiple Bro analyzers to work on the same connection, either concurrently - or one nested inside the other (we have not taken much advantage of this - latter capability yet, but see the FTP events discussed below). - - There are a number of new policy scripts, events, and variables associated - with DPD processing, as follows. - - Scripts: - - You activate DPD by @load'ing dpd.bro. It in turn instructs Bro - to load the signature file policy/sigs/dpd.sig. Note that Bro - uses signatures to expedite deciding which analyzers to try on - a given connection; it does *not* simply use the signatures to - make the determination of which protocol is in use, as this is - insufficiently robust. (At this point, Bro provides signatures - for FTP, IRC, HTTP, SMTP, and SSH. In the future we plan to add - other protocols.) - - Along with dpd.bro, you need to @load detect-protocols.bro or - detect-protocols-http.bro. The former enables general detection - of application-layer protocols, while the latter does further - inspection of HTTP sessions to characterize applications running - on top of HTTP such as Gnutella or SOAP. (Loading dpd.bro - is separate from loading one of these scripts because in principle - Bro could use a different means than signatures to activate - the analyzers, although currently it does not.) - - If you @load dyn-disable.bro, then once an analyzer determines - that it does not match a given connection, it is deactivated - (and a Notice is generated). Otherwise, it still proceeds to try - its best to analyze the connection (to possibly be more robust - against evasion). - - The scripts dce.bro and smb.bro enable DPD for the Windows DCE and - SMB protocols, respectively. (Note that analysis of these protocols - is undergoing a major expansion, not yet complete.) - - Events: - - event protocol_confirmation(c: connection, atype: count, aid: count) - Generated when the given connection has been confirmed as - conforming with the application type (protocol) specified - by atype. aid is a globally unique analyzer ID that identifies - a particular analyzer instance. - - The values for atype are symbolic names associated with - each of Bro's analyzers, such as ANALYZER_IRC. See the - initialization at the beginning of Analyzer.cc for the - full set of names. - - The function analyzer_name(atype: count): string translates - these symbolic names into text. For example, - - analyzer_name(ANALYZER_IRC) - - yields "IRC". - - event protocol_violation(c: connection, atype: count, aid: count, - reason: string) - Generated when the given connection has been found to - violate the protocol of the given application type, with - "reason" giving details. - - Variables: - - dpd_buffer_size: count (default 1024) - Specifies how much pending data Bro keeps for connections - that have not been classified yet. Once this fills, the - data is deleted, though classification can still continue - (see below). - - dpd_match_only_beginning: bool (default T) - If set, specifies that Bro should stop signature matching - if it has processed dpd_buffer_size bytes. - - dpd_ignore_ports: bool (default F) - If set, then Bro does not take into consideration the port - numbers associated with connections when attempting to - classify them (which can otherwise help the process in - some cases). - - dpd_reassemble_first_packets: bool (default T) - If set, then Bro does TCP stream reassembly before applying - signature-matching to detect protocols. - - likely_server_ports: set[port] - Specifies a list of ports that Bro will consider as likely - used by servers. For example, if Bro sees a connection - that has already been established (so it does not know - which side sent the initial SYN), and one side uses a port - in this set, then it will assume that that side is the - server (connection responder). The set is empty unless - you populate it or @load server-ports.bro, which specifies - a large number of values. - - dpd_config: table[AnalyzerTag] of dpd_protocol_config - Specifies the DPD configuration associated with each tag. - The type dpd_protocol_config is simply: - - type dpd_protocol_config: record { - ports: set[port] &optional; - }; - - i.e., an optional $ports field specifying a set of ports - associatd with the tag. For example, ftp.bro now includes - the equivalent of: - - redef dpd_config += { - [ANALYZER_FTP] = [$ports = 21/tcp] - }; - - Functions: - - The function - - expect_connection(orig: addr, resp: addr, resp_p: port, - analyzer: count, tout: interval) - - is called to alert Bro that a new connection is expected, initiated - by orig to a server running on resp's port resp_p (note: orig's port - is not specified) which will correspond to the specified analyzer - (e.g., "FILE", which is used to analyze files transferred by FTP - - see next item). "tout" is a timeout to associate with the waiting. - - The function - - function disable_analyzer(cid: conn_id, aid: count) - - instructs Bro to disable the analyzer that generated the current - event, assuming the analyzer is associated with the given connection - ID. This is used by the dyn-disable.bro script discussed above. - -- A much more complete BinPAC compiler, along with new HTTP, DNS, and - RPC/Portmap analyzers in binpac (Ruoming Pang). The flag "--use-binpac" - activates the BinPAC-based analyzers (currently for HTTP and DNS). - See www.cs.princeton.edu/~rpang/binpac-paper.pdf for a description of - BinPAC, and let Ruoming know if you are interested in using BinPAC to build - new analyzers. - -- A new type of analyzer, FILE, analyzes the contents of a connection as - though it were a data file (Robin Sommer). Currently, it can generate - two events: - - event file_transferred(c: connection, prefix: string, descr: string, - mime_type: string) - Indicates that the connection transferred a file. "prefix" - is the beginning of the file's data; "descr" and "mime_type" - are indicators of the file's type, as reported by the - "libmagic" library. - - descr/mime_type are only set if Bro is configured on a - system that includes the "libmagic" library. - - event file_virus(c: connection, virname: string) - Indicates the connection transferred an executable - corresponding to a known virus of the given name. - - This functionality is only available if Bro is configured - on a system that includes the "libclamav" library. - - Note, this analyzer is enabled via a call to expect_connection by - the FTP analyzer. - -- New events relating to IRC analysis (Robin Sommer): - - event irc_client(c: connection, prefix: string, data: string) - Generated upon seing a client message sent over the given - IRC connection. "prefix" is the command's prefix as defined - by the IRC protocol. It is used by servers to indicate the - true origin of the message; it may be empty. "data" contains - the message. - - event irc_server(c: connection, prefix: string, data: string) - Same for server messages. - - event irc_user_message(c: connection, user: string, host: string, - server: string, real_name: string) - Generated upon seeing an IRC "USER" command. - - event irc_password_message(c: connection, password: string) - Generated upon seeing an IRC "PASS" command. - - event irc_channel_topic(c: connection, channel: string, topic: string) - Generated upon seeing an IRC server reply that includes - the channel topic. - - event irc_global_users(c: connection, prefix: string, msg: string) - Generated upon seeing an IRC server reply that includes - a count of the number of IRC users. - -- The new experimental script irc-bot.bro tracks IRC-based bots (Robin Sommer). - The accompanying script irc-bot-syslog.bro syslog's the state of the - bot analysis every IrcBot::summary_interval seconds (default 1 minute). - -- The new script proxy.bro looks for open Web proxies by matching incoming - requests to a server with outgoing requests it makes (Robin Sommer). It - generates HTTPProxyFound Notices when it finds one. - -- Changes to notices.bro (Robin Sommer): - - - notice_policy_item's now have a default $result of - NOTICE_FILE and a default $priority of 1. - - - The new notice_action_filter, notice_alarm_per_orig, alarms - on the first NoticeType from a specific source. Subsequent - instances are tallied. - - - notice_action_filters now reside in the new script - notice-action-filter.bro (automatically loaded by notice.bro). - - - The notice actions NOTICE_ALARM_PER_CONN, NOTICE_ALARM_PER_ORIG, - and NOTICE_ALARM_ONCE have been removed, as they were never - actually implemented. - - - If the notice_policy returns IGNORE or FILE, the action_filters - filters are no longer consulted. - -- A new attribute for tables and sets, &mergeable, changes the semantics - of assignments, as follows (Robin Sommer). Given two &mergeable tables/sets - A and B, an assignment "A = B" becomes actually a join "A = A \cup B" - (i.e., union). The envisoned use is to help avoid race conditions - when doing remote state synchronization. - -- The semantics of &synchronized expire_funcs has changed (Robin Sommer). - Now, when a table entry is expired and the operation is propagated to a - a peer, the peer will call its expire_function. - -- TRW analysis now skips UDP traffic because it currently treats - all UDP connections as failures (Robin Sommer). - -- trw.bro has been split into trw-impl.bro (the algorithm) and - trw.bro (which simply activates the analysis), to facilitate writing - scripts that have hooks into TRW analysis but don't presume it's - active (Robin Sommer). - -- The option report_remote_notices in remote.bro has been replaced - by a new script you include, remote-report-notices.bro (Robin Sommer). - -- The new function connect_peer() explicitly connects to a remote host - (Robin Sommer). - -- The new script remote-send-id.bro sends the current value of an ID - to a remote Bro and then terminates processing (Robin Sommer). It's - intended for use from the command-line, as in - - bro -e "redef dst="" id="" remote-send-id - - The other scripts must set up the connection. is an index into - Remote::destinations corresponding to the destination. - -- New built-ins {suspend,resume}_state_updates() can be called to - temporarily avoid propagating updates to &sync'ed values (Robin Sommer). - This can avoid duplicated activity. - -- The new function terminate_communication() instructs Bro to end its - communication with remote peers (Robin Sommer). - -- The new event remote_state_access_performed is raised when remote state - access has been performed (Robin Sommer). This is primarily for debugging. - -- The log() built-in has been renamed to ln() to avoid conflict (Vern Paxson). - -- bifcl now generates event generation wrapper functions from event.bif - (Ruoming Pang). For example, to generate event http_reply, currently - one writes: - - val_list* vl = new val_list; - vl->append(BuildConnVal()); - vl->append(new StringVal(fmt("%.1f", reply_version))); - vl->append(new Val(reply_code, TYPE_COUNT)); - if ( reply_reason_phrase ) - vl->append(reply_reason_phrase); - else - vl->append(new StringVal("")); - ConnectionEvent(http_reply, vl); - - In the future, one will be able to just call bro_event_http_reply(), and - the code generated by bifcl looks like: - - void bro_event_http_reply(Connection* c, StringVal* version, - bro_uint_t code, StringVal* reason) - { - val_list* vl = new val_list; - - vl->append(c->BuildConnVal()); - vl->append(version); - vl->append(new Val(code, TYPE_COUNT)); - vl->append(reason); - - mgr.QueueEvent(http_reply, vl, SOURCE_LOCAL, c); - } - - Accompanying this change is a semantic shift to types "string" and "port" - in .bif files. They used to be translated to C++ types BroString* and - uint32, respectively. Now they are translated to StringVal* and PortVal*. - The functions in bro.bif are changed accordingly, and please be aware - of this change when you write built-in functions in future. - - Also for this change, the parameter 'new' for rsh_request has been renamed - 'new_session', as 'new' is a reserved word for C++. - -- Some ICMP "connections" now have services identified ("icmp-echo", - "icmp-unreach") rather than just listing the service as "other" - (Ruoming Pang). - -- The new option remote_trace_sync_interval specifies an interval after - which each Bro will stop processing its trace and wait for all others - to signal that they have reached the same time (Robin Sommer). The - intent is support for operating Bro in a distributed cluster fashion - (and in particular for debugging such clusters when running off-line - on traces). - - This option only works in pseudo-realtime mode, and requires the new - global remote_trace_sync_peers to give the total number of remote peers - (not including self). Signaling is done via a new communication message - type. - -- Extensions for DNS transformation/anonymization, including introduction - of trace transformation for protocols other than TCP (Jason Lee). - Not yet fully developed/debugged. - -- Extensions for HTTP transformation/anonymization (Martin Casado). - Not yet fully developed/debugged. - -- The $conn field is now included in HTTPProxyFound notices (Robin Sommer). - -- Changed service inference algorithm to favor lower-numbered - likely-servers over higher-numbered ones (Vern Paxson). - -- In pseudo-realtime mode, Bro now uses real-time for deciding which - peer should send state (Robin Sommer). - -- Time synchronization for Bro's running on traces in pseudo-realtime mode - added (Robin Sommer). - -- Avoidance of false content gaps improved when sorting packets with - out-of-order timestamps (Ruoming Pang). - -- Packets from the packet sorter are now more robustly drained upon - termination of input (Ruoming Pang). - -- Documentation for deep-copy updated (Christian Kreibich). - -- Nasty fragment reassembly bug fixed (Vern Paxson). - -- Serious bugs in EDNS0 processing fixed (Vern Paxson). - -- Fixed significant misfeature of interconn.bro that stopped all processing - of a connection once it makes a detection (Vern Paxson). - -- Fixes for &read_expire operation across synchronizes tables (Robin Sommer). - -- Fixes for multiple peers exchanging initial &sync state simultaneously - (Robin Sommer). - -- Improvements to graceful termination of Bro when communicating with - remote peers (Robin Sommer). - -- Fix for ICMP analyzer not always generating icmp_sent events - (Robin Sommer). This appears to still need some work, as now - it generates redundant events. - -- Fix for initial exchange of &sync state which could lead to - referencing unknown IDs (Robin Sommer). - -- Fix to scan detection for differing semantics of connection compressor - vs. non-compressor (Robin Sommer). - -- Bug fix for distinguishing regular expression matches of length 0 from - those of length 1 (Ruoming Pang). - -- Fix for SSH version parsing in the presence of content gaps (Robin Sommer). - -- Bug fix for IRC that could lead to crashes (Robin Sommer). - -- Bug fix to refrain from adding new timers when a connection has - already been removed from the connection table (Robin Sommer). - -- Bug fix for packet_contents not including the transport-layer header - (Robin Sommer). - -- Some memory leaks fixed (Robin Sommer). - -- A bunch of portability and distribution problems fixed (Christian - Kreibich, Robin Sommer, Vern Paxson). - - -1.1 Mon May 15 10:50:33 PDT 2006 - -- Bro now supports a "when" statement for taking action upon something - becoming true asynchronously (Robin Sommer). This provides a powerful - new mechanism with numerous applications. - - Syntax: - - when '(' ')' [timeout '{ '}'] - - where the first can be a single statement or a block enclosed - in {}'s, but the set associated with "timeout" must be enclosed in - {}'s (to reduce ambiguities in Bro's grammar). - - Bro executes the first statement when becomes true. If you give - a timeout and the condition has not been satisfied before it expires, Bro - executes the second statement instead. - - A simple example: - - global t: table[addr] of count; - event connection_established(c: connection) - { - local orig = c$id$orig_h; - if ( orig !in t ) - { - t[orig] = 1; - - when ( t[orig] == 5 ) - print fmt("%s has established 5 connections", orig); - timeout 1 hr - { - print fmt("%s has NOT established 5 connections", orig); - delete t[orig]; - } - } - else - ++t[orig]; - } - - Notes: - - The condition may be evaluated more than once, and at arbitrary - times. - - - When the when-body is executed, the condition is guaranteed to be - still satisfied. - - - Expression reevaluation is primarily triggered by modifications - to globals. However, reevaluations do not take place immediately - but potentially at a later point. This means that if we change a - global to a value which would execute the trigger but then change - it back, the change may go unnoticed. - - - Inside the condition you may introduce new locals. For example, - - when ( (local x = foo()) && x == 42 ) ... - - Such an assignment always yields true as its expression value - (but the assignment might be delayed, for example if foo() is - a delayed function call - see below). - - Delaying function calls - ======================= - - Functions called inside the condition of a when-clause may delay their - results until they're ready. This works for both script-level and built-in - functions. - - For script-level functions, there is a new construct, "return ", - to delay a function's result. When used, the function returns at the - time the when-stmt's condition becomes true, and it yields the value - that the when-stmt's body then returns. Toy example: - - global X: table[string] of count; - - function a() : count - { - # This delays until condition becomes true. - return when ( "a" in X ) - { - return X["a"]; - } - timeout 5 min - { - return 0; - } - } - - event bro_init() - { - # Installs a trigger which fires if a() returns 42. - when ( a() == 42 ) { print "Yippie!"; } - - X["a"] = 42; - } - - There's also a new built-in function which can delay - - lookup_addr(host: addr) - - performs asynchronous DNS address->hostname lookups. Example: - - local h; addr; - [...] - when (local name = lookup_addr(h)) { print h, name; } - - See the function gen_hot_notice_with_hostnames() in conn.bro for - a more worked-out example of using the "when" clause to translate the - local address in SensitiveConnection notices to a hostname (contributed - by Brian Tierney). This functionality is activated by redef'ing - xlate_hot_local_addr to T. - - Here is the full evaluation model of a when's condition: - - - The condition may be evaluated more than once, at arbitrary times. - - - It is always fully evaluated, no matter whether some former - evaluation has been suspended by a delaying function call. - - - All function calls which do not delay are always *fully* executed - each time the condition is evaluated. - - - Function calls which delay are only executed *once*; their result is - cached and re-used in the case the condition is evaluated again. - - - The condition is guaranteed to be true when the body is executed - (potentially using cached function results) - -- By default Bro now uses a configuration similar to what used to be - activated using reduce-memory.bro, along with some additional state - timeouts that are new (Robin Sommer and Vern Paxson). This allows for - better state management out-of-the-box, at the cost of some precision - of analysis and resilience to evasion. In particular, the intent is to - move towards being able to run Bro continuously without inexorably growing - the amount of memory used until exhaustion. - - You can access a configuration similar to the previous default state - management settings by loading heavy-analysis.bro. It turns on a - load-prefix of "heavy", so when you load XXX.bro, a file heavy.XXX.bro - will also be automatically loaded if present. Note that, as was the - case for reduce-memory, you need to load heavy-analysis prior to other - files for it to have effect. - -- The new module clear-passwords.bro monitors login/FTP/IRC/POP traffic - for cleartext passwords (Jason Lee). - -- The new script service-probe.bro looks for remote hosts that repeatedly - connect to the same service on local hosts (for a configurable set of - services and connection sizes) in order to detect brute-forcing attacks - such as password-guessing (Jim Mellander). - -- A new ARP analyzer generates three events: - - event arp_request(mac_src: string, mac_dst: string, - SPA: addr, SHA: string, TPA: addr, THA: string); - - event arp_reply(mac_src: string, mac_dst: string, - SPA: addr, SHA: string, TPA: addr, THA: string); - - event bad_arp(SPA: addr, SHA: string, TPA: addr, THA: string, - explanation: string); - - with a corresponding policy script arp.bro (Chema Gonzalez and Vern Paxson). - It writes logs to arp.$BRO_LOG_SUFFIX. It has not been tested much yet. - -- Bro Lite changes (Jason Lee): - - default user for is now user 'bro' - - now uses the correct sysctl on FreeBSD 6 - - now uses the correct Perl path if site-report.pl not installed - into '/usr/local/bro' - - no longer prompts to encrypt email unless you pick to email reports - -- The default Bro Lite install now only checkpoints Bro once a week - (Brian Tierney). - -- Implicit Bro file extensions (such as .bro for policy scripts and .sig - for signatures) are now searched for first rather than only if the - non-extension-version of the file doesn't exist (Vern Paxson). For - example, running "bro -r trace mt" now first searches $BROPATH for - "mt.bro" before searching for "mt", whereas it used to do these in - the other order. - -- There's now a simpler mechanism for redef'ing variables on the command-line - (Christian Kreibich). Any command line arguments of the form = - are now expanded into policy code of the form "redef var=val;", where - is wrapped in quotation marks if the value appears to be a string - and doesn't have quotation marks already. This works with strings with - whitespace such as foo="Hello World"; however, note that it means you - can't use the mechanism to redef an enum value. - -- The Bro distribution now includes (and builds by default) Christian - Kreibich's Broccoli library (Bro C Client Library), which enables programs - to communicate with running Bro's (Christian Kreibich and Jason Lee). - Configure with --disable-broccoli to turn this off. - -- Built-in functions log(x: double): double and exp(x: double): double - which do natural logarithms and their inverses (Jaeyeon Jung). - -- The new built-in function gethostname() returns the local host's name - (Jason Lee & Robin Sommer). - -- The new built-in function reading_traces() returns true if Bro - is reading trace files (Robin Sommer). - -- The new built-ins suspend_processing() and continue_processing() provide - script-level control for instructing the event engine to stop or resume - processing packets (Robin Sommer). This is useful for coordinating - simultaneous processing by multiple Bro's. - -- Email notices are now by default sent via /bin/mail, with "[Bro Alarm]" - in the subject. - -- redef'ing a function now replaces the existing body rather than - supplementing it (Robin Sommer), which was a bug. - -- You can now configure Bro to process encapsulated IP packets either - by setting, as before, a fixed encap_hdr_size (for VLANs), or setting - parse_udp_tunnels to T (Ruoming Pang). For the latter, you specify a - UDP tunnel port using udp_tunnel_port (the previous variable "tunnel_port" - has gone away); or you can leave it set to its default of 0/udp, in which - case Bro will look for IP encapsulated in UDP packets on any port. - -- Added a simple form of profiling based on sampling the work done - per-packet (Vern Paxson). The event engine generates a - - event load_sample(samples: load_sample_info, CPU: interval, dmem: int) - - event every load_sample_freq packets (roughly; it's randomized), where - load_sample_freq defaults to 20. "samples" is simply a set[string]; it - contains the names of the functions, event handlers, and their source - files that were accessed during the processing of the sampled packet, - along with an estimate of the CPU cost of processing the packet and - (currently broken) memory allocated/freed. - -- Bro now includes experimental support for Endace DAG cards (Gregor Maier - and Robin Sommer). To activate, configure with - - --with-DAG=/path/to/dagtool/installation - - and use "dag0" as the network interface. You may need to configure the - card with the dagtools first. In general, if dagsnap works, Bro should - work as well. - -- Log rotation has changed in a number of ways (Mark Dedlow & Robin Sommer): - - * The new variable log_rotate_base_time: string, if defined, - specifies that logs should be rotated at log_rotate_base_time + - i * rotate_interval intervals. Format is as a string in - 24-hour time, "%H:%M", e.g, "12:00". This format may change - in the future to instead be a Bro time type. - - * RotateLogs::date_format can be redefined to change format of - timestamps in rotated files. - - * RotateLogs::build_name() can be redefined to implement an - arbitrary naming scheme for rotated files. - - Note, this code has not been extensively tested. - -- Bro now by default builds a version of malloc bundled with its - distribution (Vern Paxson & Brian Tierney). - -- The syntax for the clone operator now looks like a function call, - "copy(x)" (Vern Paxson). - -- The new flag DNS::logging (default F), if T, disables generation of - dns.log (which is often uninteresting and very large), though it - still performs analysis leading to NOTICEs (Robin Sommer). - -- A new global, hostile_domain_list, has been added to dns.bro which - lists domains to be flagged if A or MX records are queried (Scott Campbell). - -- Added globals dns_skip_all_{auth,addl} to skip all DNS AUTH/ADDL processing - (Vern Paxson). Skipping these is on (true) by default, because such - processing is quite expensive. - -- backdoor.bro now turns off by default some detectors that from experience - have too many false positives, or (such as for HTTP) too many uninteresting - true positives (Brian Tierney). In addition: - - - the module now generates a BackdoorFound notice for each backdoor - - - the new variable dump_backdoor_packets (default F) if set causes - the packet that triggered the backdoor detection to be written to - backdoor-packets/: