Merge remote-tracking branch 'origin/master' into topic/vladg/radius

2025-10-02 14:48:21 +00:00 · 2014-02-11 16:17:40 -05:00 · 2014-02-11 16:17:40 -05:00 · 80ec40bc93
commit 80ec40bc93
parent 8688c764a0 506b26e5ff
169 changed files with 3244 additions and 517 deletions
--- a/198
+++ b/198
@ -1,4 +1,202 @@

+2.2-158 | 2014-02-09 23:45:39 -0500
+
+  * Change dns.log to include only standard DNS queries. (Jon Siwek)
+
+  * Improve DNS analysis. (Jon Siwek)
+
+    - Fix parsing of empty question sections (when QDCOUNT == 0).  In this
+      case, the DNS parser would extract two 2-byte fields for use in either
+      "dns_query_reply" or "dns_rejected" events (dependent on value of
+      RCODE) as qclass and qtype parameters.  This is not correct, because
+      such fields don't actually exist in the DNS message format when
+      QDCOUNT is 0.  As a result, these events are no longer raised when
+      there's an empty question section.  Scripts that depends on checking
+      for an empty question section can do that in the "dns_message" event.
+
+    - Add a new "dns_unknown_reply" event, for when Bro does not know how
+      to fully parse a particular resource record type.  This helps fix a
+      problem in the default DNS scripts where the logic to complete
+      request-reply pair matching doesn't work because it's waiting on more
+      RR events to complete the reply.  i.e. it expects ANCOUNT number of
+      dns_*_reply events and will wait until it gets that many before
+      completing a request-reply pair and logging it to dns.log.  This could
+      cause bogus replies to match a previous request if they happen to
+      share a DNS transaction ID. (Jon Siwek)
+
+    - The previous method of matching queries with replies was still
+      unreliable in cases where the reply contains no answers.  The new code
+      also takes extra measures to avoid pending state growing too large in
+      cases where the condition to match a query with a corresponding reply is
+      never met, but yet DNS messages continue to be exchanged over the same
+      connection 5-tuple (preventing cleanup of the pending state). (Jon Siwek)
+
+  * Updates to httpmonitor and mimestats documentation. (Jeannette Dopheide)
+
+  * Updates to Logs and Cluster documentation (Jeannette Dopheide)
+
+2.2-147 | 2014-02-07 08:06:53 -0800
+
+  * Fix x509-extension test sometimes failing. (Bernhard Amann)
+
+2.2-144 | 2014-02-06 20:31:18 -0800
+
+  * Fixing bug in POP3 analyzer. With certain input the analyzer could
+    end up trying to write to non-writable memory. (Robin Sommer)
+
+2.2-140 | 2014-02-06 17:58:04 -0800
+
+  * Fixing memory leaks in input framework. (Robin Sommer)
+
+  * Add script to detect filtered TCP traces. Addresses BIT-1119. (Jon
+    Siwek)
+
+2.2-137 | 2014-02-04 09:09:55 -0800
+
+  * Minor unified2 script documentation fix. (Jon Siwek)
+
+2.2-135 | 2014-01-31 11:09:36 -0800
+
+  * Added some grammar and spelling corrections to Installation and
+    Quick Start Guide. (Jeannette Dopheide)
+
+2.2-131 | 2014-01-30 16:11:11 -0800
+
+  * Extend file analysis API to allow file ID caching. This allows an
+    analyzer to either provide file IDs associated with some file
+    content or to cache a file ID that was already determined by
+    script-layer logic so that subsequent calls to the file analysis
+    interface can bypass costly detours through script-layer.  This
+    can yield a decent performance improvement for analyzers that are
+    able to take advantage of it and deal with streaming content (like
+    HTTP, which has been adapted accordingly). (Jon Siwek)
+
+2.2-128 | 2014-01-30 15:58:47 -0800
+
+  * Add leak test for Exec module. (Bernhard Amann)
+    
+  * Fix file_over_new_connection event to trigger when entire file is
+    missed. (Jon Siwek)
+
+  * Improve TCP connection size reporting for half-open connections.
+    (Jon Siwek)
+
+  * Improve gap reporting in TCP connections that never see data. We
+    no longer accomodate SYN/FIN/RST-filtered traces by not reporting
+    missing data. The behavior can be reverted by redef'ing
+    "detect_filtered_trace". (Jon Siwek)
+
+  * Improve TCP FIN retransmission handling. (Jon Siwek)
+    
+2.2-120 | 2014-01-28 10:25:23 -0800
+
+  * Fix and extend x509_extension() event, which now actually returns
+    the extension. (Bernhard Amann)
+
+    New event signauture:
+
+        event x509_extension(c: connection, is_orig: bool, cert:X509, extension: X509_extension_info)
+
+2.2-117 | 2014-01-23 14:18:19 -0800
+
+  * Fixing initialization context in anonymous functions. (Robin
+    Sommer)
+
+2.2-115 | 2014-01-22 12:11:18 -0800
+
+  * Add unit tests for new Bro Manual docs. (Jon Siwek)
+    
+  * New content for the "Using Bro" section of the manual. (Rafael
+    Bonilla/Jon Siwek)
+    
+2.2-105 | 2014-01-20 12:16:48 -0800
+
+  * Support GRE tunnel decapsulation, including enhanced GRE headers.
+    GRE tunnels are treated just like IP-in-IP tunnels by parsing past
+    the GRE header in between the delivery and payload IP packets.
+    Addresses BIT-867. (Jon Siwek)
+
+  * Simplify FragReassembler memory management. (Jon Siwek)
+    
+2.2-102 | 2014-01-20 12:00:29 -0800
+
+  * Include file information (MIME type and description) into notice
+    emails if available. (Justin Azoff)
+
+2.2-100 | 2014-01-20 11:54:58 -0800
+
+  * Fix caching of recently validated SSL certifcates. (Justin Azoff)
+    
+2.2-98 | 2014-01-20 11:50:32 -0800
+
+  * For notice suppresion, instead of storing the entire notice in
+    Notice::suppressing, just store the time the notice should be
+    suppressed until. This saves significant memory but can no longer
+    raise end_suppression, which has been removed. (Justin Azoff)
+
+2.2-96 | 2014-01-20 11:41:07 -0800
+
+  * Integrate libmagic 5.16. Bro now now always relies on
+    builtin/shipped magic library/database. (Jon Siwek)
+
+  * Bro now requires a CMake 2.8.x, but no longer a pre-installed
+    libmagic. (Jon Siwek)
+
+2.2-93 | 2014-01-13 09:16:51 -0800
+
+  * Fixing compile problems with some versions of libc++. Reported by
+    Craig Leres. (Robin Sommer)
+
+2.2-91 | 2014-01-13 01:33:28 -0800
+
+  * Improve GeoIP City database support. When trying to open a city
+    database, it now considers both the "REV0" and "REV1" versions of
+    the city database instead of just the former. (Jon Siwek)
+
+  * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek)
+    
+    - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set.
+
+    - Fix failure to locate bro binary if invoking from a relative
+      path and '.' isn't in PATH.
+
+  * Fix for packet writing to make it use the global snap length.
+    (Seth Hall)
+
+  * Fix for traffic with TCP segmentation offloading with IP header
+    len field being set to zero. (Seth Hall)
+
+  * Canonify output of a unit test. (Jon Siwek)
+    
+  * A set of documentation updates. (Daniel Thayer)
+
+      - Fix typo in Bro 2.2 NEWS on string indexing.
+      - Fix typo in the Quick Start Guide, and clarified the
+        instructions about modifying crontab.
+      - Add/fix documentation for missing/misnamed event parameters.
+      - Fix typos in BIF documentation of hexstr_to_bytestring.
+      - Update the documentation of types and attributes.
+      - Documented the new substring extraction functionality.
+      - Clarified the description of "&priority" and "void".
+
+2.2-75 | 2013-12-18 08:36:50 -0800
+
+  * Fixing segfault with mismatching set &default in record fields.
+    (Robin Sommer)
+
+2.2-74 | 2013-12-16 08:49:55 -0800
+
+  * Improve warnings emitted from raw/execute input reader. (Jon
+    Siwek)
+
+  * Further improve core.when-interpreter-exceptions unit test. (Jon
+    Siwek)
+
+2.2-72 | 2013-12-12 07:12:47 -0800
+
+  * Improve the core.when-interpreter-exceptions unit test to prevent
+    it from occasionally timing out. (Jon Siwek)
+
 2.2-70 | 2013-12-10 15:02:50 -0800

  * Fix (harmless) uninitialized field in basename/dirname util
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@ -1,5 +1,5 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
 include(cmake/CommonCMakeConfig.cmake)

 ########################################################################
@ -39,6 +39,32 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 ########################################################################
 ## Dependency Configuration

+include(ExternalProject)
+
+# LOG_* options to ExternalProject_Add appear in CMake 2.8.3.  If
+# available, using them hides external project configure/build output.
+if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2)
+    set(EXTERNAL_PROJECT_LOG_OPTIONS
+        LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1)
+else()
+    set(EXTERNAL_PROJECT_LOG_OPTIONS)
+endif()
+
+set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix)
+set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include)
+set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
+set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
+ExternalProject_Add(libmagic
+    PREFIX ${LIBMAGIC_PREFIX}
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz
+    CONFIGURE_COMMAND ./configure --enable-static --disable-shared
+                                  --prefix=${LIBMAGIC_PREFIX}
+                                  --includedir=${LIBMAGIC_INCLUDE_DIR}
+                                  --libdir=${LIBMAGIC_LIB_DIR}
+    BUILD_IN_SOURCE 1
+    ${EXTERNAL_PROJECT_LOG_OPTIONS}
+)
+
 include(FindRequiredPackage)

 # Check cache value first to avoid displaying "Found sed" messages everytime
@ -57,7 +83,6 @@ FindRequiredPackage(BISON)
 FindRequiredPackage(PCAP)
 FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
-FindRequiredPackage(LibMagic)
 FindRequiredPackage(ZLIB)

 if (NOT BinPAC_ROOT_DIR AND
@ -73,18 +98,12 @@ if (MISSING_PREREQS)
    message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()

-set(libmagic_req 5.04)
-if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
-    message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
-                        "(found ${LibMagic_VERSION})")
-endif ()
-
 include_directories(BEFORE
                    ${PCAP_INCLUDE_DIR}
                    ${OpenSSL_INCLUDE_DIR}
                    ${BIND_INCLUDE_DIR}
                    ${BinPAC_INCLUDE_DIR}
-                    ${LibMagic_INCLUDE_DIR}
+                    ${LIBMAGIC_INCLUDE_DIR}
                    ${ZLIB_INCLUDE_DIR}
 )

@ -163,7 +182,7 @@ set(brodeps
    ${PCAP_LIBRARY}
    ${OpenSSL_LIBRARIES}
    ${BIND_LIBRARY}
-    ${LibMagic_LIBRARY}
+    ${LIBMAGIC_LIBRARY}
    ${ZLIB_LIBRARY}
    ${OPTLIBS}
 )
--- a/29
+++ b/29
@ -9,9 +9,21 @@ Bro 2.3

 [In progress]

+Dependencies
+------------
+
+- Bro no longer requires a pre-installed libmagic (because it now
+  ships its own).
+
+- Compiling from source now needs a CMake version >= 2.8.0.
+
 New Functionality
 -----------------

+- Support for GRE tunnel decapsulation, including enhanced GRE
+  headers. GRE tunnels are treated just like IP-in-IP tunnels by
+  parsing past the GRE header in between the delivery and payload IP
+  packets.

 Changed Functionality
 ---------------------
@ -22,6 +34,19 @@ Changed Functionality
 - ssl_client_hello() now receives a vector of ciphers, instead of a
  set, to preserve their order.

+- Notice::end_suppression() has been removed.
+
+- Bro now parses X.509 extensions headers and, as a result, the
+  corresponding event got a new signature:
+
+      event x509_extension(c: connection, is_orig: bool, cert: X509, ext: X509_extension_info);
+
+- Bro no longer special-cases SYN/FIN/RST-filtered traces by not
+  reporting missing data. The old behavior can be reverted by
+  redef'ing "detect_filtered_trace".
+
+  TODO: Update if we add a detector for filtered traces.
+
 Bro 2.2
 =======

@ -198,9 +223,9 @@ New Functionality
        global s = MySet([$c=1], [$c=2]);

 - Strings now support the subscript operator to extract individual
-  characters and substrings (e.g., ``s[4]``, ``s[1,5]``). The index
+  characters and substrings (e.g., ``s[4]``, ``s[1:5]``). The index
  expression can take up to two indices for the start and end index of
-  the substring to return (e.g. ``mystring[1,3]``).
+  the substring to return (e.g. ``mystring[1:3]``).

 - Functions now support default parameters, e.g.::

--- a/2
+++ b/2
@ -1 +1 @@
-2.2-70
+2.2-158
--- a/aux/broccoli
+++ b/aux/broccoli
@ -1 +1 @@
-Subproject commit e02ccc0a27e64b147f01e4c7deb5b897864d59d5
+Subproject commit 52ba12128e0673a09cbc7a68b8485f5d19030633
--- a/aux/broctl
+++ b/aux/broctl
@ -1 +1 @@
-Subproject commit 2e07720b4f129802e07ca99498e2aff4542c737a
+Subproject commit 66793ec3c602439e235bee705b654aefb7ac8dec
--- a/aux/btest
+++ b/aux/btest
@ -1 +1 @@
-Subproject commit 26c3136d56493017bc33c5a2f22ae393d585c2d9
+Subproject commit 9a7b36a44a7abeaa1dc2754d3072ab08995c6af8
--- a/doc/_static/basic.css
+++ b/doc/_static/basic.css
@ -439,8 +439,17 @@ td.linenos pre {
    color: #aaa;
 }

+.highlight-guess {
+    overflow:auto;
+}
+
+.highlight-none {
+    overflow:auto;
+}
+
 table.highlighttable {
    margin-left: 0.5em;
+    overflow:scroll;
 }

 table.highlighttable td {
--- a/doc/broids/index.rst
+++ b/doc/broids/index.rst
@ -0,0 +1,79 @@
+
+.. _bro-ids:
+
+=======
+Bro IDS
+=======
+
+An Intrusion Detection System (IDS) allows you to detect suspicious
+activities happening on your network as a result of a past or active
+attack. Because of its programming capabilities, Bro can easily be
+configured to behave like traditional IDSs and detect common attacks
+with well known patterns, or you can create your own scripts to detect
+conditions specific to your particular case.
+
+In the following sections, we present a few examples of common uses of
+Bro as an IDS.
+
+------------------------------------------------
+Detecting an FTP Brute-force Attack and Notifying
+------------------------------------------------
+
+For the purpose of this exercise, we define FTP brute-forcing as too many
+rejected usernames and passwords occurring from a single address.  We
+start by defining a threshold for the number of attempts, a monitoring
+interval (in minutes), and a new notice type.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 9-25
+
+Using the ftp_reply event, we check for error codes from the `500
+series <http://en.wikipedia.org/wiki/List_of_FTP_server_return_codes>`_
+for the "USER" and "PASS" commands, representing rejected usernames or
+passwords. For this, we can use the :bro:see:`FTP::parse_ftp_reply_code`
+function to break down the reply code and check if the first digit is a
+"5" or not. If true, we then use the :ref:`Summary Statistics Framework
+<sumstats-framework>` to keep track of the number of failed attempts.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 52-60
+
+Next, we use the SumStats framework to raise a notice of the attack when
+the number of failed attempts exceeds the specified threshold during the
+measuring interval.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 28-50
+
+Below is the final code for our script.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+
+.. btest:: ftp-bruteforce
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
+    @TEST-EXEC: btest-rst-include notice.log
+
+As a final note, the :doc:`detect-bruteforcing.bro
+</scripts/policy/protocols/ftp/detect-bruteforcing.bro>` script above is
+included with Bro out of the box.  Use this feature by loading this script
+during startup.
+
+-------------
+Other Attacks
+-------------
+
+Detecting SQL Injection Attacks
+-------------------------------
+
+Checking files against known malware hashes
+-------------------------------------------
+
+Files transmitted on your network could either be completely harmless or
+contain viruses and other threats. One possible action against this
+threat is to compute the hashes of the files and compare them against a
+list of known malware hashes. Bro simplifies this task by offering a
+:doc:`detect-MHR.bro </scripts/policy/frameworks/files/detect-MHR.bro>`
+script that creates and compares hashes against the `Malware Hash
+Registry <https://www.team-cymru.org/Services/MHR/>`_ maintained by Team
+Cymru. Use this feature by loading this script during startup.
--- a/doc/cluster/index.rst
+++ b/doc/cluster/index.rst
@ -6,7 +6,13 @@ Setting up a Bro Cluster
 Intro
 ------

-Bro is not multithreaded, so once the limitations of a single processor core are reached, the only option currently is to spread the workload across many cores or even many physical computers.  The cluster deployment scenario for Bro is the current solution to build these larger systems.  The accompanying tools and scripts provide the structure to easily manage many Bro processes examining packets and doing correlation activities but acting as a singular, cohesive entity.  
+Bro is not multithreaded, so once the limitations of a single processor core
+are reached the only option currently is to spread the workload across many
+cores, or even many physical computers. The cluster deployment scenario for
+Bro is the current solution to build these larger systems. The accompanying
+tools and scripts provide the structure to easily manage many Bro processes
+examining packets and doing correlation activities but acting as a singular,
+cohesive entity.

 Architecture
 ---------------
@ -17,42 +23,98 @@ The figure below illustrates the main components of a Bro cluster.

 Tap
 ***
-This is a mechanism that splits the packet stream in order to make a copy
-available for inspection. Examples include the monitoring port on a switch and
-an optical splitter for fiber networks.
+The tap is a mechanism that splits the packet stream in order to make a copy
+available for inspection. Examples include the monitoring port on a switch
+and an optical splitter on fiber networks.

 Frontend
 ********
-This is a discrete hardware device or on-host technique that will split your traffic into many streams or flows.  The Bro binary does not do this job.  There are numerous ways to accomplish this task, some of which are described below in `Frontend Options`_.
+The frontend is a discrete hardware device or on-host technique that splits
+traffic into many streams or flows. The Bro binary does not do this job.
+There are numerous ways to accomplish this task, some of which are described
+below in `Frontend Options`_.

 Manager
 *******
-This is a Bro process which has two primary jobs.  It receives log messages and notices from the rest of the nodes in the cluster using the Bro communications protocol.  The result is that you will end up with single logs for each log instead of many discrete logs that you have to later combine in some manner with post processing.  The manager also takes the opportunity to de-duplicate notices and it has the ability to do so since it’s acting as the choke point for notices and how notices might be processed into actions such as emailing, paging, or blocking.
+The manager is a Bro process that has two primary jobs.  It receives log
+messages and notices from the rest of the nodes in the cluster using the Bro
+communications protocol.  The result is a single log instead of many
+discrete logs that you have to combine in some manner with post-processing.
+The manager also takes the opportunity to de-duplicate notices, and it has the
+ability to do so since it’s acting as the choke point for notices and how notices
+might be processed into actions (e.g., emailing, paging, or blocking).

-The manager process is started first by BroControl and it only opens it’s designated port and waits for connections, it doesn’t initiate any connections to the rest of the cluster.  Once the workers are started and connect to the manager, logs and notices will start arriving to the manager process from the workers.
+The manager process is started first by BroControl and it only opens its
+designated port and waits for connections, it doesn’t initiate any
+connections to the rest of the cluster.  Once the workers are started and
+connect to the manager, logs and notices will start arriving to the manager
+process from the workers.

 Proxy
 *****
-This is a Bro process which manages synchronized state.  Variables can be synchronized across connected Bro processes automatically in Bro and proxies will help the workers by alleviating the need for all of the workers to connect directly to each other.  
+The proxy is a Bro process that manages synchronized state.  Variables can
+be synchronized across connected Bro processes automatically. Proxies help
+the workers by alleviating the need for all of the workers to connect
+directly to each other.

-Examples of synchronized state from the scripts that ship with Bro are things such as the full list of “known” hosts and services which are hosts or services which have been detected as performing full TCP handshakes or an analyzed protocol has been found on the connection.  If worker A detects host 1.2.3.4 as an active host, it would be beneficial for worker B to know that as well so worker A shares that information as an insertion to a set <link to set documentation would be good here> which travels to the cluster’s proxy and the proxy then sends that same set insertion to worker B.  The result is that worker A and worker B have shared knowledge about host and services that are active on the network being monitored.  
+Examples of synchronized state from the scripts that ship with Bro include
+the full list of “known” hosts and services (which are hosts or services
+identified as performing full TCP handshakes) or an analyzed protocol has been
+found on the connection.  If worker A detects host 1.2.3.4 as an active host,
+it would be beneficial for worker B to know that as well.  So worker A shares
+that information as an insertion to a set
+<link to set documentation would be good here> which travels to the cluster’s
+proxy and the proxy sends that same set insertion to worker B. The result
+is that worker A and worker B have shared knowledge about host and services
+that are active on the network being monitored.

-The proxy model extends to having multiple proxies as well if necessary for performance reasons, it only adds one additional step for the Bro processes.  Each proxy connects to another proxy in a ring and the workers are shared between them as evenly as possible.  When a proxy receives some new bit of state, it will share that with it’s proxy which is then shared around the ring of proxies and down to all of the workers.  From a practical standpoint, there are no rules of thumb established yet for the number of proxies necessary for the number of workers they are serving.  Best is to start with a single proxy and add more if communication performance problems are found.
+The proxy model extends to having multiple proxies when necessary for
+performance reasons. It only adds one additional step for the Bro processes.
+Each proxy connects to another proxy in a ring and the workers are shared
+between them as evenly as possible.  When a proxy receives some new bit of
+state it will share that with its proxy, which is then shared around the
+ring of proxies, and down to all of the workers.  From a practical standpoint,
+there are no rules of thumb established for the number of proxies
+necessary for the number of workers they are serving.  It is best to start
+with a single proxy and add more if communication performance problems are
+found.

-Bro processes acting as proxies don’t tend to be extremely intense to CPU or memory and users frequently run proxy processes on the same physical host as the manager.
+Bro processes acting as proxies don’t tend to be extremely hard on CPU
+or memory and users frequently run proxy processes on the same physical
+host as the manager.

 Worker
 ******
-This is the Bro process that sniffs network traffic and does protocol analysis on the reassembled traffic streams.  Most of the work of an active cluster takes place on the workers and as such, the workers typically represent the bulk of the Bro processes that are running in a cluster.  The fastest memory and CPU core speed you can afford is best here since all of the protocol parsing and most analysis will take place here.   There are no particular requirements for the disks in workers since almost all logging is done remotely to the manager and very little is normally written to disk.
+The worker is the Bro process that sniffs network traffic and does protocol
+analysis on the reassembled traffic streams.  Most of the work of an active
+cluster takes place on the workers and as such, the workers typically
+represent the bulk of the Bro processes that are running in a cluster.
+The fastest memory and CPU core speed you can afford is recommended
+since all of the protocol parsing and most analysis will take place here.
+There are no particular requirements for the disks in workers since almost all
+logging is done remotely to the manager, and normally very little is written
+to disk.

-The rule of thumb we have followed recently is to allocate approximately 1 core for every 80Mbps of traffic that is being analyzed, however this estimate could be extremely traffic mix specific.  It has generally worked for mixed traffic with many users and servers.  For example, if your traffic peaks around 2Gbps (combined) and you want to handle traffic at peak load, you may want to have 26 cores available (2048 / 80 == 25.6).  If the 80Mbps estimate works for your traffic, this could be handled by 3 physical hosts dedicated to being workers with each one containing dual 6-core processors.  
+The rule of thumb we have followed recently is to allocate approximately 1
+core for every 80Mbps of traffic that is being analyzed. However, this
+estimate could be extremely traffic mix-specific.  It has generally worked
+for mixed traffic with many users and servers.  For example, if your traffic
+peaks around 2Gbps (combined) and you want to handle traffic at peak load,
+you may want to have 26 cores available (2048 / 80 == 25.6).  If the 80Mbps
+estimate works for your traffic, this could be handled by 3 physical hosts
+dedicated to being workers with each one containing dual 6-core processors.

-Once a flow based load balancer is put into place this model is extremely easy to scale as well so it’s recommended that you guess at the amount of hardware you will need to fully analyze your traffic.  If it turns out that you need more, it’s relatively easy to increase the size of the cluster in most cases.
+Once a flow-based load balancer is put into place this model is extremely
+easy to scale. It is recommended that you estimate the amount of
+hardware you will need to fully analyze your traffic.  If more is needed it’s
+relatively easy to increase the size of the cluster in most cases.

 Frontend Options
 ----------------

-There are many options for setting up a frontend flow distributor and in many cases it may even be beneficial to do multiple stages of flow distribution on the network and on the host.
+There are many options for setting up a frontend flow distributor.  In many
+cases it is beneficial to do multiple stages of flow distribution
+on the network and on the host.

 Discrete hardware flow balancers
 ********************************
@ -60,12 +122,24 @@ Discrete hardware flow balancers
 cPacket
 ^^^^^^^

-If you are monitoring one or more 10G physical interfaces, the recommended solution is to use either a cFlow or cVu device from cPacket because they are currently being used very successfully at a number of sites.  These devices will perform layer-2 load balancing by rewriting the destination ethernet MAC address to cause each packet associated with a particular flow to have the same destination MAC.  The packets can then be passed directly to a monitoring host where each worker has a BPF filter to limit its visibility to only that stream of flows or onward to a commodity switch to split the traffic out to multiple 1G interfaces for the workers.  This can ultimately greatly reduce costs since workers can use relatively inexpensive 1G interfaces.
+If you are monitoring one or more 10G physical interfaces, the recommended
+solution is to use either a cFlow or cVu device from cPacket because they
+are used successfully at a number of sites.  These devices will perform
+layer-2 load balancing by rewriting the destination Ethernet MAC address
+to cause each packet associated with a particular flow to have the same
+destination MAC.  The packets can then be passed directly to a monitoring
+host where each worker has a BPF filter to limit its visibility to only that
+stream of flows, or onward to a commodity switch to split the traffic out to
+multiple 1G interfaces for the workers.  This greatly reduces
+costs since workers can use relatively inexpensive 1G interfaces.

 OpenFlow Switches
 ^^^^^^^^^^^^^^^^^

-We are currently exploring the use of OpenFlow based switches to do flow based load balancing directly on the switch which can greatly reduce frontend costs for many users.  This document will be updated when we have more information.
+We are currently exploring the use of OpenFlow based switches to do flow-based
+load balancing directly on the switch, which greatly reduces frontend
+costs for many users.  This document will be updated when we have more
+information.

 On host flow balancing
 **********************
@ -73,14 +147,27 @@ On host flow balancing
 PF_RING
 ^^^^^^^

-The PF_RING software for Linux has a “clustering” feature which will do flow based load balancing across a number of processes that are sniffing the same interface.  This will allow you to easily take advantage of multiple cores in a single physical host because Bro’s main event loop is single threaded and can’t natively utilize all of the cores.  More information about Bro with PF_RING can be found here: (someone want to write a quick Bro/PF_RING tutorial to link to here?  document installing kernel module, libpcap wrapper, building Bro with the --with-pcap configure option)
+The PF_RING software for Linux has a “clustering” feature which will do
+flow-based load balancing across a number of processes that are sniffing the
+same interface.  This allows you to easily take advantage of multiple
+cores in a single physical host because Bro’s main event loop is single
+threaded and can’t natively utilize all of the cores.  More information about
+Bro with PF_RING can be found here: (someone want to write a quick Bro/PF_RING
+tutorial to link to here?  document installing kernel module, libpcap
+wrapper, building Bro with the --with-pcap configure option)

 Netmap
 ^^^^^^

-FreeBSD has an in-progress project named Netmap which will enable flow based load balancing as well.  When it becomes viable for real world use, this document will be updated.
+FreeBSD has an in-progress project named Netmap which will enable flow-based
+load balancing as well.  When it becomes viable for real world use, this
+document will be updated.

 Click! Software Router
 ^^^^^^^^^^^^^^^^^^^^^^

-Click! can be used for flow based load balancing with a simple configuration.  (link to an example for the config).  This solution is not recommended on Linux due to Bro’s PF_RING support and only as a last resort on other operating systems since it causes a lot of overhead due to context switching back and forth between kernel and userland several times per packet.
+Click! can be used for flow based load balancing with a simple configuration.
+(link to an example for the config).  This solution is not recommended on
+Linux due to Bro’s PF_RING support and only as a last resort on other
+operating systems since it causes a lot of overhead due to context switching
+back and forth between kernel and userland several times per packet.
--- a/doc/frameworks/file-analysis.rst
+++ b/doc/frameworks/file-analysis.rst
@ -1,3 +1,6 @@
+
+.. _file-analysis-framework:
+
 =============
 File Analysis
 =============
--- a/doc/frameworks/notice.rst
+++ b/doc/frameworks/notice.rst
@ -1,4 +1,6 @@

+.. _notice-framework:
+
 Notice Framework
 ================

--- a/doc/frameworks/sumstats.rst
+++ b/doc/frameworks/sumstats.rst
@ -1,3 +1,6 @@
+
+.. _sumstats-framework:
+
 ==================
 Summary Statistics
 ==================
--- a/doc/httpmonitor/file_extraction.bro
+++ b/doc/httpmonitor/file_extraction.bro
@ -0,0 +1,24 @@
+
+global mime_to_ext: table[string] of string = {
+	["application/x-dosexec"] = "exe",
+	["text/plain"] = "txt",
+	["image/jpeg"] = "jpg",
+	["image/png"] = "png",
+	["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+	{
+	if ( f$source != "HTTP" )
+		return;
+
+	if ( ! f?$mime_type )
+		return;
+
+	if ( f$mime_type !in mime_to_ext )
+		return;
+
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	print fmt("Extracting file %s", fname);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}
--- a/doc/httpmonitor/http_proxy_01.bro
+++ b/doc/httpmonitor/http_proxy_01.bro
@ -0,0 +1,5 @@
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}
--- a/doc/httpmonitor/http_proxy_02.bro
+++ b/doc/httpmonitor/http_proxy_02.bro
@ -0,0 +1,26 @@
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}
--- a/doc/httpmonitor/http_proxy_03.bro
+++ b/doc/httpmonitor/http_proxy_03.bro
@ -0,0 +1,31 @@
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}
--- a/doc/httpmonitor/http_proxy_04.bro
+++ b/doc/httpmonitor/http_proxy_04.bro
@ -0,0 +1,40 @@
+@load base/utils/site
+@load base/frameworks/notice
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	redef enum Notice::Type += {
+		Open_Proxy
+	};
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		NOTICE([$note=HTTP::Open_Proxy,
+		        $msg=fmt("A local server is acting as an open proxy: %s",
+		                 c$id$resp_h),
+		        $conn=c,
+		        $identifier=cat(c$id$resp_h),
+		        $suppress_for=1day]);
+	}
--- a/doc/httpmonitor/index.rst
+++ b/doc/httpmonitor/index.rst
@ -0,0 +1,162 @@
+
+.. _http-monitor:
+
+================================
+Monitoring HTTP Traffic with Bro
+================================
+
+Bro can be used to log the entire HTTP traffic from your network to the
+http.log file.  This file can then be used for analysis and auditing
+purposes.
+
+In the sections below we briefly explain the structure of the http.log
+file, then we show you how to perform basic HTTP traffic monitoring and
+analysis tasks with Bro. Some of these ideas and techniques can later be
+applied to monitor different protocols in a similar way.
+
+----------------------------
+Introduction to the HTTP log
+----------------------------
+
+The http.log file contains a summary of all HTTP requests and responses
+sent over a Bro-monitored network. Here are the first few columns of
+``http.log``::
+
+    # ts          uid          orig_h        orig_p  resp_h         resp_p
+    1311627961.8  HSH4uV8KVJg  192.168.1.100 52303   192.150.187.43 80
+
+Every single line in this log starts with a timestamp, a unique
+connection identifier (UID), and a connection 4-tuple (originator
+host/port and responder host/port).  The UID can be used to identify all
+logged activity (possibly across multiple log files) associated with a
+given connection 4-tuple over its lifetime.
+
+The remaining columns detail the activity that's occurring.  For
+example, the columns on the line below (shortened for brevity) show a
+request to the root of Bro website::
+
+    # method   host         uri  referrer  user_agent
+    GET        bro.org  /    -         <...>Chrome/12.0.742.122<...>
+
+Network administrators and security engineers, for instance, can use the
+information in this log to understand the HTTP activity on the network
+and troubleshoot network problems or search for anomalous activities. We must 
+stress that there is no single right way to perform an analysis. It will 
+depend on the expertise of the person performing the analysis and the 
+specific details of the task.
+
+For more information about how to handle the HTTP protocol in Bro,
+including a complete list of the fields available in http.log, go to
+Bro's :doc:`HTTP script reference
+</scripts/base/protocols/http/main.bro>`.
+
+------------------------
+Detecting a Proxy Server
+------------------------
+
+A proxy server is a device on your network configured to request a
+service on behalf of a third system; one of the most common examples is
+a Web proxy server. A client without Internet access connects to the
+proxy and requests a web page, the proxy sends the request to the web 
+server, which receives the response, and passes it to the original 
+client.
+
+Proxies were conceived to help manage a network and provide better
+encapsulation. Proxies by themselves are not a security threat, but a
+misconfigured or unauthorized proxy can allow others, either inside or
+outside the network, to access any web site and even conduct malicious
+activities anonymously using the network's resources.
+
+What Proxy Server traffic looks like
+-------------------------------------
+
+In general, when a client starts talking with a proxy server, the
+traffic consists of two parts: (i) a GET request, and (ii) an HTTP/
+reply::
+
+    Request: GET http://www.bro.org/ HTTP/1.1
+    Reply:   HTTP/1.0 200 OK
+
+This will differ from traffic between a client and a normal Web server
+because GET requests should not include "http" on the string. So we can
+use this to identify a proxy server.
+
+We can write a basic script in Bro to handle the http_reply event and
+detect a reply for a ``GET http://`` request.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
+
+.. btest:: http_proxy_01
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
+
+Basically, the script is checking for a "200 OK" status code on a reply
+for a request that includes "http:" (case insensitive). In reality, the
+HTTP protocol defines several success status codes other than 200, so we
+will extend our basic script to also consider the additional codes.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
+
+.. btest:: http_proxy_02
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
+
+Next, we will make sure that the responding proxy is part of our local
+network.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
+
+.. btest:: http_proxy_03
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
+
+.. note::
+
+    The redefinition of :bro:see:`Site::local_nets` is only done inside
+    this script to make it a self-contained example.  It's typically
+    redefined somewhere else.
+
+Finally, our goal should be to generate an alert when a proxy has been
+detected instead of printing a message on the console output.  For that,
+we will tag the traffic accordingly and define a new ``Open_Proxy``
+``Notice`` type to alert of all tagged communications. Once a
+notification has been fired, we will further suppress it for one day.
+Below is the complete script.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+
+.. btest:: http_proxy_04
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+    @TEST-EXEC: btest-rst-include notice.log
+
+Note that this script only logs the presence of the proxy to
+``notice.log``, but if an additional email is desired (and email
+functionality is enabled), then that's done simply by redefining
+:bro:see:`Notice::emailed_types` to add the ``Open_proxy`` notice type
+to it.
+
+----------------
+Inspecting Files
+----------------
+
+Files are often transmitted on regular HTTP conversations between a
+client and a server. Most of the time these files are harmless, just
+images and some other multimedia content, but there are also types of
+files, specially executable files, that can damage your system. We can
+instruct Bro to create a copy of all files of certain types that it sees
+using the :ref:`File Analysis Framework <file-analysis-framework>`
+(introduced with Bro 2.2):
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/file_extraction.bro
+
+.. btest:: file_extraction
+
+    @TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro
+
+Here, the ``mime_to_ext`` table serves two purposes.  It defines which
+mime types to extract and also the file suffix of the extracted files.
+Extracted files are written to a new ``extract_files`` subdirectory.
+Also note that the first conditional in the :bro:see:`file_new` event
+handler can be removed to make this behavior generic to other protocols
+besides HTTP.
--- a/doc/index.rst
+++ b/doc/index.rst
@ -1,9 +1,12 @@

 .. Bro documentation master file

-=================
-Bro Documentation
-=================
+==========
+Bro Manual
+==========
+
+Introduction Section
+====================

 .. toctree::
   :maxdepth: 2
@ -11,13 +14,36 @@ Bro Documentation
   intro/index.rst
   install/index.rst
   quickstart/index.rst
-   using/index.rst
+
+.. 
+
+Using Bro Section
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   logs/index.rst
+   httpmonitor/index.rst
+   broids/index.rst
+   mimestats/index.rst
+   cluster/index.rst
+
+..
+
+Reference Section
+=================
+
+.. toctree::
+   :maxdepth: 2
+
   scripting/index.rst
   frameworks/index.rst
-   cluster/index.rst
   script-reference/index.rst
   components/index.rst

+..
+
 * :ref:`General Index <genindex>`
 * :ref:`search`

--- a/doc/install/guidelines.rst
+++ b/doc/install/guidelines.rst
@ -12,32 +12,31 @@ local customizations over. In the following we summarize general
 guidelines for upgrading, see the :ref:`release-notes` for
 version-specific information.

-Re-Using Previous Install Prefix
+Reusing Previous Install Prefix
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

 If you choose to configure and install Bro with the same prefix
 directory as before, local customization and configuration to files in
 ``$prefix/share/bro/site`` and ``$prefix/etc`` won't be overwritten
 (``$prefix`` indicating the root of where Bro was installed). Also, logs
-generated at run-time won't be touched by the upgrade. (But making
-a backup of local changes before upgrading is still recommended.)
+generated at run-time won't be touched by the upgrade. Backing up local
+changes before upgrading is still recommended.

 After upgrading, remember to check ``$prefix/share/bro/site`` and
-``$prefix/etc`` for ``.example`` files, which indicate the
-distribution's version of the file differs from the local one, which may
-include local changes.  Review the differences, and make adjustments
-as necessary (for differences that aren't the result of a local change,
-use the new version's).
+``$prefix/etc`` for ``.example`` files, which indicate that the
+distribution's version of the file differs from the local one, and therefore,
+may include local changes.  Review the differences and make adjustments
+as necessary. Use the new version for differences that aren't a result of
+a local change.

-Using a New Install prefix
+Using a New Install Prefix
 ~~~~~~~~~~~~~~~~~~~~~~~~~~

-If you want to install the newer version in a different prefix
-directory than before, you can just copy local customization and
-configuration files from ``$prefix/share/bro/site`` and ``$prefix/etc``
-to the new location (``$prefix`` indicating the root of where Bro was
-originally installed).  Make sure to review the files for difference
-before copying and make adjustments as necessary (for differences that
-aren't the result of a local change, use the new version's).  Of
-particular note, the copied version of ``$prefix/etc/broctl.cfg`` is
-likely to need changes to the ``SpoolDir`` and ``LogDir`` settings.
+To install the newer version in a different prefix directory than before,
+copy local customization and configuration files from ``$prefix/share/bro/site``
+and ``$prefix/etc`` to the new location (``$prefix`` indicating the root of
+where Bro was originally installed).  Review the files for differences
+before copying and make adjustments as necessary (use the new version for
+differences that aren't a result of a local change).  Of particular note,
+the copied version of ``$prefix/etc/broctl.cfg`` is likely to need changes
+to the ``SpoolDir`` and ``LogDir`` settings.
--- a/doc/install/install.rst
+++ b/doc/install/install.rst
@ -3,7 +3,7 @@
 .. _Xcode: https://developer.apple.com/xcode/
 .. _MacPorts: http://www.macports.org
 .. _Fink: http://www.finkproject.org
-.. _Homebrew: http://mxcl.github.com/homebrew
+.. _Homebrew: http://brew.sh
 .. _bro downloads page: http://bro.org/download/index.html

 .. _installing-bro:
@ -29,14 +29,13 @@ before you begin:
    * Libpcap                           (http://www.tcpdump.org)
    * OpenSSL libraries                 (http://www.openssl.org)
    * BIND8 library
-    * Libmagic 5.04 or greater
    * Libz
    * Bash (for BroControl)
    * Python (for BroControl)

 To build Bro from source, the following additional dependencies are required:

-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8.0 or greater            (http://www.cmake.org)
    * Make
    * C/C++ compiler
    * SWIG                              (http://www.swig.org)
@ -44,7 +43,6 @@ To build Bro from source, the following additional dependencies are required:
    * Flex  (Fast Lexical Analyzer)
    * Libpcap headers                   (http://www.tcpdump.org)
    * OpenSSL headers                   (http://www.openssl.org)
-    * libmagic headers
    * zlib headers
    * Perl

@ -55,13 +53,13 @@ that ``bash`` and ``python`` are in your ``PATH``):

  .. console::

-     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel

 * DEB/Debian-based Linux:

  .. console::

-     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev

 * FreeBSD:

@ -78,15 +76,11 @@ that ``bash`` and ``python`` are in your ``PATH``):
  then going through its "Preferences..." -> "Downloads" menus to
  install the "Command Line Tools" component.

-  Lion (10.7) and Mountain Lion (10.8) come with all required
-  dependencies except for CMake_, SWIG_, and ``libmagic``.
-
+  OS X comes with all required dependencies except for CMake_ and SWIG_.
  Distributions of these dependencies can likely be obtained from your
  preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
-  or Homebrew_).
-
-  Specifically for MacPorts, the ``cmake``, ``swig``,
-  ``swig-python`` and ``file`` packages provide the required dependencies.
+  or Homebrew_).  Specifically for MacPorts, the ``cmake``, ``swig``,
+  ``swig-python`` and packages provide the required dependencies.


 Optional Dependencies
@ -95,7 +89,7 @@ Optional Dependencies
 Bro can make use of some optional libraries and tools if they are found at
 build time:

-    * LibGeoIP (for geo-locating IP addresses)
+    * LibGeoIP (for geolocating IP addresses)
    * sendmail (enables Bro and BroControl to send mail)
    * gawk (enables all features of bro-cut)
    * curl (used by a Bro script that implements active HTTP)
@ -143,14 +137,14 @@ The primary install prefix for binary packages is ``/opt/bro``.
 Non-MacOS packages that include BroControl also put variable/runtime
 data (e.g. Bro logs) in ``/var/opt/bro``.

-Installing From Source
+Installing from Source
 ==========================

-Bro releases are bundled into source packages for convenience and
-available from the `bro downloads page`_. Alternatively, the latest
+Bro releases are bundled into source packages for convenience and are
+available on the `bro downloads page`_. Alternatively, the latest
 Bro development version can be obtained through git repositories
 hosted at ``git.bro.org``.  See our `git development documentation
-<http://bro.org/development/process.html>`_ for comprehensive
+<http://bro.org/development/howtos/process.html>`_ for comprehensive
 information on Bro's use of git revision control, but the short story
 for downloading the full source code experience for Bro via git is:

--- a/doc/using/index.rst
+++ b/doc/using/index.rst
@ -1,9 +1,9 @@

-.. _using-bro:
+.. _bro-logging:

-=========
-Using Bro
-=========
+===========
+Bro Logging
+===========

 .. contents::

@ -24,17 +24,17 @@ Working with Log Files

 Generally, all of Bro's log files are produced by a corresponding
 script that defines their individual structure. However, as each log
-file flows through the Logging Framework, there share a set of
+file flows through the Logging Framework, they share a set of
 structural similarities. Without breaking into the scripting aspect of
-Bro here, a bird's eye view of how the log files are produced would
-progress as follows.  The script's author defines the kinds of data,
+Bro here, a bird's eye view of how the log files are produced
+progresses as follows.  The script's author defines the kinds of data,
 such as the originating IP address or the duration of a connection,
 which will make up the fields (i.e., columns) of the log file.  The
 author then decides what network activity should generate a single log
-file entry (i.e., one line); that could, e.g., be a connection having
-been completed or an HTTP ``GET`` method being issued by an
+file entry (i.e., one line). For example, this could be a connection 
+having been completed or an HTTP ``GET`` request being issued by an
 originator. When these behaviors are observed during operation, the
-data is passed to the Logging Framework which, in turn, adds the entry
+data is passed to the Logging Framework which adds the entry
 to the appropriate log file.

 As the fields of the log entries can be further customized by the
@ -57,7 +57,7 @@ data, the string ``(empty)`` as the indicator for an empty field and
 the ``-`` character as the indicator for a field that hasn't been set.
 The timestamp for when the file was created is included under
 ``#open``. The header then goes on to detail the fields being listed
-in the file and the data types of those fields in ``#fields`` and
+in the file and the data types of those fields, in ``#fields`` and
 ``#types``, respectively. These two entries are often the two most
 significant points of interest as they detail not only the field names
 but the data types used. When navigating through the different log
@ -66,12 +66,12 @@ definitions readily available saves the user some mental leg work. The
 field names are also a key resource for using the :ref:`bro-cut
 <bro-cut>` utility included with Bro, see below.

-Next to the header follows the main content; in this example we see 7
+Next to the header follows the main content. In this example we see 7
 connections with their key properties, such as originator and
-responder IP addresses (note how Bro transparely handles both IPv4 and
-IPv6), transport-layer ports, application-layer services - the
-``service`` field is filled ias Bro determines a specific protocol to
-be in use, independent of the connection's ports - payload size, and
+responder IP addresses (note how Bro transparently handles both IPv4 and
+IPv6), transport-layer ports, application-layer services ( - the
+``service`` field is filled in as Bro determines a specific protocol to
+be in use, independent of the connection's ports), payload size, and
 more. See :bro:type:`Conn::Info` for a description of all fields.

 In addition to ``conn.log``, Bro generates many further logs by
@ -87,8 +87,8 @@ default, including:
    A log of FTP session-level activity.

 ``files.log``
-    Summaries of files transfered over the network. This information
-    is aggregrated from different protocols, including HTTP, FTP, and
+    Summaries of files transferred over the network. This information
+    is aggregated from different protocols, including HTTP, FTP, and
    SMTP.

 ``http.log``
@ -106,7 +106,7 @@ default, including:
 ``weird.log``
    A log of unexpected protocol-level activity. Whenever Bro's
    protocol analysis encounters a situation it would not expect
-    (e.g., an RFC violation) is logs it in this file. Note that in
+    (e.g., an RFC violation) it logs it in this file. Note that in
    practice, real-world networks tend to exhibit a large number of
    such "crud" that is usually not worth following up on.

@ -120,7 +120,7 @@ Using ``bro-cut``

 The ``bro-cut`` utility can be used in place of other tools to build
 terminal commands that remain flexible and accurate independent of
-possible changes to log file itself.  It accomplishes this by parsing
+possible changes to the log file itself.  It accomplishes this by parsing
 the header in each file and allowing the user to refer to the specific
 columnar data available (in contrast to tools like ``awk`` that
 require the user to refer to fields referenced by their position).
@ -131,7 +131,7 @@ from a ``conn.log``:

   @TEST-EXEC: btest-rst-cmd -n 10 "cat conn.log | bro-cut id.orig_h id.orig_p id.resp_h duration"

-The correspding ``awk`` command would look like this:
+The corresponding ``awk`` command will look like this:

 .. btest:: using_bro

@ -185,8 +185,8 @@ Working with Timestamps

 ``bro-cut`` accepts the flag ``-d`` to convert the epoch time values
 in the log files to human-readable format.  The following command
-includes the human readable time stamp, the unique identifier and the
-HTTP ``Host`` and HTTP ``URI`` as extracted from the ``http.log``
+includes the human readable time stamp, the unique identifier, the
+HTTP ``Host``, and HTTP ``URI`` as extracted from the ``http.log``
 file:

 .. btest:: using_bro
@ -218,7 +218,7 @@ See ``man strfime`` for more options for the format string.
 Using UIDs
 ----------

-While Bro can do signature based analysis, its primary focus is on
+While Bro can do signature-based analysis, its primary focus is on
 behavioral detection which alters the practice of log review from
 "reactionary review" to a process a little more akin to a hunting
 trip.  A common progression of review includes correlating a session
@ -251,3 +251,43 @@ stream and Bro is able to extract and track that information for you,
 giving you an in-depth and structured view into HTTP traffic on your
 network.

+-----------------------
+Common Log Files
+-----------------------
+As a monitoring tool, Bro records a detailed view of the traffic inspected 
+and the events generated in a series of relevant log files. These files can 
+later be reviewed for monitoring, auditing and troubleshooting purposes.
+
+In this section we present a brief explanation of the most commonly used log 
+files generated by Bro including links to descriptions of some of the fields 
+for each log type.
+
+-----------------+---------------------------------------+------------------------------+
+| Log File        | Description                           | Field Descriptions           |
+=================+=======================================+==============================+
+| http.log        | Shows all HTTP requests and replies   | :bro:type:`HTTP::Info`       |
+-----------------+---------------------------------------+------------------------------+
+| ftp.log         | Records FTP activity                  | :bro:type:`FTP::Info`        |
+-----------------+---------------------------------------+------------------------------+
+| ssl.log         | Records SSL sessions including        | :bro:type:`SSL::Info`        |
+|                 | certificates used                     |                              |
+-----------------+---------------------------------------+------------------------------+
+| known_certs.log | Includes SSL certificates used        | :bro:type:`Known::CertsInfo` |
+-----------------+---------------------------------------+------------------------------+
+| smtp.log        | Summarizes SMTP traffic on a network  | :bro:type:`SMTP::Info`       |
+-----------------+---------------------------------------+------------------------------+
+| dns.log         | Shows all DNS activity on a network   | :bro:type:`DNS::Info`        |
+-----------------+---------------------------------------+------------------------------+
+| conn.log        | Records all connections seen by Bro   | :bro:type:`Conn::Info`       |
+-----------------+---------------------------------------+------------------------------+
+| dpd.log         | Shows network activity on             | :bro:type:`DPD::Info`        |
+|                 | non-standard ports                    |                              |
+-----------------+---------------------------------------+------------------------------+
+| files.log       | Records information about all files   | :bro:type:`Files::Info`      |
+|                 | transmitted over the network          |                              |
+-----------------+---------------------------------------+------------------------------+
+| weird.log       | Records unexpected protocol-level     | :bro:type:`Weird::Info`      |
+|                 | activity                              |                              |
+-----------------+---------------------------------------+------------------------------+
+
+
--- a/doc/mimestats/index.rst
+++ b/doc/mimestats/index.rst
@ -0,0 +1,71 @@
+
+.. _mime-stats:
+
+====================
+MIME Type Statistics
+====================
+
+Files are constantly transmitted over HTTP on regular networks. These
+files belong to a specific category (e.g., executable, text, image) 
+identified by a `Multipurpose Internet Mail Extension (MIME)
+<http://en.wikipedia.org/wiki/MIME>`_. Although MIME was originally
+developed to identify the type of non-text attachments on email, it is
+also used by a web browser to identify the type of files transmitted and
+present them accordingly.
+
+In this tutorial, we will demonstrate how to use the Sumstats Framework 
+to collect statistical information based on MIME types; specifically,
+the total number of occurrences, size in bytes, and number of unique
+hosts transmitting files over HTTP per each type. For instructions on
+extracting and creating a local copy of these files, visit :ref:`this 
+tutorial <http-monitor>`.
+
+------------------------------------------------
+MIME Statistics with Sumstats
+------------------------------------------------
+
+When working with the :ref:`Summary Statistics Framework
+<sumstats-framework>`, you need to define three different pieces: (i)
+Observations, where the event is observed and fed into the framework.
+(ii) Reducers, where observations are collected and measured. (iii)
+Sumstats, where the main functionality is implemented.
+
+We start by defining our observation along with a record to store
+all statistical values and an observation interval. We are conducting our
+observation on the :bro:see:`HTTP::log_http` event and are interested
+in the MIME type, size of the file ("response_body_len"), and the
+originator host ("orig_h"). We use the MIME type as our key and create
+observers for the other two values.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+    :lines: 6-29, 54-64
+
+Next, we create the reducers. The first will accumulate file sizes
+and the second will make sure we only store a host ID once. Below is
+the partial code from a :bro:see:`bro_init` handler.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+    :lines: 34-37
+
+In our final step, we create the SumStats where we check for the
+observation interval.  Once it expires, we populate the record
+(defined above) with all the relevant data and write it to a log.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+    :lines: 38-51
+
+After putting the three pieces together we end up with the following final code for
+our script.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+
+.. btest:: mimestats
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro
+    @TEST-EXEC: btest-rst-include mime_metrics.log
+
+.. note::
+
+    The redefinition of :bro:see:`Site::local_nets` is only done inside
+    this script to make it a self-contained example.  It's typically
+    redefined somewhere else.
--- a/doc/mimestats/mimestats.bro
+++ b/doc/mimestats/mimestats.bro
@ -0,0 +1,64 @@
+@load base/utils/site
+@load base/frameworks/sumstats
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}
--- a/doc/quickstart/index.rst
+++ b/doc/quickstart/index.rst
@ -13,7 +13,7 @@ Bro works on most modern, Unix-based systems and requires no custom
 hardware.  It can be downloaded in either pre-built binary package or
 source code forms.  See :ref:`installing-bro` for instructions on how to
 install Bro. Below, ``$PREFIX`` is used to reference the Bro
-installation root directory, which by default is ``/usr/local/`` if
+installation root directory, which by default is ``/usr/local/bro/`` if
 you install from source. 

 Managing Bro with BroControl
@ -26,8 +26,8 @@ traffic-monitoring cluster.
 A Minimal Starting Configuration
 --------------------------------

-These are the basic configuration changes to make for a minimal BroControl installation
-that will manage a single Bro instance on the ``localhost``:
+These are the basic configuration changes to make for a minimal BroControl
+installation that will manage a single Bro instance on the ``localhost``:

 1) In ``$PREFIX/etc/node.cfg``, set the right interface to monitor.
 2) In ``$PREFIX/etc/networks.cfg``, comment out the default settings and add
@ -72,7 +72,8 @@ You can leave it running for now, but to stop this Bro instance you would do:

   [BroControl] > stop

-We also recommend to insert the following entry into `crontab`::
+We also recommend to insert the following entry into the crontab of the user
+running BroControl::

      0-59/5 * * * * $PREFIX/bin/broctl cron

@ -154,7 +155,7 @@ changes we want to make:
   attempt looks like it may have been successful, and we want email when
   that happens, but only for certain servers.

-So we've defined *what* we want to do, but need to know *where* to do it.
+We've defined *what* we want to do, but need to know *where* to do it.
 The answer is to use a script written in the Bro programming language, so
 let's do a quick intro to Bro scripting.

@ -180,7 +181,7 @@ must explicitly choose if they want to load them.

 The main entry point for the default analysis configuration of a standalone
 Bro instance managed by BroControl is the ``$PREFIX/share/bro/site/local.bro``
-script.  So we'll be adding to that in the following sections, but first
+script.  We'll be adding to that in the following sections, but first
 we have to figure out what to add.

 Redefining Script Option Variables
@ -196,7 +197,7 @@ A redefineable constant might seem strange, but what that really means is that
 the variable's value may not change at run-time, but whose initial value can be
 modified via the ``redef`` operator at parse-time.

-So let's continue on our path to modify the behavior for the two SSL
+Let's continue on our path to modify the behavior for the two SSL
 and SSH notices.  Looking at :doc:`/scripts/base/frameworks/notice/main.bro`,
 we see that it advertises:

@ -210,7 +211,7 @@ we see that it advertises:
        const ignored_types: set[Notice::Type] = {} &redef;
    }

-That's exactly what we want to do for the SSL notice.  So add to ``local.bro``:
+That's exactly what we want to do for the SSL notice.  Add to ``local.bro``:

 .. code:: bro

@ -275,9 +276,9 @@ an email on the condition that the predicate function evaluates to true, which
 is whenever the notice type is an SSH login and the responding host stored
 inside the ``Info`` record's connection field is in the set of watched servers.

-.. note:: record field member access is done with the '$' character
+.. note:: Record field member access is done with the '$' character
   instead of a '.' as might be expected from other languages, in
-   order to avoid ambiguity with the builtin address type's use of '.'
+   order to avoid ambiguity with the built-in address type's use of '.'
   in IPv4 dotted decimal representations.

 Remember, to finalize that configuration change perform the ``check``,
@ -406,7 +407,7 @@ logging) and adds SSL certificate validation.
 You might notice that a script you load from the command line uses the
 ``@load`` directive in the Bro language to declare dependence on other scripts.
 This directive is similar to the ``#include`` of C/C++, except the semantics
-are "load this script if it hasn't already been loaded".
+are, "load this script if it hasn't already been loaded."

 .. note:: If one wants Bro to be able to load scripts that live outside the
   default directories in Bro's installation root, the ``BROPATH`` environment
--- a/doc/script-reference/builtins.rst
+++ b/doc/script-reference/builtins.rst
@ -23,7 +23,8 @@ The Bro scripting language supports the following built-in types.

 .. bro:type:: void

-    An internal Bro type representing the absence of a return type for a
+    An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
+    scripting language) representing the absence of a return type for a
    function.

 .. bro:type:: bool
@ -132,10 +133,23 @@ The Bro scripting language supports the following built-in types.

    Strings support concatenation (``+``), and assignment (``=``, ``+=``).
    Strings also support the comparison operators (``==``, ``!=``, ``<``,
-    ``<=``, ``>``, ``>=``).  Substring searching can be performed using
-    the "in" or "!in" operators (e.g., "bar" in "foobar" yields true).
-    The number of characters in a string can be found by enclosing the
-    string within pipe characters (e.g., ``|"abc"|`` is 3).
+    ``<=``, ``>``, ``>=``).  The number of characters in a string can be
+    found by enclosing the string within pipe characters (e.g., ``|"abc"|``
+    is 3).
+
+    The subscript operator can extract an individual character or a substring
+    of a string (string indexing is zero-based, but an index of
+    -1 refers to the last character in the string, and -2 refers to the
+    second-to-last character, etc.).  When extracting a substring, the
+    starting and ending index values are separated by a colon.  For example::
+
+        local orig = "0123456789";
+        local third_char = orig[2];
+        local last_char = orig[-1];
+        local first_three_chars = orig[0:2];
+
+    Substring searching can be performed using the "in" or "!in"
+    operators (e.g., "bar" in "foobar" yields true).

    Note that Bro represents strings internally as a count and vector of
    bytes rather than a NUL-terminated byte string (although string
@ -767,7 +781,7 @@ The Bro scripting language supports the following built-in types.
 .. bro:type:: hook

    A hook is another flavor of function that shares characteristics of
-    both a :bro:type:`function` and a :bro:type:`event`.  They are like
+    both a :bro:type:`function` and an :bro:type:`event`.  They are like
    events in that many handler bodies can be defined for the same hook
    identifier and the order of execution can be enforced with
    :bro:attr:`&priority`.  They are more like functions in the way they
@ -856,14 +870,14 @@ scripting language supports the following built-in attributes.
 .. bro:attr:: &optional

    Allows a record field to be missing. For example the type ``record {
-    a: int, b: port &optional }`` could be instantiated both as
+    a: addr; b: port &optional; }`` could be instantiated both as
    singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``.

 .. bro:attr:: &default

    Uses a default value for a record field, a function/hook/event
    parameter, or container elements.  For example, ``table[int] of
-    string &default="foo" }`` would create a table that returns the
+    string &default="foo"`` would create a table that returns the
    :bro:type:`string` ``"foo"`` for any non-existing index.

 .. bro:attr:: &redef
@ -901,7 +915,7 @@ scripting language supports the following built-in attributes.
    Called right before a container element expires.  The function's
    first parameter is of the same type of the container and the second
    parameter the same type of the container's index.  The return
-    value is a :bro:type:`interval` indicating the amount of additional
+    value is an :bro:type:`interval` indicating the amount of additional
    time to wait before expiring the container element at the given
    index (which will trigger another execution of this function).

@ -925,7 +939,7 @@ scripting language supports the following built-in attributes.

 .. bro:attr:: &persistent

-    Makes a variable persistent, i.e., its value is writen to disk (per
+    Makes a variable persistent, i.e., its value is written to disk (per
    default at shutdown time).

 .. bro:attr:: &synchronized
@ -957,8 +971,9 @@ scripting language supports the following built-in attributes.

 .. bro:attr:: &priority

-    Specifies the execution priority of an event handler. Higher values
-    are executed before lower ones. The default value is 0.
+    Specifies the execution priority (as a signed integer) of a hook or
+    event handler. Higher values are executed before lower ones. The
+    default value is 0.

 .. bro:attr:: &group

--- a/doc/scripting/connection_record_02.bro
+++ b/doc/scripting/connection_record_02.bro
@ -1,5 +1,5 @@
@load base/protocols/conn
-@load base/protocols/dns
+@load base/protocols/http

 event connection_state_remove(c: connection)
    {
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@ -232,7 +232,7 @@ overly populated.

 .. btest:: connection-record-01

-    @TEST-EXEC: btest-rst-cmd bro -b -r ${TRACES}/dns-session.trace ${DOC_ROOT}/scripting/connection_record_01.bro
+    @TEST-EXEC: btest-rst-cmd bro -b -r ${TRACES}/http/get.trace ${DOC_ROOT}/scripting/connection_record_01.bro

 As you can see from the output, the connection record is something of
 a jumble when printed on its own.  Regularly taking a peek at a
@ -248,9 +248,9 @@ originating host is referenced by ``c$id$orig_h`` which if given a
 narrative relates to ``orig_h`` which is a member of ``id`` which is
 a member of the data structure referred to as ``c`` that was passed
 into the event handler." Given that the responder port
-(``c$id$resp_p``) is ``53/tcp``, it's likely that Bro's base DNS scripts
+(``c$id$resp_p``) is ``53/tcp``, it's likely that Bro's base HTTP scripts
 can further populate the connection record.  Let's load the
-``base/protocols/dns`` scripts and check the output of our script. 
+``base/protocols/http`` scripts and check the output of our script. 

 Bro uses the dollar sign as its field delimiter and a direct
 correlation exists between the output of the connection record and the
@ -262,16 +262,16 @@ brackets, which would correspond to the ``$``-delimiter in a Bro script.

 .. btest:: connection-record-02

-    @TEST-EXEC: btest-rst-cmd bro -b -r ${TRACES}/dns-session.trace ${DOC_ROOT}/scripting/connection_record_02.bro
+    @TEST-EXEC: btest-rst-cmd bro -b -r ${TRACES}/http/get.trace ${DOC_ROOT}/scripting/connection_record_02.bro

-The addition of the ``base/protocols/dns`` scripts populates the
-``dns=[]`` member of the connection record.  While Bro is doing a
+The addition of the ``base/protocols/http`` scripts populates the
+``http=[]`` member of the connection record.  While Bro is doing a
 massive amount of work in the background, it is in what is commonly
 called "scriptland" that details are being refined and decisions
 being made. Were we to continue running in "bare mode" we could slowly
 keep adding infrastructure through ``@load`` statements.  For example,
 were we to ``@load base/frameworks/logging``, Bro would generate a
-``conn.log`` and ``dns.log`` for us in the current working directory.
+``conn.log`` and ``http.log`` for us in the current working directory.
 As mentioned above, including the appropriate ``@load`` statements is
 not only good practice, but can also help to indicate which
 functionalities are being used in a script.  Take a second to run the
--- a/2
+++ b/2
@ -1 +1 @@
-Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed
+Subproject commit 99c6b89230e2b9b0e781c42b0b9412d2ab4e14b2
--- a/scripts/base/files/unified2/main.bro
+++ b/scripts/base/files/unified2/main.bro
@ -7,10 +7,10 @@ module Unified2;
 export {
 	redef enum Log::ID += { LOG };

-	## Directory to watch for Unified2 files.
+	## File to watch for Unified2 files.
 	const watch_file = "" &redef;

-	## File to watch for Unified2 records.
+	## Directory to watch for Unified2 records.
 	const watch_dir = "" &redef;

 	## The sid-msg.map file you would like to use for your alerts.
--- a/scripts/base/frameworks/notice/cluster.bro
+++ b/scripts/base/frameworks/notice/cluster.bro
@ -23,7 +23,8 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
@if ( Cluster::local_node_type() != Cluster::MANAGER )
 event Notice::begin_suppression(n: Notice::Info)
 	{
-	suppressing[n$note, n$identifier] = n;
+	local suppress_until = n$ts + n$suppress_for;
+	suppressing[n$note, n$identifier] = suppress_until;
 	}
@endif

--- a/scripts/base/frameworks/notice/main.bro
+++ b/scripts/base/frameworks/notice/main.bro
@ -242,12 +242,6 @@ export {
 	##    being suppressed.
 	global suppressed: event(n: Notice::Info);

-	## This event is generated when a notice stops being suppressed.
-	##
-	## n: The record containing notice data regarding the notice type
-	##    that was being suppressed.
-	global end_suppression: event(n: Notice::Info);
-
 	## Call this function to send a notice in an email.  It is already used
 	## by default with the built in :bro:enum:`Notice::ACTION_EMAIL` and
 	## :bro:enum:`Notice::ACTION_PAGE` actions.
@ -285,27 +279,22 @@ export {
 }

 # This is used as a hack to implement per-item expiration intervals.
-function per_notice_suppression_interval(t: table[Notice::Type, string] of Notice::Info, idx: any): interval
+function per_notice_suppression_interval(t: table[Notice::Type, string] of time, idx: any): interval
 	{
 	local n: Notice::Type;
 	local s: string;
 	[n,s] = idx;

-	local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
+	local suppress_time = t[n,s] - network_time();
 	if ( suppress_time < 0secs )
 		suppress_time = 0secs;

-	# If there is no more suppression time left, the notice needs to be sent
-	# to the end_suppression event.
-	if ( suppress_time == 0secs )
-		event Notice::end_suppression(t[n,s]);
-
 	return suppress_time;
 	}

 # This is the internally maintained notice suppression table.  It's
 # indexed on the Notice::Type and the $identifier field from the notice.
-global suppressing: table[Type, string] of Notice::Info = {}
+global suppressing: table[Type, string] of time = {}
 		&create_expire=0secs
 		&expire_func=per_notice_suppression_interval;

@ -400,11 +389,22 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)

 	# First off, finish the headers and include the human readable messages
 	# then leave a blank line after the message.
-	email_text = string_cat(email_text, "\nMessage: ", n$msg);
-	if ( n?$sub )
-		email_text = string_cat(email_text, "\nSub-message: ", n$sub);
+	email_text = string_cat(email_text, "\nMessage: ", n$msg, "\n");

-	email_text = string_cat(email_text, "\n\n");
+	if ( n?$sub )
+		email_text = string_cat(email_text, "Sub-message: ", n$sub, "\n");
+
+	email_text = string_cat(email_text, "\n");
+
+	# Add information about the file if it exists.
+	if ( n?$file_desc )
+		email_text = string_cat(email_text, "File Description: ", n$file_desc, "\n");
+
+	if ( n?$file_mime_type )
+		email_text = string_cat(email_text, "File MIME Type: ", n$file_mime_type, "\n");
+
+	if ( n?$file_desc || n?$file_mime_type )
+		email_text = string_cat(email_text, "\n");

 	# Next, add information about the connection if it exists.
 	if ( n?$id )
@ -467,7 +467,8 @@ hook Notice::notice(n: Notice::Info) &priority=-5
 	     [n$note, n$identifier] !in suppressing &&
 	     n$suppress_for != 0secs )
 		{
-		suppressing[n$note, n$identifier] = n;
+		local suppress_until = n$ts + n$suppress_for;
+		suppressing[n$note, n$identifier] = suppress_until;
 		event Notice::begin_suppression(n);
 		}
 	}
--- a/scripts/base/init-bare.bro
+++ b/scripts/base/init-bare.bro
@ -2432,6 +2432,17 @@ type X509: record {
 	not_valid_after: time;	##< Timestamp after when certificate is not valid.
 };

+## An X509 extension.
+##
+## .. bro:see:: x509_extension
+type X509_extension_info: record {
+	name: string;	##< Long name of extension; oid if name not known.
+	short_name: string &optional;	##< Short name of extension if known.
+	oid: string;	##< Oid of extension.
+	critical: bool;	##< True if extension is critical.
+	value: string;	##< Extension content parsed to string for known extensions. Raw data otherwise.
+};
+
 ## HTTP session statistics.
 ##
 ## .. bro:see:: http_stats
@ -2866,6 +2877,12 @@ global load_sample_freq = 20 &redef;
 ## .. bro:see:: gap_report
 const gap_report_freq = 1.0 sec &redef;

+## Whether to attempt to automatically detect SYN/FIN/RST-filtered trace
+## and not report missing segments for such connections.
+## If this is enabled, then missing data at the end of connections may not
+## be reported via :bro:see:`content_gap`.
+const detect_filtered_trace = F &redef;
+
 ## Whether we want :bro:see:`content_gap` and :bro:see:`gap_report` for partial
 ## connections. A connection is partial if it is missing a full handshake. Note
 ## that gap reports for partial connections might not be reliable.
@ -3074,6 +3091,9 @@ export {
 	## Toggle whether to do GTPv1 decapsulation.
 	const enable_gtpv1 = T &redef;

+	## Toggle whether to do GRE decapsulation.
+	const enable_gre = T &redef;
+
 	## With this option set, the Teredo analysis will first check to see if
 	## other protocol analyzers have confirmed that they think they're
 	## parsing the right protocol and only continue with Teredo tunnel
@ -3099,7 +3119,8 @@ export {
 	## may work better.
 	const delay_gtp_confirmation = F &redef;

-	## How often to cleanup internal state for inactive IP tunnels.
+	## How often to cleanup internal state for inactive IP tunnels
+	## (includes GRE tunnels).
 	const ip_tunnel_timeout = 24hrs &redef;
 } # end export
 module GLOBAL;
--- a/scripts/base/init-default.bro
+++ b/scripts/base/init-default.bro
@ -61,3 +61,4 @@


@load base/misc/find-checksum-offloading
+@load base/misc/find-filtered-trace
--- a/scripts/base/misc/find-filtered-trace.bro
+++ b/scripts/base/misc/find-filtered-trace.bro
@ -0,0 +1,49 @@
+##! Discovers trace files that contain TCP traffic consisting only of
+##! control packets (e.g. it's been filtered to contain only SYN/FIN/RST
+##! packets and no content).  On finding such a trace, a warning is
+##! emitted that suggests toggling the :bro:see:`detect_filtered_trace`
+##! option may be desired if the user does not want Bro to report
+##! missing TCP segments.
+
+module FilteredTraceDetection;
+
+export {
+
+	## Flag to enable filtered trace file detection and warning message.
+	global enable: bool = T &redef;
+}
+
+global saw_tcp_conn_with_data: bool = F;
+global saw_a_tcp_conn: bool = F;
+
+event connection_state_remove(c: connection)
+	{
+	if ( ! reading_traces() )
+		return;
+
+	if ( ! enable )
+		return;
+
+	if ( saw_tcp_conn_with_data )
+		return;
+
+	if ( ! is_tcp_port(c$id$orig_p) )
+		return;
+
+	saw_a_tcp_conn = T;
+
+	if ( /[Dd]/ in c$history )
+		saw_tcp_conn_with_data = T;
+	}
+
+event bro_done()
+	{
+	if ( ! enable )
+		return;
+
+	if ( ! saw_a_tcp_conn )
+		return;
+
+	if ( ! saw_tcp_conn_with_data )
+		Reporter::warning("The analyzed trace file was determined to contain only TCP control packets, which may indicate it's been pre-filtered.  By default, Bro reports the missing segments for this type of trace, but the 'detect_filtered_trace' option may be toggled if that's not desired.");
+	}
--- a/scripts/base/protocols/dns/main.bro
+++ b/scripts/base/protocols/dns/main.bro
@ -63,15 +63,17 @@ export {
 		## The DNS query was rejected by the server.
 		rejected:      bool               &log &default=F;

-		## This value indicates if this request/response pair is ready
-		## to be logged.
-		ready:         bool            &default=F;
 		## The total number of resource records in a reply message's
 		## answer section.
 		total_answers: count           &optional;
 		## The total number of resource records in a reply message's
 		## answer, authority, and additional sections.
 		total_replies: count           &optional;
+
+		## Whether the full DNS query has been seen.
+		saw_query: bool                &default=F;
+		## Whether the full DNS reply has been seen.
+		saw_reply: bool                &default=F;
 	};

 	## An event that can be handled to access the :bro:type:`DNS::Info`
@ -90,7 +92,7 @@ export {
 	## ans: The general information of a RR response.
 	##
 	## reply: The specific response information according to RR type/class.
-	global do_reply: event(c: connection, msg: dns_msg, ans: dns_answer, reply: string);
+	global do_reply: hook(c: connection, msg: dns_msg, ans: dns_answer, reply: string);

 	## A hook that is called whenever a session is being set.
 	## This can be used if additional initialization logic needs to happen
@ -103,17 +105,42 @@ export {
 	## is_query: Indicator for if this is being called for a query or a response.
 	global set_session: hook(c: connection, msg: dns_msg, is_query: bool);

+	## Yields a queue of :bro:see:`DNS::Info` objects for a given
+	## DNS message query/transaction ID.
+	type PendingMessages: table[count] of Queue::Queue;
+
+	## Called when a pending DNS query has not been matched with a reply (or
+	## vice versa) in a sufficent amount of time.
+	##
+	## pending: table of pending messages, indexed by transaction ID.
+	##
+	## id: the index of he element being expired.
+	##
+	## Returns: amount of time to delay expiration of the element.
+	global expire_pending_msg: function(pending: PendingMessages, id: count): interval;
+
+	## The amount of time that DNS queries or replies for a given
+	## query/transaction ID are allowed to be queued while waiting for
+	## a matching reply or query.
+	const pending_msg_expiry_interval = 2min &redef;
+
+	## Give up trying to match pending DNS queries or replies for a given
+	## query/transaction ID once this number of unmatched queries or replies
+	## is reached (this shouldn't happen unless either the DNS server/resolver
+	## is broken, Bro is not seeing all the DNS traffic, or an AXFR query
+	## response is ongoing).
+	const max_pending_msgs = 50 &redef;
+
 	## A record type which tracks the status of DNS queries for a given
 	## :bro:type:`connection`.
 	type State: record {
 		## Indexed by query id, returns Info record corresponding to
-		## query/response which haven't completed yet.
-		pending: table[count] of Queue::Queue;
+		## queries that haven't been matched with a response yet.
+		pending_queries: PendingMessages &read_expire=pending_msg_expiry_interval &expire_func=expire_pending_msg;

-		## This is the list of DNS responses that have completed based
-		## on the number of responses declared and the number received.
-		## The contents of the set are transaction IDs.
-		finished_answers: set[count];
+		## Indexed by query id, returns Info record corresponding to
+		## replies that haven't been matched with a query yet.
+		pending_replies: PendingMessages &read_expire=pending_msg_expiry_interval &expire_func=expire_pending_msg;
 	};
 }

@ -143,6 +170,51 @@ function new_session(c: connection, trans_id: count): Info
 	return info;
 	}

+function log_unmatched_msgs_queue(q: Queue::Queue)
+	{
+	local infos: vector of Info;
+	Queue::get_vector(q, infos);
+
+	for ( i in infos )
+		Log::write(DNS::LOG, infos[i]);
+	}
+
+function log_unmatched_msgs(msgs: PendingMessages)
+	{
+	for ( trans_id in msgs )
+		{
+		log_unmatched_msgs_queue(msgs[trans_id]);
+		delete msgs[trans_id];
+		}
+	}
+
+function enqueue_new_msg(msgs: PendingMessages, id: count, msg: Info)
+	{
+	if ( id !in msgs )
+		msgs[id] = Queue::init();
+	else if ( Queue::len(msgs[id]) > max_pending_msgs )
+		{
+		local info: Info = Queue::peek(msgs[id]);
+		event flow_weird("dns_unmatched_msg_quantity", info$id$orig_h,
+		                 info$id$resp_h);
+		log_unmatched_msgs_queue(msgs[id]);
+		# Throw away all unmatched on assumption they'll never be matched.
+		msgs[id] = Queue::init();
+		}
+
+	Queue::put(msgs[id], msg);
+	}
+
+function pop_msg(msgs: PendingMessages, id: count): Info
+	{
+	local rval: Info = Queue::get(msgs[id]);
+
+	if ( Queue::len(msgs[id]) == 0 )
+		delete msgs[id];
+
+	return rval;
+	}
+
 hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
 	{
 	if ( ! c?$dns_state )
@ -151,29 +223,39 @@ hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
 		c$dns_state = state;
 		}

-	if ( msg$id !in c$dns_state$pending )
-		c$dns_state$pending[msg$id] = Queue::init();
-	
-	local info: Info;
-	# If this is either a query or this is the reply but
-	# no Info records are in the queue (we missed the query?)
-	# we need to create an Info record and put it in the queue.  
-	if ( is_query ||
-	     Queue::len(c$dns_state$pending[msg$id]) == 0 )
-		{
-		info = new_session(c, msg$id);
-		Queue::put(c$dns_state$pending[msg$id], info);
-		}
-
 	if ( is_query )
-		# If this is a query, assign the newly created info variable
-		# so that the world looks correct to anything else handling
-		# this query.
-		c$dns = info;
+		{
+		if ( msg$id in c$dns_state$pending_replies &&
+		     Queue::len(c$dns_state$pending_replies[msg$id]) > 0 )
+			{
+			# Match this DNS query w/ what's at head of pending reply queue.
+			c$dns = pop_msg(c$dns_state$pending_replies, msg$id);
+			}
 		else
-		# Peek at the next item in the queue for this trans_id and 
-		# assign it to c$dns since this is a response.
-		c$dns = Queue::peek(c$dns_state$pending[msg$id]);
+			{
+			# Create a new DNS session and put it in the query queue so
+			# we can wait for a matching reply.
+			c$dns = new_session(c, msg$id);
+			enqueue_new_msg(c$dns_state$pending_queries, msg$id, c$dns);
+			}
+		}
+	else
+		{
+		if ( msg$id in c$dns_state$pending_queries &&
+		     Queue::len(c$dns_state$pending_queries[msg$id]) > 0 )
+			{
+			# Match this DNS reply w/ what's at head of pending query queue.
+			c$dns = pop_msg(c$dns_state$pending_queries, msg$id);
+			}
+		else
+			{
+			# Create a new DNS session and put it in the reply queue so
+			# we can wait for a matching query.
+			c$dns = new_session(c, msg$id);
+			event conn_weird("dns_unmatched_reply", c, "");
+			enqueue_new_msg(c$dns_state$pending_replies, msg$id, c$dns);
+			}
+		}

 	if ( ! is_query )
 		{
@ -183,36 +265,36 @@ hook set_session(c: connection, msg: dns_msg, is_query: bool) &priority=5
 		if ( ! c$dns?$total_answers )
 			c$dns$total_answers = msg$num_answers;

-		if ( c$dns?$total_replies &&
-		     c$dns$total_replies != msg$num_answers + msg$num_addl + msg$num_auth )
-			{
-			event conn_weird("dns_changed_number_of_responses", c,
-			                      fmt("The declared number of responses changed from %d to %d",
-			                          c$dns$total_replies,
-			                          msg$num_answers + msg$num_addl + msg$num_auth));
-			}
-		else
-			{
-			# Store the total number of responses expected from the first reply.
+		if ( ! c$dns?$total_replies )
 			c$dns$total_replies = msg$num_answers + msg$num_addl + msg$num_auth;
-			}
+
+		if ( msg$rcode != 0 && msg$num_queries == 0 )
+			c$dns$rejected = T;
 		}
 	}

 event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count) &priority=5
 	{
-	hook set_session(c, msg, is_orig);
+	if ( msg$opcode != 0 )
+		# Currently only standard queries are tracked.
+		return;
+
+	hook set_session(c, msg, ! msg$QR);
 	}

-event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
+hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
 	{
+	if ( msg$opcode != 0 )
+		# Currently only standard queries are tracked.
+		return;
+
+	if ( ! msg$QR )
+		# This is weird: the inquirer must also be providing answers in
+		# the request, which is not what we want to track.
+		return;
+
 	if ( ans$answer_type == DNS_ANS )
 		{
-		if ( ! c?$dns )
-			{
-			event conn_weird("dns_unmatched_reply", c, "");
-			hook set_session(c, msg, F);
-			}
 		c$dns$AA    = msg$AA;
 		c$dns$RA    = msg$RA;

@ -226,29 +308,35 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 				c$dns$TTLs = vector();
 			c$dns$TTLs[|c$dns$TTLs|] = ans$TTL;
 			}
-
-		if ( c$dns?$answers && c$dns?$total_answers &&
-		     |c$dns$answers| == c$dns$total_answers )
-			{
-			# Indicate this request/reply pair is ready to be logged.
-			c$dns$ready = T;
-			}
 		}
 	}

-event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=-5
+event dns_end(c: connection, msg: dns_msg) &priority=5
 	{
-	if ( c$dns$ready )
+	if ( ! c?$dns )
+		return;
+
+	if ( msg$QR )
+		c$dns$saw_reply = T;
+	else
+		c$dns$saw_query = T;
+	}
+
+event dns_end(c: connection, msg: dns_msg) &priority=-5
+	{
+	if ( c?$dns && c$dns$saw_reply && c$dns$saw_query )
 		{
 		Log::write(DNS::LOG, c$dns);
-		# This record is logged and no longer pending.
-		Queue::get(c$dns_state$pending[c$dns$trans_id]);
 		delete c$dns;
 		}
 	}

 event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
 	{
+	if ( msg$opcode != 0 )
+		# Currently only standard queries are tracked.
+		return;
+
 	c$dns$RD          = msg$RD;
 	c$dns$TC          = msg$TC;
 	c$dns$qclass      = qclass;
@ -265,60 +353,66 @@ event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qcla
 	c$dns$query = query;
 	}

+
+event dns_unknown_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
+	{
+	hook DNS::do_reply(c, msg, ans, fmt("<unknown type=%s>", ans$qtype));
+	}
+
 event dns_A_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, fmt("%s", a));
+	hook DNS::do_reply(c, msg, ans, fmt("%s", a));
 	}

 event dns_TXT_reply(c: connection, msg: dns_msg, ans: dns_answer, str: string) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, str);
+	hook DNS::do_reply(c, msg, ans, str);
 	}

 event dns_AAAA_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, fmt("%s", a));
+	hook DNS::do_reply(c, msg, ans, fmt("%s", a));
 	}

 event dns_A6_reply(c: connection, msg: dns_msg, ans: dns_answer, a: addr) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, fmt("%s", a));
+	hook DNS::do_reply(c, msg, ans, fmt("%s", a));
 	}

 event dns_NS_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, name);
+	hook DNS::do_reply(c, msg, ans, name);
 	}

 event dns_CNAME_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, name);
+	hook DNS::do_reply(c, msg, ans, name);
 	}

 event dns_MX_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string,
                   preference: count) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, name);
+	hook DNS::do_reply(c, msg, ans, name);
 	}

 event dns_PTR_reply(c: connection, msg: dns_msg, ans: dns_answer, name: string) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, name);
+	hook DNS::do_reply(c, msg, ans, name);
 	}

 event dns_SOA_reply(c: connection, msg: dns_msg, ans: dns_answer, soa: dns_soa) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, soa$mname);
+	hook DNS::do_reply(c, msg, ans, soa$mname);
 	}

 event dns_WKS_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, "");
+	hook DNS::do_reply(c, msg, ans, "");
 	}

 event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
 	{
-	event DNS::do_reply(c, msg, ans, "");
+	hook DNS::do_reply(c, msg, ans, "");
 	}

 # TODO: figure out how to handle these
@ -339,6 +433,7 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5

 event dns_rejected(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
 	{
+	if ( c?$dns )
 		c$dns$rejected = T;
 	}

@ -347,16 +442,23 @@ event connection_state_remove(c: connection) &priority=-5
 	if ( ! c?$dns_state )
 		return;

-	# If Bro is expiring state, we should go ahead and log all unlogged
-	# request/response pairs now.
-	for ( trans_id in c$dns_state$pending )
+	# If Bro is expiring state, we should go ahead and log all unmatched
+	# queries and replies now.
+	log_unmatched_msgs(c$dns_state$pending_queries);
+	log_unmatched_msgs(c$dns_state$pending_replies);
+	}
+
+function expire_pending_msg(pending: PendingMessages, id: count): interval
 	{
 	local infos: vector of Info;
-		Queue::get_vector(c$dns_state$pending[trans_id], infos);
+	Queue::get_vector(pending[id], infos);
+
 	for ( i in infos )
 		{
 		Log::write(DNS::LOG, infos[i]);
-			}
-		}
+		event flow_weird("dns_unmatched_msg", infos[i]$id$orig_h,
+		                 infos[i]$id$resp_h);
 		}

+	return 0sec;
+	}
--- a/scripts/base/protocols/http/dpd.sig
+++ b/scripts/base/protocols/http/dpd.sig
@ -1,6 +1,8 @@
+# List of HTTP headers pulled from:
+#   http://annevankesteren.nl/2007/10/http-methods
 signature dpd_http_client {
  ip-proto == tcp
-  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
+  payload /^[[:space:]]*(OPTIONS|GET|HEAD|POST|PUT|DELETE|TRACE|CONNECT|PROPFIND|PROPPATCH|MKCOL|COPY|MOVE|LOCK|UNLOCK|VERSION-CONTROL|REPORT|CHECKOUT|CHECKIN|UNCHECKOUT|MKWORKSPACE|UPDATE|LABEL|MERGE|BASELINE-CONTROL|MKACTIVITY|ORDERPATCH|ACL|PATCH|SEARCH|BCOPY|BDELETE|BMOVE|BPROPFIND|BPROPPATCH|NOTIFY|POLL|SUBSCRIBE|UNSUBSCRIBE|X-MS-ENUMATTS|RPC_OUT_DATA|RPC_IN_DATA)[[:space:]]*/
  tcp-state originator
 }

@ -11,3 +13,5 @@ signature dpd_http_server {
  requires-reverse-signature dpd_http_client
  enable "http"
 }
+
+
--- a/scripts/policy/protocols/dns/auth-addl.bro
+++ b/scripts/policy/protocols/dns/auth-addl.bro
@ -19,12 +19,16 @@ export {
 	};
 }

-event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=4
+hook DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
 	{
-	# The "ready" flag will be set here.  This causes the setting from the 
-	# base script to be overridden since the base script will log immediately 
-	# after all of the ANS replies have been seen.
-	c$dns$ready=F;
+	if ( msg$opcode != 0 )
+		# Currently only standard queries are tracked.
+		return;
+
+	if ( ! msg$QR )
+		# This is weird: the inquirer must also be providing answers in
+		# the request, which is not what we want to track.
+		return;

 	if ( ans$answer_type == DNS_AUTH )
 		{
@ -38,11 +42,4 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 			c$dns$addl = set();
 		add c$dns$addl[reply];
 		}
-	
-	if ( c$dns?$answers && c$dns?$auth && c$dns?$addl &&
-	     c$dns$total_replies == |c$dns$answers| + |c$dns$auth| + |c$dns$addl| )
-		{
-		# *Now* all replies desired have been seen.
-		c$dns$ready = T;
-		}
 	}
--- a/scripts/policy/protocols/ssl/validate-certs.bro
+++ b/scripts/policy/protocols/ssl/validate-certs.bro
@ -40,6 +40,7 @@ event ssl_established(c: connection) &priority=3
 		{
 		local result = x509_verify(c$ssl$cert, c$ssl$cert_chain, root_certs);
 		c$ssl$validation_status = x509_err2str(result);
+		recently_validated_certs[c$ssl$cert_hash] = c$ssl$validation_status;
 		}
 		
 	if ( c$ssl$validation_status != "ok" )
--- a/src/3rdparty
+++ b/src/3rdparty
@ -1 +1 @@
-Subproject commit 12b5cb446c8128bb22e5cbd7baa7d53669539487
+Subproject commit 42a4c9694a2b2677b050fbb7cbae26bc5ec4605a
--- a/src/Attr.cc
+++ b/src/Attr.cc
@ -317,8 +317,9 @@ void Attributes::CheckAttr(Attr* a)
 				break;

 			// Table defaults may be promotable.
-			if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
-			      record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) )
+			if ( ytype && ytype->Tag() == TYPE_RECORD &&
+			     atype->Tag() == TYPE_RECORD &&
+			     record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType()) )
 				// Ok.
 				break;

--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@ -388,6 +388,9 @@ install(TARGETS bro DESTINATION bin)
 set(BRO_EXE bro
    CACHE STRING "Bro executable binary" FORCE)

+# External libmagic project must be built before bro.
+add_dependencies(bro libmagic)
+
 # Target to create all the autogenerated files.
 add_custom_target(generate_outputs_stage1)
 add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS})
--- a/src/NetVar.cc
+++ b/src/NetVar.cc
@ -48,6 +48,7 @@ int tcp_max_above_hole_without_any_acks;
 int tcp_excessive_data_without_further_acks;

 RecordType* x509_type;
+RecordType* x509_extension_type;

 RecordType* socks_address;

@ -356,6 +357,7 @@ void init_net_var()
 		opt_internal_int("tcp_excessive_data_without_further_acks");

 	x509_type = internal_type("X509")->AsRecordType();
+	x509_extension_type = internal_type("X509_extension_info")->AsRecordType();

 	socks_address = internal_type("SOCKS::Address")->AsRecordType();

--- a/src/NetVar.h
+++ b/src/NetVar.h
@ -51,6 +51,7 @@ extern int tcp_max_above_hole_without_any_acks;
 extern int tcp_excessive_data_without_further_acks;

 extern RecordType* x509_type;
+extern RecordType* x509_extension_type;

 extern RecordType* socks_address;

--- a/src/PktSrc.cc
+++ b/src/PktSrc.cc
@ -661,7 +661,7 @@ PktDumper::PktDumper(const char* arg_filename, bool arg_append)
 	if ( linktype < 0 )
 		linktype = DLT_EN10MB;

-	pd = pcap_open_dead(linktype, 8192);
+	pd = pcap_open_dead(linktype, snaplen);
 	if ( ! pd )
 		{
 		Error("error for pcap_open_dead");
--- a/src/Sessions.cc
+++ b/src/Sessions.cc
@ -376,6 +376,31 @@ int NetSessions::CheckConnectionTag(Connection* conn)
 	return 1;
 	}

+static unsigned int gre_header_len(uint16 flags)
+	{
+	unsigned int len = 4;  // Always has 2 byte flags and 2 byte protocol type.
+
+	if ( flags & 0x8000 )
+		// Checksum/Reserved1 present.
+		len += 4;
+
+	// Not considering routing presence bit since it's deprecated ...
+
+	if ( flags & 0x2000 )
+		// Key present.
+		len += 4;
+
+	if ( flags & 0x1000 )
+		// Sequence present.
+		len += 4;
+
+	if ( flags & 0x0080 )
+		// Acknowledgement present.
+		len += 4;
+
+	return len;
+	}
+
 void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 				const IP_Hdr* ip_hdr, const u_char* const pkt,
 				int hdr_size, const EncapsulationStack* encapsulation)
@ -384,6 +409,15 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	const struct ip* ip4 = ip_hdr->IP4_Hdr();

 	uint32 len = ip_hdr->TotalLen();
+	if ( len == 0 )
+		{
+		// TCP segmentation offloading can zero out the ip_len field.
+		Weird("ip_hdr_len_zero", hdr, pkt, encapsulation);
+
+		// Cope with the zero'd out ip_len field by using the caplen.
+		len = hdr->caplen - hdr_size;
+		}
+
 	if ( hdr->len < len + hdr_size )
 		{
 		Weird("truncated_IP", hdr, pkt, encapsulation);
@ -437,6 +471,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 		}

+	FragReassemblerTracker frt(this, f);
+
 	len -= ip_hdr_len;	// remove IP header
 	caplen -= ip_hdr_len;

@ -451,7 +487,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			vl->append(ip_hdr->BuildPktHdrVal());
 			mgr.QueueEvent(esp_packet, vl);
 			}
-		Remove(f);
+
 		// Can't do more since upper-layer payloads are going to be encrypted.
 		return;
 		}
@ -466,7 +502,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( ! ignore_checksums && mobility_header_checksum(ip_hdr) != 0xffff )
 			{
 			Weird("bad_MH_checksum", hdr, pkt, encapsulation);
-			Remove(f);
 			return;
 			}

@ -480,7 +515,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( ip_hdr->NextProto() != IPPROTO_NONE )
 			Weird("mobility_piggyback", hdr, pkt, encapsulation);

-		Remove(f);
 		return;
 		}
 #endif
@ -488,10 +522,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	int proto = ip_hdr->NextProto();

 	if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt, encapsulation) )
-		{
-		Remove(f);
 		return;
-		}

 	const u_char* data = ip_hdr->Payload();

@ -553,13 +584,100 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		break;
 		}

+	case IPPROTO_GRE:
+		{
+		if ( ! BifConst::Tunnel::enable_gre )
+			{
+			Weird("GRE_tunnel", ip_hdr, encapsulation);
+			return;
+			}
+
+		uint16 flags_ver = ntohs(*((uint16*)(data + 0)));
+		uint16 proto_typ = ntohs(*((uint16*)(data + 2)));
+		int gre_version = flags_ver & 0x0007;
+
+		if ( gre_version != 0 && gre_version != 1 )
+			{
+			Weird(fmt("unknown_gre_version_%d", gre_version), ip_hdr,
+			      encapsulation);
+			return;
+			}
+
+		if ( gre_version == 0 )
+			{
+			if ( proto_typ != 0x0800 && proto_typ != 0x86dd )
+				{
+				// Not IPv4/IPv6 payload.
+				Weird(fmt("unknown_gre_protocol_%"PRIu16, proto_typ), ip_hdr,
+				      encapsulation);
+				return;
+				}
+
+			proto = (proto_typ == 0x0800) ? IPPROTO_IPV4 : IPPROTO_IPV6;
+			}
+
+		else // gre_version == 1
+			{
+			if ( proto_typ != 0x880b )
+				{
+				// Enhanced GRE payload must be PPP.
+				Weird("egre_protocol_type", ip_hdr, encapsulation);
+				return;
+				}
+			}
+
+		if ( flags_ver & 0x4000 )
+			{
+			// RFC 2784 deprecates the variable length routing field
+			// specified by RFC 1701. It could be parsed here, but easiest
+			// to just skip for now.
+			Weird("gre_routing", ip_hdr, encapsulation);
+			return;
+			}
+
+		if ( flags_ver & 0x0078 )
+			{
+			// Expect last 4 bits of flags are reserved, undefined.
+			Weird("unknown_gre_flags", ip_hdr, encapsulation);
+			return;
+			}
+
+		unsigned int gre_len = gre_header_len(flags_ver);
+		unsigned int ppp_len = gre_version == 1 ? 1 : 0;
+
+		if ( len < gre_len + ppp_len || caplen < gre_len + ppp_len )
+			{
+			Weird("truncated_GRE", ip_hdr, encapsulation);
+			return;
+			}
+
+		if ( gre_version == 1 )
+			{
+			int ppp_proto = *((uint8*)(data + gre_len));
+
+			if ( ppp_proto != 0x0021 && ppp_proto != 0x0057 )
+				{
+				Weird("non_ip_packet_in_egre", ip_hdr, encapsulation);
+				return;
+				}
+
+			proto = (ppp_proto == 0x0021) ? IPPROTO_IPV4 : IPPROTO_IPV6;
+			}
+
+		data += gre_len + ppp_len;
+		len -= gre_len + ppp_len;
+		caplen -= gre_len + ppp_len;
+
+		// Treat GRE tunnel like IP tunnels, fallthrough to logic below now
+		// that GRE header is stripped and only payload packet remains.
+		}
+
 	case IPPROTO_IPV4:
 	case IPPROTO_IPV6:
 		{
 		if ( ! BifConst::Tunnel::enable_ip )
 			{
 			Weird("IP_tunnel", ip_hdr, encapsulation);
-			Remove(f);
 			return;
 			}

@ -567,7 +685,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		     encapsulation->Depth() >= BifConst::Tunnel::max_depth )
 			{
 			Weird("exceeded_tunnel_max_depth", ip_hdr, encapsulation);
-			Remove(f);
 			return;
 			}

@ -584,7 +701,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( result != 0 )
 			{
 			delete inner;
-			Remove(f);
 			return;
 			}

@ -611,7 +727,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		DoNextInnerPacket(t, hdr, inner, encapsulation,
 		                  ip_tunnels[tunnel_idx].first);

-		Remove(f);
 		return;
 		}

@ -624,13 +739,11 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		     encapsulation->LastType() == BifEnum::Tunnel::TEREDO ) )
 			Weird("ipv6_no_next", hdr, pkt);

-		Remove(f);
 		return;
 		}

 	default:
 		Weird(fmt("unknown_protocol_%d", proto), hdr, pkt, encapsulation);
-		Remove(f);
 		return;
 	}

@ -656,7 +769,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( consistent < 0 )
 			{
 			delete h;
-			Remove(f);
 			return;
 			}

@ -680,7 +792,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	if ( ! conn )
 		{
 		delete h;
-		Remove(f);
 		return;
 		}

@ -712,7 +823,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		{
 		// Above we already recorded the fragment in its entirety.
 		f->DeleteTimer();
-		Remove(f);
 		}

 	else if ( record_packet )
@ -813,6 +923,9 @@ bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
 	case IPPROTO_NONE:
 		min_hdr_len = 0;
 		break;
+	case IPPROTO_GRE:
+		min_hdr_len = 4;
+		break;
 	case IPPROTO_ICMP:
 	case IPPROTO_ICMPV6:
 	default:
--- a/src/Sessions.h
+++ b/src/Sessions.h
@ -286,6 +286,21 @@ protected:
 	NetSessions::IPPair tunnel_idx;
 };

+
+class FragReassemblerTracker {
+public:
+	FragReassemblerTracker(NetSessions* s, FragReassembler* f)
+		: net_sessions(s), frag_reassembler(f)
+		{ }
+
+	~FragReassemblerTracker()
+		{ net_sessions->Remove(frag_reassembler); }
+
+private:
+	NetSessions* net_sessions;
+	FragReassembler* frag_reassembler;
+};
+
 // Manager for the currently active sessions.
 extern NetSessions* sessions;

--- a/src/analyzer/protocol/dnp3/events.bif
+++ b/src/analyzer/protocol/dnp3/events.bif
@ -32,6 +32,8 @@ event dnp3_application_response_header%(c: connection, is_orig: bool, fc: count,
 ##
 ## qua_field: qualifier field.
 ##
+## number: TODO.
+##
 ## rf_low: the structure of the range field depends on the qualified field.
 ##         In some cases, the range field contains only one logic part, e.g.,
 ##         number of objects, so only *rf_low* contains useful values.
--- a/src/analyzer/protocol/dns/DNS.cc
+++ b/src/analyzer/protocol/dns/DNS.cc
@ -137,18 +137,6 @@ int DNS_Interpreter::ParseQuestions(DNS_MsgInfo* msg,
 	{
 	int n = msg->qdcount;

-	if ( n == 0 )
-		{
-		// Generate event here because we won't go into ParseQuestion.
-		EventHandlerPtr dns_event =
-			msg->rcode == DNS_CODE_OK ?
-				dns_query_reply : dns_rejected;
-		BroString* question_name = new BroString("<no query>");
-
-		SendReplyOrRejectEvent(msg, dns_event, data, len, question_name);
-		return 1;
-		}
-
 	while ( n > 0 && ParseQuestion(msg, data, len, msg_start) )
 		--n;
 	return n == 0;
@ -299,6 +287,16 @@ int DNS_Interpreter::ParseAnswer(DNS_MsgInfo* msg,
 			break;

 		default:
+
+			if ( dns_unknown_reply && ! msg->skip_event )
+				{
+				val_list* vl = new val_list;
+				vl->append(analyzer->BuildConnVal());
+				vl->append(msg->BuildHdrVal());
+				vl->append(msg->BuildAnswerVal());
+				analyzer->ConnectionEvent(dns_unknown_reply, vl);
+				}
+
 			analyzer->Weird("DNS_RR_unknown_type");
 			data += rdlength;
 			len -= rdlength;
--- a/src/analyzer/protocol/dns/events.bif
+++ b/src/analyzer/protocol/dns/events.bif
@ -50,7 +50,7 @@ event dns_message%(c: connection, is_orig: bool, msg: dns_msg, len: count%);
 event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%);

 ## Generated for DNS replies that reject a query. This event is raised if a DNS
-## reply either indicates failure via its status code or does not pass on any
+## reply indicates failure because it does not pass on any
 ## answers to a query. Note that all of the event's parameters are parsed out of
 ## the reply; there's no stateful correlation with the query.
 ##
@ -78,7 +78,7 @@ event dns_request%(c: connection, msg: dns_msg, query: string, qtype: count, qcl
 ##    dns_skip_all_addl dns_skip_all_auth dns_skip_auth
 event dns_rejected%(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count%);

-## Generated for DNS replies with an *ok* status code but no question section.
+## Generated for each entry in the Question section of a DNS reply.
 ##
 ## See `Wikipedia <http://en.wikipedia.org/wiki/Domain_Name_System>`__ for more
 ## information about the DNS protocol. Bro analyzes both UDP and TCP DNS
@ -401,6 +401,22 @@ event dns_TXT_reply%(c: connection, msg: dns_msg, ans: dns_answer, str: string%)
 ##    dns_skip_addl dns_skip_all_addl dns_skip_all_auth dns_skip_auth
 event dns_SRV_reply%(c: connection, msg: dns_msg, ans: dns_answer%);

+## Generated on DNS reply resource records when the type of record is not one
+## that Bro knows how to parse and generate another more specific specific
+## event.
+##
+## c: The connection, which may be UDP or TCP depending on the type of the
+##    transport-layer session being analyzed.
+##
+## msg: The parsed DNS message header.
+##
+## ans: The type-independent part of the parsed answer record.
+##
+## .. bro:see:: dns_AAAA_reply dns_A_reply dns_CNAME_reply dns_EDNS_addl
+##    dns_HINFO_reply dns_MX_reply dns_NS_reply dns_PTR_reply dns_SOA_reply
+##    dns_TSIG_addl dns_TXT_reply dns_WKS_reply dns_SRV_reply dns_end
+event dns_unknown_reply%(c: connection, msg: dns_msg, ans: dns_answer%);
+
 ## Generated for DNS replies of type *EDNS*. For replies with multiple answers,
 ## an individual event of the corresponding type is raised for each.
 ##
--- a/src/analyzer/protocol/http/HTTP.cc
+++ b/src/analyzer/protocol/http/HTTP.cc
@ -242,10 +242,17 @@ int HTTP_Entity::Undelivered(int64_t len)
 	if ( end_of_data && in_header )
 		return 0;

+	if ( is_partial_content )
 		file_mgr->Gap(body_length, len,
 		              http_message->MyHTTP_Analyzer()->GetAnalyzerTag(),
 		              http_message->MyHTTP_Analyzer()->Conn(),
 		              http_message->IsOrig());
+	else
+		precomputed_file_id = file_mgr->Gap(body_length, len,
+		                  http_message->MyHTTP_Analyzer()->GetAnalyzerTag(),
+		                  http_message->MyHTTP_Analyzer()->Conn(),
+		                  http_message->IsOrig(),
+		                  precomputed_file_id);

 	if ( chunked_transfer_state != NON_CHUNKED_TRANSFER )
 		{
@ -314,15 +321,18 @@ void HTTP_Entity::SubmitData(int len, const char* buf)
 	else
 		{
 		if ( send_size && content_length > 0 )
-			file_mgr->SetSize(content_length,
+			precomputed_file_id = file_mgr->SetSize(content_length,
 			                  http_message->MyHTTP_Analyzer()->GetAnalyzerTag(),
 			                  http_message->MyHTTP_Analyzer()->Conn(),
-			                  http_message->IsOrig());
+			                  http_message->IsOrig(),
+			                  precomputed_file_id);

-		file_mgr->DataIn(reinterpret_cast<const u_char*>(buf), len,
+		precomputed_file_id = file_mgr->DataIn(reinterpret_cast<const u_char*>(buf),
+		                 len,
 		                 http_message->MyHTTP_Analyzer()->GetAnalyzerTag(),
 		                 http_message->MyHTTP_Analyzer()->Conn(),
-		                 http_message->IsOrig());
+		                 http_message->IsOrig(),
+		                 precomputed_file_id);
 		}

 	send_size = false;
--- a/src/analyzer/protocol/http/HTTP.h
+++ b/src/analyzer/protocol/http/HTTP.h
@ -64,6 +64,7 @@ protected:
 	uint64_t offset;
 	int64_t instance_length; // total length indicated by content-range
 	bool send_size; // whether to send size indication to FAF
+	std::string precomputed_file_id;

 	MIME_Entity* NewChildEntity() { return new HTTP_Entity(http_message, this, 1); }

--- a/src/analyzer/protocol/modbus/events.bif
+++ b/src/analyzer/protocol/modbus/events.bif
@ -149,7 +149,7 @@ event modbus_write_single_register_response%(c: connection, headers: ModbusHeade
 ##
 ## start_address: The memory address of the first coil to be written.
 ##
-## value: The values to be written to the coils.
+## coils: The values to be written to the coils.
 event modbus_write_multiple_coils_request%(c: connection, headers: ModbusHeaders, start_address: count, coils: ModbusCoils%);

 ## Generated for a Modbus write multiple coils response.
--- a/src/analyzer/protocol/pop3/POP3.cc
+++ b/src/analyzer/protocol/pop3/POP3.cc
@ -192,14 +192,13 @@ void POP3_Analyzer::ProcessRequest(int length, const char* line)

 		case AUTH_CRAM_MD5:
 			{ // Format: "user<space>password-hash"
-			char* s;
-			char* str = (char*) decoded->CheckString();
+			const char* s;
+			const char* str = (char*) decoded->CheckString();

 			for ( s = str; *s && *s != '\t' && *s != ' '; ++s )
 				;
-			*s = '\0';

-			user = str;
+			user = std::string(str, s);
 			password = "";

 			break;
--- a/src/analyzer/protocol/ssl/events.bif
+++ b/src/analyzer/protocol/ssl/events.bif
@ -178,11 +178,13 @@ event x509_certificate%(c: connection, is_orig: bool, cert: X509, chain_idx: cou
 ##
 ## is_orig: True if event is raised for originator side of the connection.
 ##
-## data: The raw data associated with the extension.
+## cert: The parsed certificate.
+##
+## extension: The parsed extension.
 ##
 ## .. bro:see:: ssl_alert ssl_client_hello ssl_established ssl_extension
 ##    ssl_server_hello x509_certificate x509_error  x509_verify
-event x509_extension%(c: connection, is_orig: bool, data: string%);
+event x509_extension%(c: connection, is_orig: bool, cert: X509, extension: X509_extension_info%);

 ## Generated when errors occur during parsing an X509 certificate.
 ##
--- a/src/analyzer/protocol/ssl/ssl-analyzer.pac
+++ b/src/analyzer/protocol/ssl/ssl-analyzer.pac
@ -9,6 +9,7 @@
 #include "util.h"

 #include <openssl/x509.h>
+#include <openssl/x509v3.h>
 #include <openssl/asn1.h>
 %}

@ -298,27 +299,61 @@ refine connection SSL_Conn += {
 					int num_ext = X509_get_ext_count(pTemp);
 					for ( int k = 0; k < num_ext; ++k )
 						{
-						unsigned char *pBuffer = 0;
-						int length = 0;
+						char name[256];
+						char oid[256];
+
+						memset(name, 0, sizeof(name));
+						memset(oid, 0, sizeof(oid));

 						X509_EXTENSION* ex = X509_get_ext(pTemp, k);
-						if (ex)
-							{
-							ASN1_STRING *pString = X509_EXTENSION_get_data(ex);
-							length = ASN1_STRING_to_UTF8(&pBuffer, pString);
-							//i2t_ASN1_OBJECT(&pBuffer, length, obj)
-							// printf("extension length: %d\n", length);
-							// -1 indicates an error.
-							if ( length >= 0 )
-								{
-								StringVal* value = new StringVal(length, (char*)pBuffer);
+
+						if ( ! ex )
+							continue;
+
+						ASN1_OBJECT* ext_asn = X509_EXTENSION_get_object(ex);
+						const char* short_name = OBJ_nid2sn(OBJ_obj2nid(ext_asn));
+
+						OBJ_obj2txt(name, sizeof(name) - 1, ext_asn, 0);
+						OBJ_obj2txt(oid, sizeof(oid) - 1, ext_asn, 1);
+
+						int critical = 0;
+						if ( X509_EXTENSION_get_critical(ex) != 0 )
+							critical = 1;
+
+						BIO *bio = BIO_new(BIO_s_mem());
+						if( ! X509V3_EXT_print(bio, ex, 0, 0))
+							M_ASN1_OCTET_STRING_print(bio, ex->value);
+
+						BIO_flush(bio);
+						int length = BIO_pending(bio);
+
+						// Use OPENSSL_malloc here. Using new or anything else can lead
+						// to interesting, hard to debug segfaults.
+						char *buffer = (char*) OPENSSL_malloc(length);
+						BIO_read(bio, buffer, length);
+						StringVal* ext_val = new StringVal(length, buffer);
+						OPENSSL_free(buffer);
+
+						BIO_free_all(bio);
+
+						RecordVal* pX509Ext = new RecordVal(x509_extension_type);
+						pX509Ext->Assign(0, new StringVal(name));
+
+						if ( short_name && strlen(short_name) > 0 )
+							pX509Ext->Assign(1, new StringVal(short_name));
+
+						pX509Ext->Assign(2, new StringVal(oid));
+						pX509Ext->Assign(3, new Val(critical, TYPE_BOOL));
+						pX509Ext->Assign(4, ext_val);
+
 						BifEvent::generate_x509_extension(bro_analyzer(),
-											bro_analyzer()->Conn(), ${rec.is_orig}, value);
-								}
-							OPENSSL_free(pBuffer);
-							}
+									bro_analyzer()->Conn(),
+									${rec.is_orig},
+									pX509Cert->Ref(),
+									pX509Ext);
 						}
 					}
+
 				X509_free(pTemp);
 				}
 			}
--- a/src/analyzer/protocol/tcp/TCP.cc
+++ b/src/analyzer/protocol/tcp/TCP.cc
@ -373,14 +373,11 @@ void TCP_Analyzer::ProcessSYN(const IP_Hdr* ip, const struct tcphdr* tp,
 void TCP_Analyzer::ProcessFIN(double t, TCP_Endpoint* endpoint,
 				int& seq_len, uint32 base_seq)
 	{
-	if ( endpoint->FIN_cnt == 0 )
-		{
-		++seq_len;	// FIN consumes a byte of sequence space
+	++seq_len;  // FIN consumes a byte of sequence space.
 	++endpoint->FIN_cnt;  // remember that we've seen a FIN
-		}

-	else if ( t < endpoint->last_time + tcp_storm_interarrival_thresh &&
-		  ++endpoint->FIN_cnt == tcp_storm_thresh )
+	if ( t < endpoint->last_time + tcp_storm_interarrival_thresh &&
+	     endpoint->FIN_cnt == tcp_storm_thresh )
 		Weird("FIN_storm");

 	// Remember the relative seq in FIN_seq.
--- a/src/analyzer/protocol/tcp/TCP_Endpoint.cc
+++ b/src/analyzer/protocol/tcp/TCP_Endpoint.cc
@ -161,6 +161,13 @@ void TCP_Endpoint::SetState(EndpointState new_state)

 bro_int_t TCP_Endpoint::Size() const
 	{
+	if ( prev_state == TCP_ENDPOINT_SYN_SENT && state == TCP_ENDPOINT_RESET &&
+	     peer->state == TCP_ENDPOINT_INACTIVE && ! NoDataAcked() )
+		// This looks like a half-open connection was discovered and aborted.
+		// Sequence numbers could be misleading if used in context of data size
+		// and there was never a chance for this endpoint to send data anyway.
+		return 0;
+
 	bro_int_t size;

 	uint64 last_seq_64 = (uint64(last_seq_high) << 32) | last_seq;
--- a/src/analyzer/protocol/tcp/TCP_Reassembler.cc
+++ b/src/analyzer/protocol/tcp/TCP_Reassembler.cc
@ -178,7 +178,7 @@ void TCP_Reassembler::Undelivered(int up_to_seq)
 		// to this method and only if this condition is not true).
 		reporter->InternalError("Calling Undelivered for data that has already been delivered (or has already been marked as undelivered");

-	if ( last_reassem_seq == 1 &&
+	if ( BifConst::detect_filtered_trace && last_reassem_seq == 1 &&
 	     (endpoint->FIN_cnt > 0 || endpoint->RST_cnt > 0 ||
 	      peer->FIN_cnt > 0 || peer->RST_cnt > 0) )
 		{
--- a/src/analyzer/protocol/tcp/events.bif
+++ b/src/analyzer/protocol/tcp/events.bif
@ -108,6 +108,8 @@ event connection_half_finished%(c: connection%);
 ## originator attempted to setup a TCP connection but the responder replied
 ## with a RST packet denying it.
 ##
+## c: The connection.
+##
 ## .. bro:see:: connection_EOF connection_SYN_packet connection_attempt
 ##    connection_established connection_external connection_finished
 ##    connection_first_ACK connection_half_finished connection_partial_close
@ -115,8 +117,6 @@ event connection_half_finished%(c: connection%);
 ##    connection_status_update connection_timeout scheduled_analyzer_applied
 ##    new_connection new_connection_contents partial_connection
 ##
-## c: The connection.
-##
 ## .. note::
 ##
 ##    If the responder does not respond at all, :bro:id:`connection_attempt` is
--- a/src/bro.bif
+++ b/src/bro.bif
@ -973,6 +973,8 @@ function entropy_test_finish%(handle: opaque of entropy%): entropy_test_result
 ##
 ## prefix: A custom string prepended to the result.
 ##
+## Returns: A string identifier that is unique.
+##
 ## .. bro:see:: unique_id_from
 function unique_id%(prefix: string%) : string
 	%{
@ -987,6 +989,8 @@ function unique_id%(prefix: string%) : string
 ##
 ## prefix: A custom string prepended to the result.
 ##
+## Returns: A string identifier that is unique.
+##
 ## .. bro:see:: unique_id
 function unique_id_from%(pool: int, prefix: string%) : string
 	%{
@ -1857,8 +1861,8 @@ function global_ids%(%): id_table
 ##
 ## id: The global identifier.
 ##
-## Returns the value of *id*. If *id* does not describe a valid identifier, the
-## function returns the string ``"<unknown id>"`` or ``"<no ID value>"``.
+## Returns: The value of *id*. If *id* does not describe a valid identifier,
+##          the string ``"<unknown id>"`` or ``"<no ID value>"`` is returned.
 function lookup_ID%(id: string%) : any
 	%{
 	ID* i = global_scope()->Lookup(id->CheckString());
@ -2635,12 +2639,12 @@ function bytestring_to_hexstr%(bytestring: string%): string
 ## Converts a hex-string into its binary representation.
 ## For example, ``"3034"`` would be converted to ``"04"``.
 ##
-## The input string is assumed to only contain 0-9 and a-f,
-## otherwhise behavior is undefines.
+## The input string is assumed to contain an even number of hexadecimal digits
+## (0-9, a-f, or A-F), otherwise behavior is undefined.
 ##
-## hexstring: The hexadecimal string representation.
+## hexstr: The hexadecimal string representation.
 ##
-## Returns: The binary representation of *hexstring*.
+## Returns: The binary representation of *hexstr*.
 ##
 ## .. bro:see:: hexdump bytestring_to_hexstr
 function hexstr_to_bytestring%(hexstr: string%): string
@ -3439,9 +3443,59 @@ static GeoIP* open_geoip_db(GeoIPDBTypes type)
 	if ( GeoIP_db_avail(type) )
 		geoip = GeoIP_open_type(type, GEOIP_MEMORY_CACHE);

+	return geoip;
+	}
+
+static GeoIP* open_geoip_city_db()
+	{
+	GeoIP* geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0);
+
 	if ( ! geoip )
-		reporter->Info("Failed to open GeoIP database: %s",
-		                  GeoIPDBFileName[type]);
+		geoip = open_geoip_db(GEOIP_CITY_EDITION_REV1);
+
+	if ( ! geoip )
+		{
+		string rev0_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV0];
+		string rev1_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV1];
+		string db_path = rev0_path;
+
+		// Maybe in the future the revisions won't share a common default path.
+		if ( rev0_path != rev1_path )
+			db_path = rev0_path + " or " + rev1_path;
+
+		reporter->Info("Failed to open GeoIP City database: %s",
+		               db_path.c_str());
+		}
+
+	return geoip;
+	}
+
+static GeoIP* open_geoip_city_db_v6()
+	{
+	GeoIP* geoip = 0;
+
+	// Both city edition revisions for IPv6 show up in libGeoIP 1.4.7.
+#ifdef HAVE_GEOIP_CITY_EDITION_REV0_V6
+	geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0_V6);
+
+	if ( ! geoip )
+		geoip = open_geoip_db(GEOIP_CITY_EDITION_REV1_V6);
+
+	if ( ! geoip )
+		{
+		string rev0_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV0_V6];
+		string rev1_path = GeoIPDBFileName[GEOIP_CITY_EDITION_REV1_V6];
+		string db_path = rev0_path;
+
+		// Maybe in the future the revisions won't share a common default path.
+		if ( rev0_path != rev1_path )
+			db_path = rev0_path + " or " + rev1_path;
+
+		reporter->Info("Failed to open GeoIP Cityv6 database: %s",
+		               db_path.c_str());
+		}
+#endif
+
 	return geoip;
 	}

@ -3472,31 +3526,41 @@ function lookup_location%(a: addr%) : geo_location
 	if ( ! geoip_initialized )
 		{
 		geoip_initialized = true;
-		geoip = open_geoip_db(GEOIP_CITY_EDITION_REV0);
+		geoip = open_geoip_city_db();

 		if ( ! geoip )
 			{
 			geoip = open_geoip_db(GEOIP_COUNTRY_EDITION);
+			string db_path = GeoIPDBFileName[GEOIP_COUNTRY_EDITION];
+
 			if ( ! geoip )
-				builtin_error("Can't initialize GeoIP City/Country database");
+				builtin_error(fmt("Failed fall back to GeoIP Country "
+				                  "database: %s",
+				                  GeoIPDBFileName[GEOIP_COUNTRY_EDITION]));
 			else
 				reporter->Info("Fell back to GeoIP Country database");
 			}
 		else
 			have_city_db = true;

-#ifdef HAVE_GEOIP_CITY_EDITION_REV0_V6
-		geoip_v6 = open_geoip_db(GEOIP_CITY_EDITION_REV0_V6);
+		geoip_v6 = open_geoip_city_db_v6();
+
 		if ( geoip_v6 )
 			have_cityv6_db = true;
-#endif

 #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
 		if ( ! geoip_v6 )
+			{
 			geoip_v6 = open_geoip_db(GEOIP_COUNTRY_EDITION_V6);
-#endif
+
 			if ( ! geoip_v6 )
-			builtin_error("Can't initialize GeoIPv6 City/Country database");
+				reporter->Info("Failed to open GeoIPv6 Country database: %s",
+				               GeoIPDBFileName[GEOIP_COUNTRY_EDITION_V6]);
+			}
+#endif
+
+		if ( ! geoip_v6 )
+			builtin_error("Can't open GeoIPv6 City/Country database");
 		}

 #ifdef HAVE_GEOIP_COUNTRY_EDITION_V6
@ -3588,8 +3652,10 @@ function lookup_asn%(a: addr%) : count
 		{
 		geoip_asn_initialized = true;
 		geoip_asn = open_geoip_db(GEOIP_ASNUM_EDITION);
+
 		if ( ! geoip_asn )
-			builtin_error("Can't initialize GeoIP ASNUM database");
+			builtin_error(fmt("Can't open GeoIP ASNUM database: %s",
+			                  GeoIPDBFileName[GEOIP_ASNUM_EDITION]));
 		}

 	if ( geoip_asn )
@ -4964,8 +5030,6 @@ function preserve_prefix%(a: addr, width: count%): any
 ##
 ## a: The subnet to preserve.
 ##
-## width: The number of bits from the top that should remain intact.
-##
 ## .. bro:see:: preserve_prefix anonymize_addr
 ##
 ## .. todo:: Currently dysfunctional.
--- a/src/broxygen/Manager.cc
+++ b/src/broxygen/Manager.cc
@ -35,8 +35,12 @@ Manager::Manager(const string& arg_config, const string& bro_command)
 	if ( getenv("BRO_DISABLE_BROXYGEN") )
 		disabled = true;

-	const char* path = getenv("PATH");
-	string path_to_bro = path ? find_file(bro_command, path): "";
+	if ( disabled )
+		return;
+
+	const char* env_path = getenv("PATH");
+	string path = env_path ? string(env_path) + ":." : ".";
+	string path_to_bro = find_file(bro_command, path);
 	struct stat s;

 	if ( path_to_bro.empty() || stat(path_to_bro.c_str(), &s) < 0 )
--- a/src/const.bif
+++ b/src/const.bif
@ -5,6 +5,7 @@
 const ignore_keep_alive_rexmit: bool;
 const skip_http_data: bool;
 const use_conn_size_analyzer: bool;
+const detect_filtered_trace: bool;
 const report_gaps_for_partial: bool;
 const exit_only_after_terminate: bool;

@ -17,6 +18,7 @@ const Tunnel::enable_ip: bool;
 const Tunnel::enable_ayiya: bool;
 const Tunnel::enable_teredo: bool;
 const Tunnel::enable_gtpv1: bool;
+const Tunnel::enable_gre: bool;
 const Tunnel::yielding_teredo_decapsulation: bool;
 const Tunnel::delay_teredo_confirmation: bool;
 const Tunnel::delay_gtp_confirmation: bool;
--- a/src/event.bif
+++ b/src/event.bif
@ -591,7 +591,11 @@ event software_unparsed_version_found%(c: connection, host: addr, str: string%);
 ## and it raises this event for each system identified. The p0f fingerprints are
 ## defined by :bro:id:`passive_fingerprint_file`.
 ##
-## TODO.
+## c: The connection.
+##
+## host: The host running the reported OS.
+##
+## OS: The OS version string.
 ##
 ## .. bro:see:: passive_fingerprint_file software_parse_error
 ##    software_version_found software_unparsed_version_found
--- a/src/file_analysis/File.cc
+++ b/src/file_analysis/File.cc
@ -103,7 +103,6 @@ File::~File()
 	DBG_LOG(DBG_FILE_ANALYSIS, "Destroying File object %s", id.c_str());
 	Unref(val);

-	// Queue may not be empty in the case where only content gaps were seen.
 	while ( ! fonc_queue.empty() )
 		{
 		delete_vals(fonc_queue.front().second);
@ -460,20 +459,27 @@ void File::FileEvent(EventHandlerPtr h)
 	FileEvent(h, vl);
 	}

+static void flush_file_event_queue(queue<pair<EventHandlerPtr, val_list*> >& q)
+	{
+	while ( ! q.empty() )
+		{
+		pair<EventHandlerPtr, val_list*> p = q.front();
+		mgr.QueueEvent(p.first, p.second);
+		q.pop();
+		}
+	}
+
 void File::FileEvent(EventHandlerPtr h, val_list* vl)
 	{
+	if ( h == file_state_remove )
+		flush_file_event_queue(fonc_queue);
+
 	mgr.QueueEvent(h, vl);

 	if ( h == file_new )
 		{
 		did_file_new_event = true;
-
-		while ( ! fonc_queue.empty() )
-			{
-			pair<EventHandlerPtr, val_list*> p = fonc_queue.front();
-			mgr.QueueEvent(p.first, p.second);
-			fonc_queue.pop();
-			}
+		flush_file_event_queue(fonc_queue);
 		}

 	if ( h == file_new || h == file_timeout || h == file_extraction_limit )
--- a/src/file_analysis/Manager.cc
+++ b/src/file_analysis/Manager.cc
@ -75,36 +75,47 @@ void Manager::SetHandle(const string& handle)
 	current_file_id = HashHandle(handle);
 	}

-void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
-                     analyzer::Tag tag, Connection* conn, bool is_orig)
+string Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
+                       analyzer::Tag tag, Connection* conn, bool is_orig,
+                       const string& precomputed_id)
 	{
-	GetFileHandle(tag, conn, is_orig);
-	File* file = GetFile(current_file_id, conn, tag, is_orig);
+	string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
+	File* file = GetFile(id, conn, tag, is_orig);

 	if ( ! file )
-		return;
+		return "";

 	file->DataIn(data, len, offset);

 	if ( file->IsComplete() )
+		{
 		RemoveFile(file->GetID());
+		return "";
 		}

-void Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
-                     Connection* conn, bool is_orig)
+	return id;
+	}
+
+string Manager::DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
+                       Connection* conn, bool is_orig, const string& precomputed_id)
 	{
-	GetFileHandle(tag, conn, is_orig);
+	string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
 	// Sequential data input shouldn't be going over multiple conns, so don't
 	// do the check to update connection set.
-	File* file = GetFile(current_file_id, conn, tag, is_orig, false);
+	File* file = GetFile(id, conn, tag, is_orig, false);

 	if ( ! file )
-		return;
+		return "";

 	file->DataIn(data, len);

 	if ( file->IsComplete() )
+		{
 		RemoveFile(file->GetID());
+		return "";
+		}
+
+	return id;
 	}

 void Manager::DataIn(const u_char* data, uint64 len, const string& file_id,
@ -133,8 +144,7 @@ void Manager::EndOfFile(analyzer::Tag tag, Connection* conn)
 void Manager::EndOfFile(analyzer::Tag tag, Connection* conn, bool is_orig)
 	{
 	// Don't need to create a file if we're just going to remove it right away.
-	GetFileHandle(tag, conn, is_orig);
-	RemoveFile(current_file_id);
+	RemoveFile(GetFileID(tag, conn, is_orig));
 	}

 void Manager::EndOfFile(const string& file_id)
@ -142,31 +152,37 @@ void Manager::EndOfFile(const string& file_id)
 	RemoveFile(file_id);
 	}

-void Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
-                  Connection* conn, bool is_orig)
+string Manager::Gap(uint64 offset, uint64 len, analyzer::Tag tag,
+                    Connection* conn, bool is_orig, const string& precomputed_id)
 	{
-	GetFileHandle(tag, conn, is_orig);
-	File* file = GetFile(current_file_id, conn, tag, is_orig);
+	string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
+	File* file = GetFile(id, conn, tag, is_orig);

 	if ( ! file )
-		return;
+		return "";

 	file->Gap(offset, len);
+	return id;
 	}

-void Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
-                      bool is_orig)
+string Manager::SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
+                        bool is_orig, const string& precomputed_id)
 	{
-	GetFileHandle(tag, conn, is_orig);
-	File* file = GetFile(current_file_id, conn, tag, is_orig);
+	string id = precomputed_id.empty() ? GetFileID(tag, conn, is_orig) : precomputed_id;
+	File* file = GetFile(id, conn, tag, is_orig);

 	if ( ! file )
-		return;
+		return "";

 	file->SetTotalBytes(size);

 	if ( file->IsComplete() )
+		{
 		RemoveFile(file->GetID());
+		return "";
+		}
+
+	return id;
 	}

 bool Manager::SetTimeoutInterval(const string& file_id, double interval) const
@ -317,15 +333,15 @@ bool Manager::IsIgnored(const string& file_id)
 	return ignored.find(file_id) != ignored.end();
 	}

-void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)
+string Manager::GetFileID(analyzer::Tag tag, Connection* c, bool is_orig)
 	{
 	current_file_id.clear();

 	if ( IsDisabled(tag) )
-		return;
+		return "";

 	if ( ! get_file_handle )
-		return;
+		return "";

 	EnumVal* tagval = tag.AsEnumVal();
 	Ref(tagval);
@ -337,6 +353,7 @@ void Manager::GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig)

 	mgr.QueueEvent(get_file_handle, vl);
 	mgr.Drain(); // need file handle immediately so we don't have to buffer data
+	return current_file_id;
 	}

 bool Manager::IsDisabled(analyzer::Tag tag)
--- a/src/file_analysis/Manager.h
+++ b/src/file_analysis/Manager.h
@ -82,9 +82,17 @@ public:
 	 * @param conn network connection over which the file data is transferred.
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
+	 * @param precomputed_file_id may be set to a previous return value in order to
+	 *        bypass costly file handle lookups.
+	 * @return a unique file ID string which, in certain contexts, may be
+	 *         cached and passed back in to a subsequent function call in order
+	 *         to avoid costly file handle lookups (which have to go through
+	 *         the \c get_file_handle script-layer event).  An empty string
+	 *         indicates the associate file is not going to be analyzed further.
 	 */
-	void DataIn(const u_char* data, uint64 len, uint64 offset,
-		    analyzer::Tag tag, Connection* conn, bool is_orig);
+	std::string DataIn(const u_char* data, uint64 len, uint64 offset,
+	                   analyzer::Tag tag, Connection* conn, bool is_orig,
+	                   const std::string& precomputed_file_id = "");

 	/**
 	 * Pass in sequential file data.
@ -94,9 +102,17 @@ public:
 	 * @param conn network connection over which the file data is transferred.
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
+	 * @param precomputed_file_id may be set to a previous return value in order to
+	 *        bypass costly file handle lookups.
+	 * @return a unique file ID string which, in certain contexts, may be
+	 *         cached and passed back in to a subsequent function call in order
+	 *         to avoid costly file handle lookups (which have to go through
+	 *         the \c get_file_handle script-layer event).  An empty string
+	 *         indicates the associate file is not going to be analyzed further.
 	 */
-	void DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
-	            Connection* conn, bool is_orig);
+	std::string DataIn(const u_char* data, uint64 len, analyzer::Tag tag,
+	                   Connection* conn, bool is_orig,
+	                   const std::string& precomputed_file_id = "");

 	/**
 	 * Pass in sequential file data from external source (e.g. input framework).
@ -140,9 +156,17 @@ public:
 	 * @param conn network connection over which the file data is transferred.
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
+	 * @param precomputed_file_id may be set to a previous return value in order to
+	 *        bypass costly file handle lookups.
+	 * @return a unique file ID string which, in certain contexts, may be
+	 *         cached and passed back in to a subsequent function call in order
+	 *         to avoid costly file handle lookups (which have to go through
+	 *         the \c get_file_handle script-layer event).  An empty string
+	 *         indicates the associate file is not going to be analyzed further.
 	 */
-	void Gap(uint64 offset, uint64 len, analyzer::Tag tag, Connection* conn,
-	         bool is_orig);
+	std::string Gap(uint64 offset, uint64 len, analyzer::Tag tag,
+	                Connection* conn, bool is_orig,
+	                const std::string& precomputed_file_id = "");

 	/**
 	 * Provide the expected number of bytes that comprise a file.
@ -151,9 +175,16 @@ public:
 	 * @param conn network connection over which the file data is transferred.
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
+	 * @param precomputed_file_id may be set to a previous return value in order to
+	 *        bypass costly file handle lookups.
+	 * @return a unique file ID string which, in certain contexts, may be
+	 *         cached and passed back in to a subsequent function call in order
+	 *         to avoid costly file handle lookups (which have to go through
+	 *         the \c get_file_handle script-layer event).  An empty string
+	 *         indicates the associate file is not going to be analyzed further.
 	 */
-	void SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
-	             bool is_orig);
+	std::string SetSize(uint64 size, analyzer::Tag tag, Connection* conn,
+	                    bool is_orig, const std::string& precomputed_file_id = "");

 	/**
 	 * Starts ignoring a file, which will finally be removed from internal
@ -283,8 +314,10 @@ protected:
 	 * @param conn network connection over which the file is transferred.
 	 * @param is_orig true if the file is being sent from connection originator
 	 *        or false if is being sent in the opposite direction.
+	 * @return #current_file_id, which is a hash of a unique file handle string
+	 *         set by a \c get_file_handle event handler.
 	 */
-	void GetFileHandle(analyzer::Tag tag, Connection* c, bool is_orig);
+	std::string GetFileID(analyzer::Tag tag, Connection* c, bool is_orig);

 	/**
 	 * Check if analysis is available for files transferred over a given
--- a/src/file_analysis/analyzer/unified2/events.bif
+++ b/src/file_analysis/analyzer/unified2/events.bif
@ -1,7 +1,17 @@

 ## Abstract all of the various Unified2 event formats into 
 ## a single event.
+##
+## f: The file.
+##
+## ev: TODO.
+##
 event unified2_event%(f: fa_file, ev: Unified2::IDSEvent%);

 ## The Unified2 packet format event.
+##
+## f: The file.
+##
+## pkt: TODO.
+##
 event unified2_packet%(f: fa_file, pkt: Unified2::Packet%);
--- a/src/input/Manager.cc
+++ b/src/input/Manager.cc
@ -397,7 +397,9 @@ bool Manager::CreateEventStream(RecordVal* fval)
 	string stream_name = name_val->AsString()->CheckString();
 	Unref(name_val);

-	RecordType *fields = fval->Lookup("fields", true)->AsType()->AsTypeType()->Type()->AsRecordType();
+	Val* fields_val = fval->Lookup("fields", true);
+	RecordType *fields = fields_val->AsType()->AsTypeType()->Type()->AsRecordType();
+	Unref(fields_val);

 	Val *want_record = fval->Lookup("want_record", true);

@ -548,13 +550,17 @@ bool Manager::CreateTableStream(RecordVal* fval)

 	Val* pred = fval->Lookup("pred", true);

-	RecordType *idx = fval->Lookup("idx", true)->AsType()->AsTypeType()->Type()->AsRecordType();
+	Val* idx_val = fval->Lookup("idx", true);
+	RecordType *idx = idx_val->AsType()->AsTypeType()->Type()->AsRecordType();
+	Unref(idx_val);
+
 	RecordType *val = 0;

-	if ( fval->Lookup("val", true) != 0 )
+	Val* val_val = fval->Lookup("val", true);
+	if ( val_val )
 		{
-		val = fval->Lookup("val", true)->AsType()->AsTypeType()->Type()->AsRecordType();
-		Unref(val); // The lookupwithdefault in the if-clause ref'ed val.
+		val = val_val->AsType()->AsTypeType()->Type()->AsRecordType();
+		Unref(val_val);
 		}

 	TableVal *dst = fval->Lookup("destination", true)->AsTableVal();
@ -729,7 +735,7 @@ bool Manager::CreateTableStream(RecordVal* fval)
 	stream->pred = pred ? pred->AsFunc() : 0;
 	stream->num_idx_fields = idxfields;
 	stream->num_val_fields = valfields;
-	stream->tab = dst->AsTableVal();
+	stream->tab = dst->AsTableVal(); // ref'd by lookupwithdefault
 	stream->rtype = val ? val->AsRecordType() : 0;
 	stream->itype = idx->AsRecordType();
 	stream->event = event ? event_registry->Lookup(event->Name()) : 0;
--- a/src/input/readers/Raw.cc
+++ b/src/input/readers/Raw.cc
@ -203,11 +203,24 @@ bool Raw::Execute()

 		// Parent also sets child process group immediately to avoid a race.
 		if ( setpgid(childpid, childpid) == -1 )
+			{
+			if ( errno == EACCES )
+				// Child already did exec. That's fine since then it must have
+				// already done the setpgid() itself.
+				;
+
+			else if ( errno == ESRCH && kill(childpid, 0) == 0 )
+				// Sometimes (e.g. FreeBSD) this error is reported even though
+				// child exists, so do extra sanity check of whether it exists.
+				;
+
+			else
 				{
 				char buf[256];
 				strerror_r(errno, buf, sizeof(buf));
 				Warning(Fmt("Could not set child process group: %s", buf));
 				}
+			}

 		if ( ! UnlockForkMutex() )
 			return false;
@ -352,7 +365,7 @@ bool Raw::DoInit(const ReaderInfo& info, int num_fields, const Field* const* fie
 		fname = source.substr(0, fname.length() - 1);
 		}

-	map<const char*, const char*>::const_iterator it = info.config.find("stdin"); // data that is sent to the child process
+	ReaderInfo::config_map::const_iterator it = info.config.find("stdin"); // data that is sent to the child process
 	if ( it != info.config.end() )
 		{
 		stdin_string = it->second;
--- a/src/input/readers/SQLite.cc
+++ b/src/input/readers/SQLite.cc
@ -85,7 +85,7 @@ bool SQLite::DoInit(const ReaderInfo& info, int arg_num_fields, const threading:
 	fullpath.append(".sqlite");

 	string query;
-	map<const char*, const char*>::const_iterator it = info.config.find("query");
+	ReaderInfo::config_map::const_iterator it = info.config.find("query");
 	if ( it == info.config.end() )
 		{
 		Error(Fmt("No query specified when setting up SQLite data source. Aborting.", info.source));
--- a/src/logging/writers/SQLite.cc
+++ b/src/logging/writers/SQLite.cc
@ -126,7 +126,7 @@ bool SQLite::DoInit(const WriterInfo& info, int arg_num_fields,
 	fullpath.append(".sqlite");
 	string tablename;

-	map<const char*, const char*>::const_iterator it = info.config.find("tablename");
+	WriterInfo::config_map::const_iterator it = info.config.find("tablename");
 	if ( it == info.config.end() )
 		{
 		MsgThread::Info(Fmt("tablename configuration option not found. Defaulting to path %s", info.path));
--- a/src/parse.y
+++ b/src/parse.y
@ -117,6 +117,7 @@ int in_init = 0;
 int in_record = 0;
 bool resolving_global_ID = false;
 bool defining_global_ID = false;
+std::vector<int> saved_in_init;

 ID* func_id = 0;
 EnumType *cur_enum_type = 0;
@ -1113,12 +1114,24 @@ func_hdr:
 	;

 func_body:
-		opt_attr '{' stmt_list '}'
+		opt_attr '{'
+			{
+			saved_in_init.push_back(in_init);
+			in_init = 0;
+			}
+
+		stmt_list
+			{
+			in_init = saved_in_init.back();
+			saved_in_init.pop_back();
+			}
+
+		'}'
 			{
 			if ( optimize )
-				$3 = $3->Simplify();
+				$4 = $4->Simplify();

-			end_func($3, $1);
+			end_func($4, $1);
 			}
 	;

--- a/src/probabilistic/top-k.bif
+++ b/src/probabilistic/top-k.bif
@ -138,6 +138,10 @@ function topk_sum%(handle: opaque of topk%): count

 ## Merge the second top-k data structure into the first.
 ##
+## handle1: the first TopK handle.
+##
+## handle2: the second TopK handle.
+##
 ## .. note:: This does not remove any elements, the resulting data structure
 ##    can be bigger than the maximum size given on initialization.
 ##
--- a/src/util.cc
+++ b/src/util.cc
@ -1649,48 +1649,28 @@ void operator delete[](void* v)

 #endif

-// Being selective of which components of MAGIC_NO_CHECK_BUILTIN are actually
-// known to be problematic, but keeping rest of libmagic's builtin checks.
-#define DISABLE_LIBMAGIC_BUILTIN_CHECKS  ( \
-/*  MAGIC_NO_CHECK_COMPRESS | */ \
-/*  MAGIC_NO_CHECK_TAR  | */ \
-/*  MAGIC_NO_CHECK_SOFT | */ \
-/*  MAGIC_NO_CHECK_APPTYPE  | */ \
-/*  MAGIC_NO_CHECK_ELF  | */ \
-/*  MAGIC_NO_CHECK_TEXT | */ \
-    MAGIC_NO_CHECK_CDF  | \
-    MAGIC_NO_CHECK_TOKENS  \
-/*  MAGIC_NO_CHECK_ENCODING */ \
-)
-
 void bro_init_magic(magic_t* cookie_ptr, int flags)
 	{
 	if ( ! cookie_ptr || *cookie_ptr )
 		return;

-	*cookie_ptr = magic_open(flags|DISABLE_LIBMAGIC_BUILTIN_CHECKS);
+	*cookie_ptr = magic_open(flags);

-	// Use our custom database for mime types, but the default database
-	// from libmagic for the verbose file type.
-	const char* database = (flags & MAGIC_MIME) ? bro_magic_path() : 0;
+	// Always use Bro's custom magic database.
+	const char* database = bro_magic_path();

 	if ( ! *cookie_ptr )
 		{
 		const char* err = magic_error(*cookie_ptr);
-		if ( ! err )
-			err = "unknown";
-
-		reporter->InternalError("can't init libmagic: %s", err);
+		reporter->InternalError("can't init libmagic: %s",
+		                        err ? err : "unknown");
 		}

 	else if ( magic_load(*cookie_ptr, database) < 0 )
 		{
 		const char* err = magic_error(*cookie_ptr);
-		if ( ! err )
-			err = "unknown";
-
-		const char* db_name = database ? database : "<default>";
-		reporter->InternalError("can't load magic file %s: %s", db_name, err);
+		reporter->InternalError("can't load magic file %s: %s", database,
+		                        err ? err : "unknown");
 		magic_close(*cookie_ptr);
 		*cookie_ptr = 0;
 		}
--- a/testing/btest/Baseline/core.tcp.fin-retransmit/out
+++ b/testing/btest/Baseline/core.tcp.fin-retransmit/out
@ -0,0 +1,2 @@
+[size=0, state=5, num_pkts=3, num_bytes_ip=156, flow_label=0]
+[size=0, state=6, num_pkts=2, num_bytes_ip=92, flow_label=0]
--- a/testing/btest/Baseline/core.tcp.miss-end-data/conn.log
+++ b/testing/btest/Baseline/core.tcp.miss-end-data/conn.log
@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-01-24-22-19-38
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1331764471.664131	CXWv6p3arKYeMETxOg	192.168.122.230	60648	77.238.160.184	80	tcp	http	10.048360	538	2902	SF	-	2902	ShADafF	5	750	4	172	(empty)
+#close	2014-01-24-22-19-38
--- a/testing/btest/Baseline/core.tcp.miss-end-data/out
+++ b/testing/btest/Baseline/core.tcp.miss-end-data/out
@ -0,0 +1 @@
+content_gap, [orig_h=192.168.122.230, orig_p=60648/tcp, resp_h=77.238.160.184, resp_p=80/tcp], F, 1, 2902
--- a/testing/btest/Baseline/core.tunnels.gre-in-gre/conn.log
+++ b/testing/btest/Baseline/core.tunnels.gre-in-gre/conn.log
@ -0,0 +1,12 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-01-16-21-51-36
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1341436440.002928	CRJuHdVW0XPVINV8a	3.3.3.2	520	224.0.0.9	520	udp	-	26.148268	48	0	S0	-	0	D	2	104	0	0	CjhGID4nQcgTWjvg4c
+1341436424.378840	CsRx2w45OKnoww6xl4	3.3.3.1	520	224.0.0.9	520	udp	-	28.555457	168	0	S0	-	0	D	2	224	0	0	CjhGID4nQcgTWjvg4c
+1341436424.204043	CCvvfg3TEfuqmmG4bh	10.10.25.1	8	192.168.1.2	0	icmp	-	42.380221	22464	22464	OTH	-	0	-	312	31200	312	31200	CjhGID4nQcgTWjvg4c
+#close	2014-01-16-21-51-36
--- a/testing/btest/Baseline/core.tunnels.gre-in-gre/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.gre-in-gre/tunnel.log
@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	tunnel
+#open	2014-01-16-21-51-36
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
+#types	time	string	addr	port	addr	port	enum	enum
+1341436424.204043	CXWv6p3arKYeMETxOg	72.205.54.70	0	86.106.164.150	0	Tunnel::IP	Tunnel::DISCOVER
+1341436424.204043	CjhGID4nQcgTWjvg4c	10.10.11.2	0	10.10.13.2	0	Tunnel::IP	Tunnel::DISCOVER
+#close	2014-01-16-21-51-36
--- a/testing/btest/Baseline/core.tunnels.gre/conn.log
+++ b/testing/btest/Baseline/core.tunnels.gre/conn.log
@ -0,0 +1,16 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	conn
+#open	2014-01-16-21-51-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	service	duration	orig_bytes	resp_bytes	conn_state	local_orig	missed_bytes	history	orig_pkts	orig_ip_bytes	resp_pkts	resp_ip_bytes	tunnel_parents
+#types	time	string	addr	port	addr	port	enum	string	interval	count	count	string	bool	count	string	count	count	count	count	table[string]
+1055289978.756932	CsRx2w45OKnoww6xl4	66.59.111.190	40264	172.28.2.3	22	tcp	ssh	3.157831	952	1671	SF	-	0	ShAdDaFf	12	1584	10	2199	CXWv6p3arKYeMETxOg
+1055289987.055189	CRJuHdVW0XPVINV8a	66.59.111.190	37675	172.28.2.3	53	udp	dns	5.001141	66	0	S0	-	0	D	2	122	0	0	CXWv6p3arKYeMETxOg
+1055289996.849099	CIPOse170MGiRM1Qf4	66.59.111.190	123	129.170.17.4	123	udp	-	0.072374	48	48	SF	-	0	Dd	1	76	1	76	CXWv6p3arKYeMETxOg
+1055289973.849878	CCvvfg3TEfuqmmG4bh	66.59.111.190	123	18.26.4.105	123	udp	-	0.074086	48	48	SF	-	0	Dd	1	76	1	76	CXWv6p3arKYeMETxOg
+1055289992.849231	C6pKV8GSxOnSLghOa	66.59.111.190	123	66.59.111.182	123	udp	-	0.056629	48	48	SF	-	0	Dd	1	76	1	76	CXWv6p3arKYeMETxOg
+1055289968.793044	CjhGID4nQcgTWjvg4c	66.59.111.190	8	172.28.2.3	0	icmp	-	3.061298	224	224	OTH	-	0	-	4	336	4	336	CXWv6p3arKYeMETxOg
+1055289987.106744	CPbrpk1qSsw6ESzHV4	172.28.2.3	3	66.59.111.190	3	icmp	-	4.994662	122	0	OTH	-	0	-	2	178	0	0	CXWv6p3arKYeMETxOg
+#close	2014-01-16-21-51-12
--- a/testing/btest/Baseline/core.tunnels.gre/dns.log
+++ b/testing/btest/Baseline/core.tunnels.gre/dns.log
@ -0,0 +1,11 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	dns
+#open	2014-01-16-21-51-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	proto	trans_id	query	qclass	qclass_name	qtype	qtype_name	rcode	rcode_name	AA	TC	RD	RA	Z	answers	TTLs	rejected
+#types	time	string	addr	port	addr	port	enum	count	string	count	string	count	string	count	string	bool	bool	bool	bool	count	vector[string]	vector[interval]	bool
+1055289987.055189	CRJuHdVW0XPVINV8a	66.59.111.190	37675	172.28.2.3	53	udp	48554	www.gleeble.org	1	C_INTERNET	255	*	-	-	F	F	T	F	0	-	-	F
+1055289992.056330	CRJuHdVW0XPVINV8a	66.59.111.190	37675	172.28.2.3	53	udp	48554	www.gleeble.org	1	C_INTERNET	255	*	-	-	F	F	T	F	0	-	-	F
+#close	2014-01-16-21-51-12
--- a/testing/btest/Baseline/core.tunnels.gre/ssh.log
+++ b/testing/btest/Baseline/core.tunnels.gre/ssh.log
@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	ssh
+#open	2014-01-16-21-51-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	status	direction	client	server
+#types	time	string	addr	port	addr	port	string	enum	string	string
+1055289978.855137	CsRx2w45OKnoww6xl4	66.59.111.190	40264	172.28.2.3	22	failure	INBOUND	SSH-2.0-OpenSSH_3.6.1p1	SSH-1.99-OpenSSH_3.1p1
+#close	2014-01-16-21-51-12
--- a/testing/btest/Baseline/core.tunnels.gre/tunnel.log
+++ b/testing/btest/Baseline/core.tunnels.gre/tunnel.log
@ -0,0 +1,10 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	tunnel
+#open	2014-01-16-21-51-12
+#fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	tunnel_type	action
+#types	time	string	addr	port	addr	port	enum	enum
+1055289968.793044	CXWv6p3arKYeMETxOg	172.27.1.66	0	66.59.109.137	0	Tunnel::IP	Tunnel::DISCOVER
+#close	2014-01-16-21-51-12
--- a/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log
+++ b/testing/btest/Baseline/core.tunnels.gtp.outer_ip_frag/http.log
@ -6,5 +6,5 @@
 #open	2013-08-26-19-02-18
 #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	trans_depth	method	host	uri	referrer	user_agent	request_body_len	response_body_len	status_code	status_msg	info_code	info_msg	filename	tags	username	password	proxied	orig_fuids	orig_mime_types	resp_fuids	resp_mime_types
 #types	time	string	addr	port	addr	port	count	string	string	string	string	string	count	count	count	string	count	string	string	table[enum]	string	string	table[string]	vector[string]	vector[string]	vector[string]	vector[string]
-1333458850.375568	CjhGID4nQcgTWjvg4c	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	-	-	FNJkBA1b8FSHt5N8jl	application/octet-stream
+1333458850.375568	CjhGID4nQcgTWjvg4c	10.131.47.185	1923	79.101.110.141	80	1	GET	o-o.preferred.telekomrs-beg1.v2.lscache8.c.youtube.com	/videoplayback?upn=MTU2MDY5NzQ5OTM0NTI3NDY4NDc&sparams=algorithm,burst,cp,factor,id,ip,ipbits,itag,source,upn,expire&fexp=912300,907210&algorithm=throttle-factor&itag=34&ip=212.0.0.0&burst=40&sver=3&signature=832FB1042E20780CFCA77A4DB5EA64AC593E8627.D1166C7E8365732E52DAFD68076DAE0146E0AE01&source=youtube&expire=1333484980&key=yt1&ipbits=8&factor=1.25&cp=U0hSSFRTUl9NSkNOMl9MTVZKOjh5eEN2SG8tZF84&id=ebf1e932d4bd1286&cm2=1	http://s.ytimg.com/yt/swfbin/watch_as3-vflqrJwOA.swf	Mozilla/5.0 (Windows NT 5.1) AppleWebKit/535.11 (KHTML, like Gecko; X-SBLSP) Chrome/17.0.963.83 Safari/535.11	0	56320	206	Partial Content	-	-	-	(empty)	-	-	-	-	-	FNJkBA1b8FSHt5N8jl	binary
 #close	2013-08-26-19-02-18
--- a/testing/btest/Baseline/core.when-interpreter-exceptions/bro.output
+++ b/testing/btest/Baseline/core.when-interpreter-exceptions/bro.output
@ -1,13 +1,13 @@
-1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 96: field value missing [p$ip]
-1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 63: field value missing [p$ip]
-1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 79: field value missing [p$ip]
-1386110869.157209 expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 36: field value missing [p$ip]
-1386110869.157209 received termination signal
+expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 48: field value missing [myrecord$notset]
+expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 92: field value missing [myrecord$notset]
+expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 73: field value missing [myrecord$notset]
+expression error in /Users/jon/Projects/bro/bro/testing/btest/.tmp/core.when-interpreter-exceptions/when-interpreter-exceptions.bro, line 104: field value missing [myrecord$notset]
+received termination signal
+[f(F)]
+f() done, no exception, T
+[f(T)]
+[bro_init()]
+timeout g(), T
 timeout
 timeout g(), F
-timeout g(), T
 g() done, no exception, T
-localhost resolved
-localhost resolved from f(), T
-localhost resolved from f(), F
-f() done, no exception, T
--- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
+++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log
@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2013-10-30-16-52-28
+#open	2014-01-31-22-54-38
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@ -220,5 +220,6 @@ scripts/base/init-default.bro
  scripts/base/files/unified2/__load__.bro
    scripts/base/files/unified2/main.bro
  scripts/base/misc/find-checksum-offloading.bro
+  scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2013-10-30-16-52-28
+#close	2014-01-31-22-54-38
--- a/testing/btest/Baseline/doc.sphinx.connection-record-01/btest-doc.sphinx.connection-record-01#1
+++ b/testing/btest/Baseline/doc.sphinx.connection-record-01/btest-doc.sphinx.connection-record-01#1
@ -4,10 +4,10 @@
      :linenos:
      :emphasize-lines: 1,1

-      # bro -b -r dns-session.trace connection_record_01.bro
-      [id=[orig_h=212.180.42.100, orig_p=25000/tcp, resp_h=131.243.64.3, resp_p=53/tcp], orig=[size=29, state=5, num_pkts=6, num_bytes_ip=273, flow_label=0], resp=[size=44, state=5, num_pkts=5, num_bytes_ip=248, flow_label=0], start_time=930613226.067666, duration=0.709643, service={
+      # bro -b -r http/get.trace connection_record_01.bro
+      [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0], start_time=1362692526.869344, duration=0.211484, service={
      
-      }, addl=, hot=0, history=ShADadFf, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, conn=[ts=930613226.067666, uid=CXWv6p3arKYeMETxOg, id=[orig_h=212.180.42.100, orig_p=25000/tcp, resp_h=131.243.64.3, resp_p=53/tcp], proto=tcp, service=<uninitialized>, duration=0.709643, orig_bytes=29, resp_bytes=44, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=ShADadFf, orig_pkts=6, orig_ip_bytes=273, resp_pkts=5, resp_ip_bytes=248, tunnel_parents={
+      }, addl=, hot=0, history=ShADadFf, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, conn=[ts=1362692526.869344, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], proto=tcp, service=<uninitialized>, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={
      
      }], extract_orig=F, extract_resp=F]

--- a/testing/btest/Baseline/doc.sphinx.connection-record-02/btest-doc.sphinx.connection-record-02#1
+++ b/testing/btest/Baseline/doc.sphinx.connection-record-02/btest-doc.sphinx.connection-record-02#1
@ -4,16 +4,14 @@
      :linenos:
      :emphasize-lines: 1,1

-      # bro -b -r dns-session.trace connection_record_02.bro
-      [id=[orig_h=212.180.42.100, orig_p=25000/tcp, resp_h=131.243.64.3, resp_p=53/tcp], orig=[size=29, state=5, num_pkts=6, num_bytes_ip=273, flow_label=0], resp=[size=44, state=5, num_pkts=5, num_bytes_ip=248, flow_label=0], start_time=930613226.067666, duration=0.709643, service={
+      # bro -b -r http/get.trace connection_record_02.bro
+      [id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], orig=[size=136, state=5, num_pkts=7, num_bytes_ip=512, flow_label=0], resp=[size=5007, state=5, num_pkts=7, num_bytes_ip=5379, flow_label=0], start_time=1362692526.869344, duration=0.211484, service={
      
-      }, addl=, hot=0, history=ShADadFf, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, conn=[ts=930613226.067666, uid=CXWv6p3arKYeMETxOg, id=[orig_h=212.180.42.100, orig_p=25000/tcp, resp_h=131.243.64.3, resp_p=53/tcp], proto=tcp, service=<uninitialized>, duration=0.709643, orig_bytes=29, resp_bytes=44, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=ShADadFf, orig_pkts=6, orig_ip_bytes=273, resp_pkts=5, resp_ip_bytes=248, tunnel_parents={
+      }, addl=, hot=0, history=ShADadFf, uid=CXWv6p3arKYeMETxOg, tunnel=<uninitialized>, conn=[ts=1362692526.869344, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], proto=tcp, service=<uninitialized>, duration=0.211484, orig_bytes=136, resp_bytes=5007, conn_state=SF, local_orig=<uninitialized>, missed_bytes=0, history=ShADadFf, orig_pkts=7, orig_ip_bytes=512, resp_pkts=7, resp_ip_bytes=5379, tunnel_parents={
      
-      }], extract_orig=F, extract_resp=F, dns=<uninitialized>, dns_state=[pending={
-      [34798] = [initialized=T, vals={
+      }], extract_orig=F, extract_resp=F, http=[ts=1362692526.939527, uid=CXWv6p3arKYeMETxOg, id=[orig_h=141.142.228.5, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp], trans_depth=1, method=GET, host=bro.org, uri=/download/CHANGES.bro-aux.txt, referrer=<uninitialized>, user_agent=Wget/1.14 (darwin12.2.0), request_body_len=0, response_body_len=4705, status_code=200, status_msg=OK, info_code=<uninitialized>, info_msg=<uninitialized>, filename=<uninitialized>, tags={
      
-      }, settings=[max_len=<uninitialized>], top=1, bottom=1, size=0]
-      }, finished_answers={
+      }, username=<uninitialized>, password=<uninitialized>, capture_password=F, proxied=<uninitialized>, range_request=F, orig_fuids=<uninitialized>, orig_mime_types=<uninitialized>, resp_fuids=[FakNcS1Jfe01uljb3], resp_mime_types=[text/plain], current_entity=<uninitialized>, orig_mime_depth=1, resp_mime_depth=1], http_state=[pending={
      
-      }]]
+      }, current_request=1, current_response=1]]

--- a/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1
+++ b/testing/btest/Baseline/doc.sphinx.file_extraction/btest-doc.sphinx.file_extraction#1
@ -0,0 +1,14 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/bro.org.pcap file_extraction.bro
+      Extracting file HTTP-FiIpIB2hRQSDBOSJRg.html
+      Extracting file HTTP-FnaT2a3UDd093opCB9.txt
+      Extracting file HTTP-FsvATF146kf1Emc21j.txt
+      Extracting file HTTP-FkMQHg2nBr44fc5h63.txt
+      Extracting file HTTP-FfQGqj4Fhh3pH7nVQj.txt
+      [...]
+
--- a/testing/btest/Baseline/doc.sphinx.ftp-bruteforce/btest-doc.sphinx.ftp-bruteforce#1
+++ b/testing/btest/Baseline/doc.sphinx.ftp-bruteforce/btest-doc.sphinx.ftp-bruteforce#1
@ -0,0 +1,24 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
+
+.. rst-class:: btest-include
+
+    .. code-block:: guess
+      :linenos:
+
+      #separator \x09
+      #set_separator	,
+      #empty_field	(empty)
+      #unset_field	-
+      #path	notice
+      #open	2014-01-21-21-56-07
+      #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+      #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+      1389721084.522861	-	-	-	-	-	-	-	-	-	FTP::Bruteforcing	192.168.56.1 had 20 failed logins on 1 FTP server in 0m37s	-	192.168.56.1	-	-	-	bro	Notice::ACTION_LOG	3600.000000	F	-	-	-	-	-
+      #close	2014-01-21-21-56-07
+
--- a/testing/btest/Baseline/doc.sphinx.http_proxy_01/btest-doc.sphinx.http_proxy_01#1
+++ b/testing/btest/Baseline/doc.sphinx.http_proxy_01/btest-doc.sphinx.http_proxy_01#1
@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_01.bro
+      A local server is acting as an open proxy: 192.168.56.101
+
--- a/testing/btest/Baseline/doc.sphinx.http_proxy_02/btest-doc.sphinx.http_proxy_02#1
+++ b/testing/btest/Baseline/doc.sphinx.http_proxy_02/btest-doc.sphinx.http_proxy_02#1
@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_02.bro
+      A local server is acting as an open proxy: 192.168.56.101
+
--- a/testing/btest/Baseline/doc.sphinx.http_proxy_03/btest-doc.sphinx.http_proxy_03#1
+++ b/testing/btest/Baseline/doc.sphinx.http_proxy_03/btest-doc.sphinx.http_proxy_03#1
@ -0,0 +1,9 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_03.bro
+      A local server is acting as an open proxy: 192.168.56.101
+
--- a/testing/btest/Baseline/doc.sphinx.http_proxy_04/btest-doc.sphinx.http_proxy_04#1
+++ b/testing/btest/Baseline/doc.sphinx.http_proxy_04/btest-doc.sphinx.http_proxy_04#1
@ -0,0 +1,24 @@
+.. rst-class:: btest-cmd
+
+    .. code-block:: none
+      :linenos:
+      :emphasize-lines: 1,1
+
+      # bro -r http/proxy.pcap http_proxy_04.bro
+
+.. rst-class:: btest-include
+
+    .. code-block:: guess
+      :linenos:
+
+      #separator \x09
+      #set_separator	,
+      #empty_field	(empty)
+      #unset_field	-
+      #path	notice
+      #open	2014-01-21-20-11-20
+      #fields	ts	uid	id.orig_h	id.orig_p	id.resp_h	id.resp_p	fuid	file_mime_type	file_desc	proto	note	msg	sub	src	dst	p	n	peer_descr	actions	suppress_for	dropped	remote_location.country_code	remote_location.region	remote_location.city	remote_location.latitude	remote_location.longitude
+      #types	time	string	addr	port	addr	port	string	string	string	enum	enum	string	string	addr	addr	port	count	string	table[enum]	interval	bool	string	string	string	double	double
+      1389654450.449603	CXWv6p3arKYeMETxOg	192.168.56.1	52679	192.168.56.101	80	-	-	-	tcp	HTTP::Open_Proxy	A local server is acting as an open proxy: 192.168.56.101	-	192.168.56.1	192.168.56.101	80	-	bro	Notice::ACTION_LOG	86400.000000	F	-	-	-	-	-
+      #close	2014-01-21-20-11-20
+
--- a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output
+++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_file_extraction_bro/output
@ -0,0 +1,28 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+file_extraction.bro
+
+
+global mime_to_ext: table[string] of string = {
+	["application/x-dosexec"] = "exe",
+	["text/plain"] = "txt",
+	["image/jpeg"] = "jpg",
+	["image/png"] = "png",
+	["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+	{
+	if ( f$source != "HTTP" )
+		return;
+
+	if ( ! f?$mime_type )
+		return;
+
+	if ( f$mime_type !in mime_to_ext )
+		return;
+
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	print fmt("Extracting file %s", fname);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}
--- a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_01_bro/output
+++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_01_bro/output
@ -0,0 +1,9 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_01.bro
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}
--- a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_02_bro/output
+++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_02_bro/output
@ -0,0 +1,30 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_02.bro
+
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}
--- a/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_03_bro/output
+++ b/testing/btest/Baseline/doc.sphinx.include-doc_httpmonitor_http_proxy_03_bro/output
@ -0,0 +1,35 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+http_proxy_03.bro
+
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}
--- a/Show more
+++ b/Show more
 @ -1 +1 @@
 .2-70
 .2-158
				`@ -0,0 +1 @@`
				`content_gap, [orig_h=192.168.122.230, orig_p=60648/tcp, resp_h=77.238.160.184, resp_p=80/tcp], F, 1, 2902`