From e5a4123b4e1bb901b9ff495f0d4c4171915d7360 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@illinois.edu>
Date: Fri, 20 Sep 2013 09:38:52 -0500
Subject: [PATCH 01/17] Kill raw input reader's child by process group.

This should help on systems where the default shell (e.g. dash on
Ubuntu) fork-execs in order to run commands.  In that case, we were
just killing the shell and the shell would exit without killing
the actual process corresponding to command that was requested.
Setting a process group for the shell exec'd from Bro and killing
by process group should help clean everything up since process group
will be inherited by any procs that shell fork-execs.
---
 src/input/readers/Raw.cc | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/src/input/readers/Raw.cc b/src/input/readers/Raw.cc
index 64c29d632c..34408b79d2 100644
--- a/src/input/readers/Raw.cc
+++ b/src/input/readers/Raw.cc
@@ -73,15 +73,15 @@ void Raw::DoClose()
 
 	if ( execute && childpid > 0 && kill(childpid, 0) == 0 )
 		{
-		// kill child process
-		kill(childpid, SIGTERM);
+		// kill child process group
+		kill(-childpid, SIGTERM);
 
 		if ( forcekill )
 			{
 			usleep(200); // 200 msecs should be enough for anyone ;)
 
 			if ( kill(childpid, 0) == 0 ) // perhaps it is already gone
-				kill(childpid, SIGKILL);
+				kill(-childpid, SIGKILL);
 			}
 		}
 	}
@@ -146,6 +146,11 @@ bool Raw::Execute()
 	else if ( childpid == 0 )
 		{
 		// we are the child.
+
+		// Obtain a process group w/ child's PID.
+		if ( setpgid(0, 0) == -1 )
+			_exit(251);
+
 		close(pipes[stdout_in]);
 		if ( dup2(pipes[stdout_out], stdout_fileno) == -1 )
 			_exit(252);
@@ -180,6 +185,15 @@ bool Raw::Execute()
 	else
 		{
 		// we are the parent
+
+		// Parent also sets child process group immediately to avoid a race.
+		if ( setpgid(childpid, childpid) == -1 )
+			{
+			char buf[256];
+			strerror_r(errno, buf, sizeof(buf));
+			Warning(Fmt("Could not set child process group: %s", buf));
+			}
+
 		if ( ! UnlockForkMutex() )
 			return false;
 

From 17bc3955f90b62659f55a85ad11dfc710be2a484 Mon Sep 17 00:00:00 2001
From: Scott Runnels <srunnels@gmail.com>
Date: Fri, 20 Sep 2013 11:43:45 -0400
Subject: [PATCH 02/17] Update the lines included from events.bif.bro.

Previously listed connection_established and connection_finished which
are no longer in place in events.bif.bro.
---
 doc/scripting/index.rst | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst
index 197180241e..aca1a9472e 100644
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@@ -197,13 +197,8 @@ such, there are events defined for the primary parts of the connection
 life-cycle as you'll see from the small selection of
 connection-related events below.  
 
-.. todo::
-
-   Update the line numbers, this isn't pulling in the right events
-   anymore but I don't know which ones it were.
-
 .. btest-include:: ${BRO_SRC_ROOT}/build/scripts/base/bif/event.bif.bro
-   :lines: 135-138,154,204-208,218,255-256,266,335-340,351
+   :lines: 69-72,88,106-109,129,132-137,148
 
 Of the events listed, the event that will give us the best insight
 into the connection record data type will be

From 5fede2f73e6d7758bbb27a73440a01efba04731c Mon Sep 17 00:00:00 2001
From: Scott Runnels <srunnels@gmail.com>
Date: Fri, 20 Sep 2013 12:22:12 -0400
Subject: [PATCH 03/17] Spelling corrections.

Apparently I am unable to spell "separate".
---
 doc/scripting/index.rst | 44 ++++++++++++++++++++---------------------
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst
index aca1a9472e..b01165a5bc 100644
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@@ -33,7 +33,7 @@ are invalid.  This entire process is setup by telling Bro that should
 it see a server or client issue an SSL ``HELLO`` message, we want to know
 about the information about that connection.
 
-It's often the easiest to understand Bro's scripting language by
+It's often easiest to understand Bro's scripting language by
 looking at a complete script and breaking it down into its
 identifiable components.  In this example, we'll take a look at how
 Bro queries the `Team Cymru Malware hash registry
@@ -76,7 +76,7 @@ this level of granularity might not be entirely necessary though.
 
 The export section redefines an enumerable constant that describes the
 type of notice we will generate with the logging framework.  Bro
-allows for redefinable constants, which at first, might seem
+allows for re-definable constants, which at first, might seem
 counter-intuitive.  We'll get more in-depth with constants in a later
 chapter, for now, think of them as variables that can only be altered
 before Bro starts running.  The notice type listed allows for the use
@@ -84,7 +84,7 @@ of the :bro:id:`NOTICE` function to generate notices of type
 ``Malware_Hash_Registry_Match`` as done in the next section.  Notices
 allow Bro to generate some kind of extra notification beyond its
 default log types.  Often times, this extra notification comes in the
-form of an email generated and sent to a pre-configured address.  
+form of an email generated and sent to a preconfigured address.  
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
    :lines: 26-44
@@ -112,9 +112,9 @@ The ``when`` block performs a DNS TXT lookup and stores the result
 in the local variable ``MHR_result``.  Effectively, processing for
 this event continues and upon receipt of the values returned by
 :bro:id:`lookup_hostname_txt`, the ``when`` block is executed.  The
-``when`` block splits the string returned into two seperate values and
+``when`` block splits the string returned into two separate values and
 checks to ensure an expected format.  If the format is invalid, the
-script assumes that the hash wasn't found in the respository and
+script assumes that the hash wasn't found in the repository and
 processing is concluded.  If the format is as expected and the
 detection rate is above the threshold set by ``MHR_threshold``, two
 new local variables are created and used in the notice issued by
@@ -168,7 +168,7 @@ the event, and a concise explanation of the functions use.
    :lines: 29-54
 
 Above is a segment of the documentation for the event
-:bro:id:`dns_request` (and the preceeding link points to the
+:bro:id:`dns_request` (and the preceding link points to the
 documentation generated out of that).  It's organized such that the
 documentation, commentary, and list of arguments precede the actual
 event definition used by Bro.  As Bro detects DNS requests being
@@ -240,7 +240,7 @@ information gleaned from the analysis of a connection as a complete
 unit.  To break down this collection of information, you will have to
 make use of use Bro's field delimiter ``$``.  For example, the
 originating host is referenced by ``c$id$orig_h`` which if given a
-narritive relates to ``orig_h`` which is a member of ``id`` which is
+narrative relates to ``orig_h`` which is a member of ``id`` which is
 a member of the data structure referred to as ``c`` that was passed
 into the event handler." Given that the responder port
 (``c$id$resp_p``) is ``53/tcp``, it's likely that Bro's base DNS scripts
@@ -338,7 +338,7 @@ Constants
 Bro also makes use of constants, which are denoted by the ``const``
 keyword.  Unlike globals, constants can only be set or altered at
 parse time if the ``&redef`` attribute has been used.  Afterwards (in
-runtime) the constants are unalterable.  In most cases, redefinable
+runtime) the constants are unalterable.  In most cases, re-definable
 constants are used in Bro scripts as containers for configuration
 options.  For example, the configuration option to log password
 decrypted from HTTP streams is stored in
@@ -354,7 +354,7 @@ following line to our ``site/local.bro`` file before firing up Bro.
 
 .. btest-include:: ${DOC_ROOT}/scripting/data_type_const_simple.bro
 
-While the idea of a redefinable constant might be odd, the constraint
+While the idea of a re-definable constant might be odd, the constraint
 that constants can only be altered at parse-time remains even with the
 ``&redef`` attribute.  In the code snippet below, a table of strings
 indexed by ports is declared as a constant before two values are added
@@ -412,7 +412,7 @@ The table below shows the atomic types used in Bro, of which the
 first four should seem familiar if you have some scripting experience,
 while the remaining six are less common in other languages. It should
 come as no surprise that a scripting language for a Network Security
-Monitoring platform has a fairly robust set of network centric data
+Monitoring platform has a fairly robust set of network-centric data
 types and taking note of them here may well save you a late night of
 reinventing the wheel.  
 
@@ -474,7 +474,7 @@ the ``for`` loop, the next element is chosen.  Since sets are not an
 ordered data type, you cannot guarantee the order of the elements as
 the ``for`` loop processes.
    
-To test for membership in a set the ``in`` statment can be combined
+To test for membership in a set the ``in`` statement can be combined
 with an ``if`` statement to return a true or false value.  If the
 exact element in the condition is already in the set, the condition
 returns true and the body executes.  The ``in`` statement can also be
@@ -541,7 +541,7 @@ iterate over, say, the directors; we have to iterate with the exact
 format as the keys themselves.  In this case, we need squared brackets
 surrounding four temporary variables to act as a collection for our
 iteration.  While this is a contrived example, we could easily have
-had keys containin IP addresses (``addr``), ports (``port``) and even a ``string``
+had keys containing IP addresses (``addr``), ports (``port``) and even a ``string``
 calculated as the result of a reverse hostname lookup.
 
 .. btest-include:: ${DOC_ROOT}/scripting/data_struct_table_complex.bro
@@ -642,7 +642,7 @@ subnet
 ~~~~~~
 
 Bro has full support for CIDR notation subnets as a base data type. 
-There is no need to manage the IP and the subnet mask as two seperate
+There is no need to manage the IP and the subnet mask as two separate
 entities when you can provide the same information in CIDR notation in
 your scripts.  The following example below uses a Bro script to
 determine if a series of IP addresses are within a set of subnets
@@ -802,7 +802,7 @@ composite type.  We have, in fact, already encountered a a complex
 example of the ``record`` data type in the earlier sections, the
 :bro:type:`connection` record passed to many events. Another one,
 :bro:type:`Conn::Info`, which corresponds to the fields logged into
-``conn.log``, is shown by the exerpt below.
+``conn.log``, is shown by the excerpt below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro
    :lines: 10-12,16,17,19,21,23,25,28,31,35,37,56,62,68,90,93,97,100,104,108,109,114
@@ -813,7 +813,7 @@ definition is within the confines of an export block, what is defined
 is, in fact, ``Conn::Info``.
 
 The formatting for a declaration of a record type in Bro includes the
-descriptive name of the type being defined and the seperate fields
+descriptive name of the type being defined and the separate fields
 that make up the record.  The individual fields that make up the new
 record are not limited in type or number as long as the name for each
 field is unique.
@@ -829,7 +829,7 @@ string, a set of ports, and a count to define a service type.  Also
 included is a function to print each field of a record in a formatted
 fashion and a :bro:id:`bro_init` event handler to show some
 functionality of working with records.  The definitions of the DNS and
-HTTP services are both done inline using squared brackets before being
+HTTP services are both done in-line using squared brackets before being
 passed to the ``print_service`` function.  The ``print_service``
 function makes use of the ``$`` dereference operator to access the
 fields within the newly defined Service record type.  
@@ -846,7 +846,7 @@ record.
    @TEST-EXEC: btest-rst-cmd bro ${DOC_ROOT}/scripting/data_struct_record_02.bro
 
 The example above includes a second record type in which a field is
-used as the data type for a set.  Records can be reapeatedly nested
+used as the data type for a set.  Records can be repeatedly nested
 within other records, their fields reachable through repeated chains
 of the ``$`` dereference operator.  
 
@@ -1123,7 +1123,7 @@ which we will cover shortly.
 +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
 | policy_items        | set[count]                                                       | &log &optional | Policy items that have been applied    |
 +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
-| email_body_sections | vector                                                           | &optinal       | Body of the email for email notices.   |
+| email_body_sections | vector                                                           | &optional       | Body of the email for email notices.   |
 +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
 | email_delay_tokens  | set[string]                                                      | &optional      | Delay functionality for email notices. |
 +---------------------+------------------------------------------------------------------+----------------+----------------------------------------+
@@ -1137,7 +1137,7 @@ has been heuristically detected and the originating hostname is one
 that would raise suspicion.  Effectively, the script attempts to
 define a list of hosts from which you would never want to see SSH
 traffic originating, like DNS servers, mail servers, etc.  To
-accomplish this, the script adhere's to the seperation of detection
+accomplish this, the script adheres to the separation of detection
 and reporting by detecting a behavior and raising a notice.  Whether
 or not that notice is acted upon is decided by the local Notice
 Policy, but the script attempts to supply as much information as
@@ -1221,7 +1221,7 @@ Bro.
 
 In the :doc:`/scripts/policy/protocols/ssl/expiring-certs` script
 which identifies when SSL certificates are set to expire and raises
-notices when it crosses a pre-defined threshold, the call to
+notices when it crosses a predefined threshold, the call to
 ``NOTICE`` above also sets the ``$identifier`` entry by concatenating
 the responder IP, port, and the hash of the certificate.  The
 selection of responder IP, port and certificate hash fits perfectly
@@ -1257,7 +1257,7 @@ In short, there will be notice policy considerations where a broad
 decision can be made based on the ``Notice::Type`` alone.  To
 facilitate these types of decisions, the Notice Framework supports
 Notice Policy shortcuts.  These shortcuts are implemented through the
-means of a group of data structures that map specific, pre-defined
+means of a group of data structures that map specific, predefined
 details and actions to the effective name of a notice.  Primarily
 implemented as a set or table of enumerables of :bro:type:`Notice::Type`,
 Notice Policy shortcuts can be placed as a single directive in your
@@ -1303,5 +1303,3 @@ Notice::emailed_types set while the shortcut below alters the length
 of time for which those notices will be suppressed.
 
 .. btest-include:: ${DOC_ROOT}/scripting/framework_notice_shortcuts_02.bro
-
-

From 8e3c6ada0fc0249fece2864faaa04dfd1b330c2c Mon Sep 17 00:00:00 2001
From: Scott Runnels <srunnels@gmail.com>
Date: Fri, 20 Sep 2013 13:25:49 -0400
Subject: [PATCH 04/17] Rewrite the MHR detection description.

Now that the MHR script uses the file analysis framework, the
description needed to be rewritten to reflect the changes.  Robin
commented that he didn't feel the MHR script was a good introductory
script and he might be right, however, I couldn't find one that was
easier to explain.
---
 doc/scripting/index.rst | 95 ++++++++++++++++++++++-------------------
 1 file changed, 51 insertions(+), 44 deletions(-)

diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst
index b01165a5bc..077d5a9c45 100644
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@@ -10,13 +10,6 @@ Writing Bro Scripts
 Understanding Bro Scripts
 =========================
 
-.. todo::
-
-    The MHR integration has changed significantly since the text was
-    written. We need to update it, however I'm actually not sure this
-    script is a good introductory example anymore unfortunately.
-    -Robin
-
 Bro includes an event-driven scripting language that provides
 the primary means for an organization to extend and customize Bro's
 functionality. Virtually all of the output generated by Bro
@@ -51,82 +44,96 @@ appropriate DNS lookup and parsing the response.
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
 
 Visually, there are three distinct sections of the script.  A base
-level with no indentation followed by an indented and formatted
-section explaining the custom variables being provided (``export``) and another
-indented and formatted section describing the instructions for a
-specific event (``event log_http``).  Don't get discouraged if you don't
+level with no indentation where libraries are included in the script through ``@load``
+and a namespace is defined with ``module``.  This is followed by an indented and formatted
+section explaining the custom variables being provided (``export``) as part of the script's namespace.
+Finally there is a second indented and formatted section describing the instructions to take for a
+specific event (``event file_hash``).  Don't get discouraged if you don't
 understand every section of the script; we'll cover the basics of the
 script and much more in following sections.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
-   :lines: 7-11
+   :lines: 4-6
 
 Lines 7 and 8 of the script process the ``__load__.bro`` script in the
 respective directories being loaded.  The ``@load`` directives are
 often considered good practice or even just good manners when writing
-Bro scripts to make sure they can be
-used on their own. While it's unlikely that in a
+Bro scripts to make sure they can be used on their own. While it's unlikely that in a
 full production deployment of Bro these additional resources wouldn't
 already be loaded, it's not a bad habit to try to get into as you get
 more experienced with Bro scripting.  If you're just starting out,
-this level of granularity might not be entirely necessary though.  
+this level of granularity might not be entirely necessary.  The ``@load`` directives
+are ensuring the Files framework, the Notice framework and the script to hash all files has
+been loaded by Bro.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
-   :lines: 12-24
+   :lines: 10-31
 
 The export section redefines an enumerable constant that describes the
-type of notice we will generate with the logging framework.  Bro
+type of notice we will generate with the Notice framework.  Bro
 allows for re-definable constants, which at first, might seem
 counter-intuitive.  We'll get more in-depth with constants in a later
 chapter, for now, think of them as variables that can only be altered
 before Bro starts running.  The notice type listed allows for the use
 of the :bro:id:`NOTICE` function to generate notices of type
-``Malware_Hash_Registry_Match`` as done in the next section.  Notices
+``TeamCymruMalwareHashRegistry::Match`` as done in the next section.  Notices
 allow Bro to generate some kind of extra notification beyond its
 default log types.  Often times, this extra notification comes in the
-form of an email generated and sent to a preconfigured address.  
+form of an email generated and sent to a preconfigured address, but can be altered
+depending on the needs of the deployment.  The export section is finished off with
+the definition of two constants that list the kind of files we want to match against and
+the minimum percentage of detection threshold in which we are interested.
+
+Up until this point, the script has merely done some basic setup.  With the next section,
+the script starts to define instructions to take in a given event.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
-   :lines: 26-44
+   :lines: 33-57
 
 The workhorse of the script is contained in the event handler for
-``log_http``.  The ``log_http`` event is defined as an event-hook in
-the :doc:`/scripts/base/protocols/http/main` script and allows scripts
-to handle a connection as it is being passed to the logging framework.
-The event handler is passed an :bro:type:`HTTP::Info` data structure
-which will be referred to as ``rec`` in body of the event handler.
+``file_hash``.  The ``file_hash`` event is defined in the
+:doc:`/scripts/base/bif/plugins/Bro_FileHash.events.bif.bro` script and allows scripts to access
+the information associated with a file for which Bro's file analysis framework has
+generated a hash.  The event handler is passed the file itself as ``f``, the type of digest
+algorithm used as ``kind`` and the hash generated as ``hash``.
 
-An ``if`` statement is used to check for the existence of a data structure
-named ``md5`` nested within the ``rec`` data structure.  Bro uses the ``$`` as
-a deference operator and as such, and it is employed in this script to
-check if ``rec$md5`` is present by including the ``?`` operator within the
-path.  If the ``rec`` data structure includes a nested data structure
-named ``md5``, the statement is processed as true and a local variable
-named ``hash_domain`` is provisioned and given a format string based on
-the contents of ``rec$md5`` to produce a valid DNS lookup.  
+On line 35, an ``if`` statement is used to check for the correct type of hash, in this case
+a SHA1 hash.  It also checks for a mime type we've defined as being of interest as defined in the
+constant ``match_file_types``.  The comparison is made against the variable ``f$mime_type`` which uses
+the ``$`` dereference operator to check the value ``mime_type`` inside the variable ``f``.  Once both
+values resolve to true, a local variable is defined to hold a string comprised of the SHA1 hash concatenated
+with ".malware.hash.cymru.com"; this value will be the domain queried in the malware hash registry.
 
 The rest of the script is contained within a ``when`` block.  In
 short, a ``when`` block is used when Bro needs to perform asynchronous
-actions, such a DNS lookup, to ensure that performance isn't effected.
+actions, such as a DNS lookup, to ensure that performance isn't effected.
 The ``when`` block performs a DNS TXT lookup and stores the result
 in the local variable ``MHR_result``.  Effectively, processing for
 this event continues and upon receipt of the values returned by
 :bro:id:`lookup_hostname_txt`, the ``when`` block is executed.  The
-``when`` block splits the string returned into two separate values and
-checks to ensure an expected format.  If the format is invalid, the
-script assumes that the hash wasn't found in the repository and
-processing is concluded.  If the format is as expected and the
-detection rate is above the threshold set by ``MHR_threshold``, two
-new local variables are created and used in the notice issued by
-:bro:id:`NOTICE`.
+``when`` block splits the string returned into a portion for the date on which
+the malware was first detected and the detection rate by splitting on an text space
+and storing the values returned in a local table variable.  In line 42, if the table
+returned by ``split1`` has two entries, indicating a sucessful split, we store the detection
+date in ``mhr_first_detect`` and the rate in ``mhr_detect_rate`` on lines 45 and 45 respectively
+using the appropriate conversion functions.  From this point on, Bro knows it has seen a file
+transmitted which has a hash that has been seen by the Team Cymru Malware Hash Registry, the rest
+of the script is dedicated to producing a notice.
 
-In approximately 15 lines of actual code, Bro provides an amazing
+On line 47, the detection time is processed into a string representation and stored in
+``readable_first_detected``.  The script then compares the detection rate against the
+``notice_threshold`` that was defined on line 30.  If the detection rate is high enough, the script
+creates a concise description of the notice on line 50, a possible URL to check the sample against
+virustotal.com's database, and makes the call to :bro:id:`NOTICE` to hand the relevant information
+off to the Notice framework.
+
+In approximately 25 lines of code, Bro provides an amazing
 utility that would be incredibly difficult to implement and deploy
-with other products.  In truth, claiming that Bro does this in 15
+with other products.  In truth, claiming that Bro does this in 25
 lines is a misdirection; there is a truly massive number of things
 going on behind-the-scenes in Bro, but it is the inclusion of the
 scripting language that gives analysts access to those underlying
-layers in a succinct and well defined manner.  
+layers in a succinct and well defined manner.
 
 The Event Queue and Event Handlers
 ==================================

From 89090ec34af8e2bba63d069fd077825080962103 Mon Sep 17 00:00:00 2001
From: Scott Runnels <srunnels@gmail.com>
Date: Fri, 20 Sep 2013 13:33:44 -0400
Subject: [PATCH 05/17] Include a better description for detect-MHR.bro

I added a better more concise and accurate description of what is going
on behind the scenes of detect-MHR.bro to not only bring it into line
with the Files framework but to help make it a bit more clear as to
where the various responsibilities lie.
---
 doc/scripting/index.rst | 21 ++++++++++-----------
 1 file changed, 10 insertions(+), 11 deletions(-)

diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst
index 077d5a9c45..aeaeac1726 100644
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@@ -29,17 +29,16 @@ about the information about that connection.
 It's often easiest to understand Bro's scripting language by
 looking at a complete script and breaking it down into its
 identifiable components.  In this example, we'll take a look at how
-Bro queries the `Team Cymru Malware hash registry
-<http://www.team-cymru.org/Services/MHR/>`_ for downloads via
-HTTP. Part of the Team Cymru Malware Hash registry includes the
-ability to do a host lookup on a domain with the format
-``MALWARE_HASH.malware.hash.cymru.com`` where ``MALWARE_HASH`` is the MD5 or
-SHA1 hash of a file.  Team Cymru also populates the TXT record of
-their DNS responses with both a "last seen" timestamp and a numerical
-"detection rate".  The important aspect to understand is Bro already
-generates hashes for files it can parse from HTTP streams, but the
-script ``detect-MHR.bro`` is responsible for generating the
-appropriate DNS lookup and parsing the response.
+Bro checks the SHA1 hash of various files extracted from network traffic
+against the `Team Cymru Malware hash registry
+<http://www.team-cymru.org/Services/MHR/>`_.  Part of the Team Cymru Malware
+Hash registry includes the ability to do a host lookup on a domain with the format
+``MALWARE_HASH.malware.hash.cymru.com`` where ``MALWARE_HASH`` is the SHA1 hash of a file.
+Team Cymru also populates the TXT record of their DNS responses with both a "first seen"
+timestamp and a numerical "detection rate".  The important aspect to understand is Bro already
+generating hashes for files via the Files framework, but it is the
+script ``detect-MHR.bro`` that is responsible for generating the
+appropriate DNS lookup, parsing the response, and generating a notice if appropriate.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/frameworks/files/detect-MHR.bro
 

From 261b9e1e9747631bd5b0ae8f4bbfcc3b4cda9f0a Mon Sep 17 00:00:00 2001
From: Scott Runnels <srunnels@gmail.com>
Date: Fri, 20 Sep 2013 13:36:56 -0400
Subject: [PATCH 06/17] Spelling corrections.

---
 doc/scripting/index.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/doc/scripting/index.rst b/doc/scripting/index.rst
index aeaeac1726..7c484af7e9 100644
--- a/doc/scripting/index.rst
+++ b/doc/scripting/index.rst
@@ -113,7 +113,7 @@ this event continues and upon receipt of the values returned by
 ``when`` block splits the string returned into a portion for the date on which
 the malware was first detected and the detection rate by splitting on an text space
 and storing the values returned in a local table variable.  In line 42, if the table
-returned by ``split1`` has two entries, indicating a sucessful split, we store the detection
+returned by ``split1`` has two entries, indicating a successful split, we store the detection
 date in ``mhr_first_detect`` and the rate in ``mhr_detect_rate`` on lines 45 and 45 respectively
 using the appropriate conversion functions.  From this point on, Bro knows it has seen a file
 transmitted which has a hash that has been seen by the Team Cymru Malware Hash Registry, the rest

From d070b018286b4d78e6968066f390a870250c68cb Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Sep 2013 14:38:15 -0700
Subject: [PATCH 07/17] Updating baselines for doc changes.

---
 CHANGES                                       |  2 +-
 VERSION                                       |  2 +-
 .../output                                    | 32 ++++++++-----------
 .../output                                    |  8 ++---
 .../output                                    |  9 ++++++
 .../output                                    | 20 ++++++++----
 6 files changed, 41 insertions(+), 32 deletions(-)

diff --git a/CHANGES b/CHANGES
index aca5882f6c..9ff7b2368c 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,5 @@
 
-2.1-1376 | 2013-09-20 14:18:30 -0700
+2.1-1377 | 2013-09-20 14:38:15 -0700
 
   * Updates to the scripting introduction. (Scott Runnels)
 
diff --git a/VERSION b/VERSION
index 174fa556a3..326b107a34 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-1376
+2.1-1377
diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output b/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output
index fc14c86399..0d981e7fc0 100644
--- a/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output
+++ b/testing/btest/Baseline/doc.sphinx.include-scripts_base_bif_event_bif_bro/output
@@ -2,24 +2,20 @@
 
 -- event.bif.bro
 
+## Generated for every new connection. This event is raised with the first
+## packet of a previously unknown connection. Bro uses a flow-based definition
+## of "connection" here that includes not only TCP sessions but also UDP and
+## ICMP flows.
+global new_connection: event(c: connection );
+## Generated when a TCP connection timed out. This event is raised when
+## no activity was seen for an interval of at least
+## :bro:id:`tcp_connection_linger`, and either one endpoint has already
+## closed the connection or one side never became active.
+global connection_timeout: event(c: connection );
+## Generated when a connection's internal state is about to be removed from
+## memory. Bro generates this event reliably once for every connection when it
+## is about to delete the internal state. As such, the event is well-suited for
 ## script-level cleanup that needs to be performed for every connection.  This
 ## event is generated not only for TCP sessions but also for UDP and ICMP
 ## flows.
-##
-##
-global connection_external: event(c: connection , tag: string );
-
-
-## Generated when a UDP session for a supported protocol has finished. Some of
-## Bro's application-layer UDP analyzers flag the end of a session by raising
-## Generated when a connection is seen that is marked as being expected.
-
-
-global ipv6_ext_headers: event(c: connection , p: pkt_hdr );
-##    their specifics differ slightly. Often, however, both will be raised for
-##    the same connection if some of its data is missing. We should eventually
-##    merge the two.
-global ack_above_hole: event(c: connection );
-
-
-##
+global connection_state_remove: event(c: connection );
diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output
index 8d26caba6c..656abf3ba1 100644
--- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output
+++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@2/output
@@ -2,8 +2,6 @@
 
 -- detect-MHR.bro
 
-
-module TeamCymruMalwareHashRegistry;
-
-export {
-	redef enum Notice::Type += {
+@load base/frameworks/files
+@load base/frameworks/notice
+@load frameworks/files/hash-all-files
diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output
index ace6e79c5e..75d94f6990 100644
--- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output
+++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@3/output
@@ -2,6 +2,8 @@
 
 -- detect-MHR.bro
 
+export {
+	redef enum Notice::Type += {
 		## The hash value of a file transferred over HTTP matched in the
 		## malware hash registry.
 		Match
@@ -15,3 +17,10 @@
 	                         /application\/x-java-applet/ |
 	                         /application\/jar/ |
 	                         /video\/mp4/ &redef;
+
+	## The malware hash registry runs each malware sample through several A/V engines.
+	## Team Cymru returns a percentage to indicate how many A/V engines flagged the
+	## sample as malicious. This threshold allows you to require a minimum detection
+	## rate.
+	const notice_threshold = 10 &redef;
+}
diff --git a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output
index 6f4cda878c..7e3e1b0a11 100644
--- a/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output
+++ b/testing/btest/Baseline/doc.sphinx.include-scripts_policy_frameworks_files_detect-MHR_bro@4/output
@@ -2,13 +2,6 @@
 
 -- detect-MHR.bro
 
-	## The malware hash registry runs each malware sample through several A/V engines.
-	## Team Cymru returns a percentage to indicate how many A/V engines flagged the
-	## sample as malicious. This threshold allows you to require a minimum detection
-	## rate.
-	const notice_threshold = 10 &redef;
-}
-
 event file_hash(f: fa_file, kind: string, hash: string)
 	{
 	if ( kind=="sha1" && match_file_types in f$mime_type )
@@ -21,3 +14,16 @@ event file_hash(f: fa_file, kind: string, hash: string)
 			if ( |MHR_answer| == 2 )
 				{
 				local mhr_first_detected = double_to_time(to_double(MHR_answer[1]));
+				local mhr_detect_rate = to_count(MHR_answer[2]);
+
+				local readable_first_detected = strftime("%Y-%m-%d %H:%M:%S", mhr_first_detected);
+				if ( mhr_detect_rate >= notice_threshold )
+					{
+					local message = fmt("Malware Hash Registry Detection rate: %d%%  Last seen: %s", mhr_detect_rate, readable_first_detected);
+					local virustotal_url = fmt("https://www.virustotal.com/en/file/%s/analysis/", hash);
+					NOTICE([$note=Match, $msg=message, $sub=virustotal_url, $f=f]);
+					}
+				}
+			}
+		}
+	}

From 8d7b9f0627b080e7cadaf0b276af219a8e3b951d Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Sep 2013 14:52:58 -0700
Subject: [PATCH 08/17] Updating submodule(s).

 [nomail]
---
 aux/binpac   | 2 +-
 aux/bro-aux  | 2 +-
 aux/broccoli | 2 +-
 aux/broctl   | 2 +-
 aux/btest    | 2 +-
 5 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/aux/binpac b/aux/binpac
index eeb19daacc..c066a3ecbe 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit eeb19daacc9f12bc4e7c885fa70e71f856a90b1f
+Subproject commit c066a3ecbe733916a1b0902d1f8e1a6a95e2a734
diff --git a/aux/bro-aux b/aux/bro-aux
index eb24e62864..ca8db41e05 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit eb24e628648c7d7b931bdb57d38ab32c28296e72
+Subproject commit ca8db41e05e3198e4d6fccb00b1a56acbdb22d71
diff --git a/aux/broccoli b/aux/broccoli
index 5bcee43070..dc596253a5 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 5bcee430700f714b19a9e794de75cb42408c9ecf
+Subproject commit dc596253a5cb0dd3be34d915536c4ecb8f1db138
diff --git a/aux/broctl b/aux/broctl
index d842372655..dbd2d48454 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit d84237265557779f02d65ac65aa5e122b35391b6
+Subproject commit dbd2d48454c924d5a0c0d926c5ad4d444bb2b03f
diff --git a/aux/btest b/aux/btest
index 3918bd9f5f..6e940b7315 160000
--- a/aux/btest
+++ b/aux/btest
@@ -1 +1 @@
-Subproject commit 3918bd9f5f99863faec2501e5bc7839ffb17bdc9
+Subproject commit 6e940b73152a14ae63a4405f6a4bc23cf6cbeec1

From c7fe809ee5de6aa3364e714f7554be1e2e42a071 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Sep 2013 14:55:29 -0700
Subject: [PATCH 09/17] Updating submodule(s).

 [nomail]
---
 aux/bro-aux | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/bro-aux b/aux/bro-aux
index ca8db41e05..11d346ba99 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit ca8db41e05e3198e4d6fccb00b1a56acbdb22d71
+Subproject commit 11d346ba992b17f0106271b28da5183937819e22

From 7c1bead09a7ed673d702c9ffe17e1d140d6a6c11 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Sep 2013 15:10:52 -0700
Subject: [PATCH 10/17] Updating submodule(s).

 [nomail]
---
 aux/broctl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/broctl b/aux/broctl
index dbd2d48454..5c6052bf5e 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit dbd2d48454c924d5a0c0d926c5ad4d444bb2b03f
+Subproject commit 5c6052bf5e2340410fa579dbc6d531a4febf98d9

From b8504a565404ac90f21ea9ae267857ab8cf7e178 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Fri, 20 Sep 2013 15:37:37 -0700
Subject: [PATCH 11/17] Updating submodule(s).

 [nomail]
---
 aux/bro-aux | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/aux/bro-aux b/aux/bro-aux
index 11d346ba99..045e25e660 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 11d346ba992b17f0106271b28da5183937819e22
+Subproject commit 045e25e660003ea61b5e4ca7d909870d08ea553f

From 2049025288dfb3652e96d5bf85d2decc92778841 Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@illinois.edu>
Date: Mon, 23 Sep 2013 13:16:13 -0500
Subject: [PATCH 12/17] Change submodules to fixed URL.

---
 .gitmodules  | 16 ++++++++--------
 aux/binpac   |  2 +-
 aux/bro-aux  |  2 +-
 aux/broccoli |  2 +-
 aux/broctl   |  2 +-
 5 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/.gitmodules b/.gitmodules
index afac9bfc6b..87826d2ef6 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,24 +1,24 @@
 [submodule "aux/bro-aux"]
 	path = aux/bro-aux
-	url = ../bro-aux
+	url = git://git.bro.org/bro-aux
 [submodule "aux/binpac"]
 	path = aux/binpac
-	url = ../binpac
+	url = git://git.bro.org/binpac
 [submodule "aux/broccoli"]
 	path = aux/broccoli
-	url = ../broccoli
+	url = git://git.bro.org/broccoli
 [submodule "aux/broctl"]
 	path = aux/broctl
-	url = ../broctl
+	url = git://git.bro.org/broctl
 [submodule "aux/btest"]
 	path = aux/btest
-	url = ../btest
+	url = git://git.bro.org/btest
 [submodule "cmake"]
 	path = cmake
-	url = ../cmake
+	url = git://git.bro.org/cmake
 [submodule "magic"]
 	path = magic
-	url = ../bromagic
+	url = git://git.bro.org/bromagic
 [submodule "src/3rdparty"]
 	path = src/3rdparty
-	url = ../bro-3rdparty
+	url = git://git.bro.org/bro-3rdparty
diff --git a/aux/binpac b/aux/binpac
index c066a3ecbe..5f9d4afe82 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit c066a3ecbe733916a1b0902d1f8e1a6a95e2a734
+Subproject commit 5f9d4afe8274b1c1125757ca0e971f66d6d35384
diff --git a/aux/bro-aux b/aux/bro-aux
index 045e25e660..387e26609a 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 045e25e660003ea61b5e4ca7d909870d08ea553f
+Subproject commit 387e26609a923f5abd10be593ae127690f7a6b72
diff --git a/aux/broccoli b/aux/broccoli
index dc596253a5..baf003ac16 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit dc596253a5cb0dd3be34d915536c4ecb8f1db138
+Subproject commit baf003ac1648f63cd871942b20f8da82184af784
diff --git a/aux/broctl b/aux/broctl
index 5c6052bf5e..629e155c71 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 5c6052bf5e2340410fa579dbc6d531a4febf98d9
+Subproject commit 629e155c712deea170330842ad7bfa9ae666f325

From 09b7ccba231c857681dd961d5b8f8b4a2bfb4d18 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 23 Sep 2013 11:48:38 -0700
Subject: [PATCH 13/17] Fix required for compiling with clang 3.3.

---
 src/Expr.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Expr.cc b/src/Expr.cc
index e64172675e..0eaa7ce918 100644
--- a/src/Expr.cc
+++ b/src/Expr.cc
@@ -2403,7 +2403,7 @@ RefExpr::RefExpr(Expr* arg_op) : UnaryExpr(EXPR_REF, arg_op)
 	if ( IsError() )
 		return;
 
-	if ( ! is_assignable(op->Type()) )
+	if ( ! ::is_assignable(op->Type()) )
 		ExprError("illegal assignment target");
 	else
 		SetType(op->Type()->Ref());

From 601cdbc6cfa75fe46cbf67c4923b1988dae43ee0 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 23 Sep 2013 11:48:56 -0700
Subject: [PATCH 14/17] Fixing an always false condition.

Reported by clang.
---
 src/logging/Manager.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc
index 79e23fb63f..0de41fb642 100644
--- a/src/logging/Manager.cc
+++ b/src/logging/Manager.cc
@@ -792,7 +792,7 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 			if ( ! v )
 				return false;
 
-			if ( ! v->Type()->Tag() == TYPE_STRING )
+			if ( v->Type()->Tag() != TYPE_STRING )
 				{
 				reporter->Error("path_func did not return string");
 				Unref(v);

From ed2798ae73f021d5af0ee925b98956766138fd2b Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 23 Sep 2013 11:49:22 -0700
Subject: [PATCH 15/17] Updating NEWS.

---
 NEWS | 334 ++++++++++++++++++++++++++++++++++++-----------------------
 1 file changed, 207 insertions(+), 127 deletions(-)

diff --git a/NEWS b/NEWS
index dba45c279d..e192c52380 100644
--- a/NEWS
+++ b/NEWS
@@ -1,53 +1,147 @@
 
 This document summarizes the most important changes in the current Bro
-release. For a complete list of changes, see the ``CHANGES`` file
+release. For an exhaustive list of changes, see the ``CHANGES`` file
 (note that submodules, such as BroControl and Broccoli, come with
-their own CHANGES.)
+their own ``CHANGES``.)
 
-Bro 2.2 (Work In Progress)
-==========================
+Bro 2.2 Beta
+============
 
 New Functionality
 -----------------
 
+- A new file analysis framework moves most of the processing of file
+  content from script-land into the core, where it belongs. See
+  ``doc/file-analysis.rst``, or the online documentation, for more
+  information.
+
+  Much of this is an internal change, but the framework also comes
+  with the following user-visible functionality (some of that was
+  already available before but is done differently, and more
+  efficiently, now):
+
+      - HTTP:
+        * Identify MIME type of messages.
+        * Extract messages to disk.
+        * Compute MD5 for messages.
+
+      - SMTP:
+        * Identify MIME type of messages.
+        * Extract messages to disk.
+        * Compute MD5 for messages.
+        * Provide access to start of entity data.
+
+      - FTP data transfers:
+        * Identify MIME types of data.
+        * Record to disk.
+
+      - IRC DCC transfers: Record to disk.
+
+      - Support for analyzing data transfered via HTTP range requests.
+
+      - A binary input reader interfaces the input framework with the
+        file analysis, allowing to inject files on disk into Bro's
+        content processing.
+
+- A new framework for computing a wide array of summary statistics,
+  such as counters and thresholds checks, standard deviation and mean,
+  set cardinality, top K, and more. The framework operates in
+  real-time, independent of the underlying data, and can aggregate
+  information from many independent monitoring points (including
+  clusters). It provides a transparent, easy-to-use user interface,
+  and can optionally deploy a set of probabilistic data structures for
+  memory-efficient operation. The framework is located in 
+  ``scripts/base/frameworks/sumstats``.
+
+  A number of new applications now ship with Bro that are built on top
+  of the summary statistics framework:
+
+    * Scan detection: Detectors for port and address scans. See
+      ``policy/misc/scan.bro`` (these scan detectors used to exist in
+      Bro versions <2.0; it's now back, but quite different).
+
+    * Tracerouter detector: ``policy/misc/detect-traceroute.bro``
+      
+    * Web application detection/measurement:
+      ``policy/misc/app-stats/*``
+
+    * FTP and SSH brute-forcing detector:
+      ``policy/protocols/ftp/detect-bruteforcing.bro``,
+      ``policy/protocols/ssh/detect-bruteforcing.bro``
+
+    * HTTP-based SQL injection detector:
+      ``policy/protocols/http/detect-sqli.bro`` (existed before, but
+      now ported to the new framework)
+
+- GridFTP support. This is an extension to the standard FTP analyzer
+  and includes:
+
+      - An analyzer for the GSI mechanism of GSSAPI FTP AUTH method.
+        GSI authentication involves an encoded TLS/SSL handshake over
+        the FTP control session. For FTP sessions that attempt GSI
+        authentication, the ``service`` field of the connection log
+        will include ``gridftp`` (as well as also ``ftp`` and
+        ``ssl``).
+
+      - An example of a GridFTP data channel detection script. It
+        relies on the heuristics of GridFTP data channels commonly
+        default to SSL mutual authentication with a NULL bulk cipher
+        and that they usually transfer large datasets (default
+        threshold of script is 1 GB). For identified GridFTP data
+        channels, the ``services`` fields of the connection log will
+        include ``gridftp-data``.
+
+- Modbus and DNP3 support. Script-level support is only basic at this
+  point but see ``src/analyzer/protocol/{modbus,dnp3}/events.bif``, or
+  the online documentation, for the events Bro generates. For Modbus,
+  there are also some example policies in
+  ``policy/protocols/modbus/*``.
+
+- The documentation now includes a new introduction to writing Bro
+  scripts. See ``doc/scripting/index.rst`` or, much better, the online
+  version. There's also the beginning of a chapter on "Using Bro" in
+  ``doc/using/index.rst``.
+
 - GPRS Tunnelling Protocol (GTPv1) decapsulation.
 
-- GridFTP support. TODO: Extend.
+- The scripting language now provide "hooks", a new flavor of
+  functions that share characteristics of both standard functions and
+  events. They are like events in that multiple bodies can be defined
+  for the same hook identifier. They are more like functions in the
+  way they are invoked/called, because, unlike events, their execution
+  is immediate and they do not get scheduled through an event queue.
+  Also, a unique feature of a hook is that a given hook handler body
+  can short-circuit the execution of remaining hook handlers simply by
+  exiting from the body as a result of a ``break`` statement (as
+  opposed to a ``return`` or just reaching the end of the body). See
+  ``doc/scripts/builtins.rst``, or the online documentation, for more
+  informatin.
 
-- Modbus support. TODO: Extend.
+- Bro's language now has a working ``switch`` statement that generally
+  behaves like C-style switches (except that case labels can be
+  comprised of multiple literal constants delimited by commas).  Only
+  atomic types are allowed for now.  Case label bodies that don't
+  execute a ``return`` or ``break`` statement will fall through to
+  subsequent cases. A ``default`` case label is supported.
 
-- DNP3 support. TODO: Extend.
-
-- ssl.log now also records the subject client and issuer certificates.
-
-- Hooks: TODO: Briefly summarize the documention from
-  doc/scripts/builtins.rst here.
-
-- The ASCII writer can now output CSV files on a per filter basis.
-
-- Bro's language now has a working "switch" statement that generally
-  behaves like C-style switches except case labels can be comprised of
-  multiple literal constants delimited by commas.  Only atomic types
-  are allowed for now.  Case label bodies that don't execute a
-  "return" or "break" statement will fall through to subsequent cases.
-  A default case label is allowed.
-
-- Bro's language now has a new set of types "opaque of X". Opaque
+- Bro's language now has a new set of types ``opaque of X``. Opaque
   values can be passed around like other values but they can only be
   manipulated with BiF functions, not with other operators. Currently,
-  the following opaque types are supported:
+  the following opaque types are supported::
 
-        - opaque of md5
-        - opaque of sha1
-        - opaque of sha256
-        - opaquey of entropy.
+        opaque of md5
+        opaque of sha1
+        opaque of sha256
+        opaque of cardinality
+        opaque of topk
+        opaque of bloomfilter
 
-  They go along with the corrsponding BiF functions md5_*, sha1_*,
-  sha256_*, and entropy_*, respectively. Note that these functions
-  have changed their signatures to work with opaques types rather
-  than global state as it was before.
+  These go along with the corrsponding BiF functions ``md5_*``,
+  ``sha1_*``, ``sha256_*``, ``entropy_*``, etc. . Note that where
+  these functions existed before, they have changed their signatures
+  to work with opaques types rather than global state.
 
-- The scripting language now supports a constructing sets, tables,
+- The scripting language now supports constructing sets, tables,
   vectors, and records by name::
 
         type MyRecordType: record {
@@ -61,57 +155,33 @@ New Functionality
         global s = MySet([$c=1], [$c=2]);
 
 - Strings now support the subscript operator to extract individual
-  characters and substrings (e.g., s[4], s[1,5]). The index expression
-  can take up to two indices for the start and end index of the
-  substring to return (e.g. "mystring[1,3]").
+  characters and substrings (e.g., ``s[4]``, ``s[1,5]``). The index
+  expression can take up to two indices for the start and end index of
+  the substring to return (e.g. ``mystring[1,3]``).
 
-- Functions now support default parameters, e.g.:
+- Functions now support default parameters, e.g.::
 
-        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+      global foo: function(s: string, t: string &default="abc", u: count &default=0);
 
-- Scripts can now use two new "magic constants" @DIR and @FILENAME
-  that expand to the directory path of the current script and just the
-  script file name without path, respectively. (Jon Siwek)
+- Scripts can now use two new "magic constants" ``@DIR`` and
+  ``@FILENAME`` that expand to the directory path of the current
+  script and just the script file name without path, respectively.
 
-- The new file analysis framework moves most of the processing of file
-  content from script-land into the core, where it belongs. See
-  doc/file-analysis.rst for more information.
+- ``ssl.log`` now also records the subject client and issuer
+  certificates.
 
-  Much of this is an internal change, but the framework also comes
-  with the following user-visibible functionality (some of that was
-  already available before, but done differently):
+- The ASCII writer can now output CSV files on a per filter basis.
 
-  [TODO: Update with changes from 984e9793db56.]
+- New SQLite reader and writer plugins for the logging framework allow
+  to read/write persistent data from on disk SQLite databases.
 
-      - A binary input reader interfaces the input framework with file
-        analysis, allowing to inject files on disk into Bro's
-        processing.
-
-      - Supports for analyzing data transfereed via HTTP range
-        requests.
-
-      - HTTP:
-        * Identify MIME type of message.
-        * Extract message to disk.
-        * Compute MD5 for messages.
-
-      - SMTP:
-        * Identify MIME type of message.
-        * Extract message to disk.
-        * Compute MD5 for messages.
-        * Provide access to start of entity data.
-
-      - FTP data transfers: Identify MIME type; record to disk.
-
-      - IRC DCC transfers: Record to disk.
-
-- New packet filter framework supports BPF-based load-balancing,
+- A new packet filter framework supports BPF-based load-balancing,
   shunting, and sampling; plus plugin support to customize filters
   dynamically.
 
 - Bro now provides Bloom filters of two kinds: basic Bloom filters
   supporting membership tests, and counting Bloom filters that track
-  the frequency of elements. The corresponding functions are:
+  the frequency of elements. The corresponding functions are::
 
     bloomfilter_basic_init(fp: double, capacity: count, name: string &default=""): opaque of bloomfilter
     bloomfilter_basic_init2(k: count, cells: count, name: string &default=""): opaque of bloomfilter
@@ -121,10 +191,11 @@ New Functionality
     bloomfilter_merge(bf1: opaque of bloomfilter, bf2: opaque of bloomfilter): opaque of bloomfilter
     bloomfilter_clear(bf: opaque of bloomfilter)
 
-  See <INSERT LINK> for full documentation.
+  See ``src/probabilistic/bloom-filter.bif``, or the online
+  documentation, for full documentation.
 
 - Bro now provides a probabilistic data structure for computing
-  "top k" elements. The corresponding functions are:
+  "top k" elements. The corresponding functions are::
 
     topk_init(size: count): opaque of topk
     topk_add(handle: opaque of topk, value: any)
@@ -136,73 +207,82 @@ New Functionality
     topk_merge(handle1: opaque of topk, handle2: opaque of topk)
     topk_merge_prune(handle1: opaque of topk, handle2: opaque of topk)
 
-  See <INSERT LINK> for full documentation.
+  See ``src/probabilistic/top-k.bif``, or the online documentation,
+  for full documentation.
 
-- base/utils/exec.bro provides a module to start external processes
-  asynchronously and retrieve their output on termination.
-  base/utils/dir.bro uses it to monitor a directory for changes, and
-  base/utils/active-http.bro for providing an interface for querying
-  remote web servers.
+- Bro now provides a probabilistic data structure for computing set
+  cardinality, using the HyperLogLog algorithm.  The corresponding
+  functions are::
 
-- Summary statistics framework. [Extend]
+    hll_cardinality_init(err: double, confidence: double): opaque of cardinality
+    hll_cardinality_add(handle: opaque of cardinality, elem: any): bool
+    hll_cardinality_merge_into(handle1: opaque of cardinality, handle2: opaque of cardinality): bool
+    hll_cardinality_estimate(handle: opaque of cardinality): double
+    hll_cardinality_copy(handle: opaque of cardinality): opaque of cardinality
 
-- A number of new applications build on top of the summary statistics
-  framework:
+  See ``src/probabilistic/cardinality-counter.bif``, or the online
+  documentation, for full documentation.
 
-    * Scan detection: Detectors for port and address scans return. See
-      policy/misc/scan.bro.
+- ``base/utils/exec.bro`` provides a module to start external
+  processes asynchronously and retrieve their output on termination.
+  ``base/utils/dir.bro`` uses it to monitor a directory for changes,
+  and ``base/utils/active-http.bro`` for providing an interface for
+  querying remote web servers.
 
-    * Tracerouter detector: policy/misc/detect-traceroute
+- BroControl can now pin Bro processes to CPUs on supported platforms:
+  To use CPU pinning, a new per-node option ``pin_cpus`` can be
+  specified in node.cfg if the OS is either Linux or FreeBSD.
 
-    * Web application detection/measurement: policy/misc/app-metrics.bro
+- BroControl comes with its own test-suite now. ``make test`` in
+  ``aux/broctl`` will run it.
 
-    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
-
-    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
-      (existed before, but now ported to the new framework)
-
-    * SSH brute-forcing detector feeding the intelligence framework:
-      policy/protocols/ssh/detect-bruteforcing.bro
+In addition to these, Bro 2.2 comes with a large set of smaller
+extensions, tweaks, and fixes across the whole code base, including
+most submodules.
 
 Changed Functionality
 ---------------------
 
-- We removed the following, already deprecated, functionality:
+- The interface to extracting content from application-layer protocols
+  (including HTTP, SMTP, FTP) has changed significantly due to the
+  introduction of the new file analysis framework (see above).
+
+- Removed the following, already deprecated, functionality:
 
     * Scripting language:
-        - &disable_print_hook attribute.
+        - ``&disable_print_hook attribute``.
 
     * BiF functions:
-        - parse_dotted_addr(), dump_config(),
-          make_connection_persistent(), generate_idmef(),
-          split_complete()
+        - ``parse_dotted_addr()``, ``dump_config()``,
+          ``make_connection_persistent()``, ``generate_idmef()``,
+          ``split_complete()``
 
-        - md5_*, sha1_*, sha256_*, and entropy_* have all changed
-          their signatures to work with opaque types (see above).
+        - ``md5_*``, ``sha1_*``, ``sha256_*``, and ``entropy_*`` have
+          all changed their signatures to work with opaque types (see
+          above).
 
+- Removed a now unused argument from ``do_split`` helper function.
 
-- Removed a now unused argument from "do_split" helper function.
+- ``this`` is no longer a reserved keyword.
 
-- "this" is no longer a reserved keyword.
-
-- The Input Framework's update_finished event has been renamed to
-  end_of_data. It will now not only fire after table-reads have been
-  completed, but also after the last event of a whole-file-read (or
-  whole-db-read, etc.).
+- The Input Framework's ``update_finished`` event has been renamed to
+  ``end_of_data``. It will now not only fire after table-reads have
+  been completed, but also after the last event of a whole-file-read
+  (or whole-db-read, etc.).
 
 - Renamed the option defining the frequency of alarm summary mails to
-  'Logging::default_alarm_mail_interval'. When using BroControl, the
+  ``Logging::default_alarm_mail_interval``. When using BroControl, the
   value can now be set with the new broctl.cfg option
-  "MailAlarmsInterval".
+  ``MailAlarmsInterval``.
 
-- We have completely reworded the "notice_policy" mechanism. It now no
-  linger uses a record of policy items but a "hook", a new language
-  element that's roughly equivalent to a function with multiple
-  bodies. The documentation [TODO: insert link] describes how to use
-  the new notice policy. For existing code, the two main changes are:
+- We have completely rewritten the ``notice_policy`` mechanism. It now
+  no longer uses a record of policy items but a ``hook``, a new
+  language element that's roughly equivalent to a function with
+  multiple bodies (see above). For existing code, the two main changes
+  are:
 
-    - What used to be a "redef" of "Notice::policy" now becomes a hook
-      implementation. Example:
+    - What used to be a ``redef`` of ``Notice::policy`` now becomes a
+      hook implementation. Example:
 
       Old::
 
@@ -221,9 +301,9 @@ Changed Functionality
                 add n$actions[Notice::ACTION_EMAIL];
             }
 
-    - notice() is now likewise a hook, no longer an event. If you have
-      handlers for that event, you'll likely just need to change the
-      type accordingly. Example:
+    - notice() is now likewise a hook, no longer an event. If you
+      have handlers for that event, you'll likely just need to change
+      the type accordingly. Example:
 
       Old::
 
@@ -233,17 +313,17 @@ Changed Functionality
 
         hook notice(n: Notice::Info) { ... }
 
-- The notice_policy.log is gone. That's a result of the new notice
+- The ``notice_policy.log`` is gone. That's a result of the new notice
   policy setup.
 
-- Removed the byte_len() and length() bif functions. Use the ``|...|``
-  operator instead.
+- Removed the ``byte_len()`` and ``length()`` bif functions. Use the
+  ``|...|`` operator instead.
 
-- The SSH::Login notice has been superseded by an corresponding
-  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+- The ``SSH::Login`` notice has been superseded by an corresponding
+  intelligence framework observation (``SSH::SUCCESSFUL_LOGIN``).
 
-- PacketFilter::all_packets has been replaced with
-  PacketFilter::enable_auto_protocol_capture_filters.
+- ``PacketFilter::all_packets`` has been replaced with
+  ``PacketFilter::enable_auto_protocol_capture_filters``.
 
 - We removed the BitTorrent DPD signatures pending further updates to
   that analyzer.

From 00a3ad03dd73ac3058be16a56c1f8a8368ecc4f0 Mon Sep 17 00:00:00 2001
From: Robin Sommer <robin@icir.org>
Date: Mon, 23 Sep 2013 11:54:48 -0700
Subject: [PATCH 16/17] Updating submodule(s).

 [nomail]
---
 CHANGES      | 10 ++++++++++
 VERSION      |  2 +-
 aux/broccoli |  2 +-
 aux/broctl   |  2 +-
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/CHANGES b/CHANGES
index 9ff7b2368c..00cd9b9176 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,14 @@
 
+2.1-1387 | 2013-09-23 11:54:48 -0700
+
+  * Change submodules to fixed URL. (Jon Siwek)
+
+  * Updating NEWS. (Robin Sommer)
+
+  * Fixing an always false condition. (Robin Sommer)
+    
+  * Fix required for compiling with clang 3.3. (Robin Sommer)
+    
 2.1-1377 | 2013-09-20 14:38:15 -0700
 
   * Updates to the scripting introduction. (Scott Runnels)
diff --git a/VERSION b/VERSION
index 326b107a34..deed660f20 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.1-1377
+2.1-1387
diff --git a/aux/broccoli b/aux/broccoli
index baf003ac16..2a886a31f2 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit baf003ac1648f63cd871942b20f8da82184af784
+Subproject commit 2a886a31f23ebecd17274af1af581f9e40f1810d
diff --git a/aux/broctl b/aux/broctl
index 629e155c71..f80ddc95cd 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit 629e155c712deea170330842ad7bfa9ae666f325
+Subproject commit f80ddc95cdfae6c6a4f20169de60767271b240e1

From 6e71c97bfe09d3b3c19cc0b137ecc784da4f7eeb Mon Sep 17 00:00:00 2001
From: Jon Siwek <jsiwek@illinois.edu>
Date: Mon, 23 Sep 2013 14:44:50 -0500
Subject: [PATCH 17/17] Update 'make dist' target.

---
 Makefile     | 13 +++++++------
 aux/binpac   |  2 +-
 aux/bro-aux  |  2 +-
 aux/broccoli |  2 +-
 aux/broctl   |  2 +-
 5 files changed, 11 insertions(+), 10 deletions(-)

diff --git a/Makefile b/Makefile
index d915050b6b..cbc44c6780 100644
--- a/Makefile
+++ b/Makefile
@@ -44,13 +44,14 @@ broxygenclean: configured
 dist:
 	@rm -rf $(VERSION_FULL) $(VERSION_FULL).tgz
 	@rm -rf $(VERSION_MIN) $(VERSION_MIN).tgz
-	@mkdir $(VERSION_FULL)
-	@tar --exclude=$(VERSION_FULL)* --exclude=$(VERSION_MIN)* --exclude=.git -cf - . | ( cd $(VERSION_FULL) && tar -xpf - )
-	@( cd $(VERSION_FULL) && cp -R ../.git . && git reset -q --hard HEAD && git clean -xdfq && rm -rf .git )
+	@git clone --recursive . $(VERSION_FULL) >/dev/null 2>&1
+	@find $(VERSION_FULL) -name .git\* | xargs rm -rf
 	@tar -czf $(VERSION_FULL).tgz $(VERSION_FULL) && echo Package: $(VERSION_FULL).tgz && rm -rf $(VERSION_FULL)
-	@$(HAVE_MODULES) && mkdir $(VERSION_MIN) || exit 0
-	@$(HAVE_MODULES) && tar --exclude=$(VERSION_FULL)* --exclude=$(VERSION_MIN)* --exclude=.git `git submodule | awk '{print "--exclude="$$2}' | grep -v cmake | tr '\n' ' '` -cf - . | ( cd $(VERSION_MIN) && tar -xpf - ) || exit 0
-	@$(HAVE_MODULES) && ( cd $(VERSION_MIN) && cp -R ../.git . && git reset -q --hard HEAD && git clean -xdfq && rm -rf .git ) || exit 0
+	@$(HAVE_MODULES) && git clone . $(VERSION_MIN) >/dev/null 2>&1 || exit 0
+	@$(HAVE_MODULES) && (cd $(VERSION_MIN) && git submodule update --init cmake >/dev/null 2>&1) || exit 0
+	@$(HAVE_MODULES) && (cd $(VERSION_MIN) && git submodule update --init src/3rdparty >/dev/null 2>&1) || exit 0
+	@$(HAVE_MODULES) && (cd $(VERSION_MIN) && git submodule update --init magic >/dev/null 2>&1) || exit 0
+	@$(HAVE_MODULES) && find $(VERSION_MIN) -name .git\* | xargs rm -rf || exit 0
 	@$(HAVE_MODULES) && tar -czf $(VERSION_MIN).tgz $(VERSION_MIN) && echo Package: $(VERSION_MIN).tgz && rm -rf $(VERSION_MIN) || exit 0
 
 bindist:
diff --git a/aux/binpac b/aux/binpac
index 5f9d4afe82..3c29b917e5 160000
--- a/aux/binpac
+++ b/aux/binpac
@@ -1 +1 @@
-Subproject commit 5f9d4afe8274b1c1125757ca0e971f66d6d35384
+Subproject commit 3c29b917e59e8d8200f669d3d9729d36c34b9245
diff --git a/aux/bro-aux b/aux/bro-aux
index 387e26609a..ee2d64928e 160000
--- a/aux/bro-aux
+++ b/aux/bro-aux
@@ -1 +1 @@
-Subproject commit 387e26609a923f5abd10be593ae127690f7a6b72
+Subproject commit ee2d64928edc38b10e508bd577a22f52b024c992
diff --git a/aux/broccoli b/aux/broccoli
index 2a886a31f2..c0d5345bf2 160000
--- a/aux/broccoli
+++ b/aux/broccoli
@@ -1 +1 @@
-Subproject commit 2a886a31f23ebecd17274af1af581f9e40f1810d
+Subproject commit c0d5345bf25d25f6965f3201048344687bacc860
diff --git a/aux/broctl b/aux/broctl
index f80ddc95cd..3582f494de 160000
--- a/aux/broctl
+++ b/aux/broctl
@@ -1 +1 @@
-Subproject commit f80ddc95cdfae6c6a4f20169de60767271b240e1
+Subproject commit 3582f494de247784fc7634b319ddf99aef44b6e1