Merge remote-tracking branch 'origin/master' into topic/seth/file-analysis-exe-analyzer

Conflicts:
	src/types.bif

CHANGES

@@ -1,4 +1,96 @@
 
+2.1-641 | 2013-05-15 18:15:09 -0700
+
+  * Test update. (Robin Sommer)
+
+2.1-640 | 2013-05-15 17:24:09 -0700
+
+  * Support for cleaning up threads that have terminated. (Bernhard
+    Amann and Robin Sommer). Includes:
+
+      - Both logging and input frameworks now clean up threads once
+        they aren't further needed anymnore.
+
+      - New function Log::remove_stream() that removes a logging
+        stream, stopping all writer threads that are associated with
+        it. Note, however, that removing a *filter* from a stream
+        still doesn't clean up any threads. The problem is that
+        because of the output paths potentially being created
+        dynamically it's unclear if the writer thread will still be
+        needed in the future.
+
+2.1-626 | 2013-05-15 16:09:31 -0700
+
+  * Add "reservoir" sampler for SumStats framework. This maintains
+    a set of N uniquely distributed random samples. (Bernhard Amann)
+
+2.1-619 | 2013-05-15 16:01:42 -0700
+
+  * SQLite reader and writer combo. This allows to read/write
+    persistent data from on disk SQLite databases. The current
+    interface is quite low-level, we'll add higher-level abstractions
+    in the future. (Bernhard Amann)
+
+2.1-576 | 2013-05-15 14:29:09 -0700
+
+  * Initial version of new file analysis framework. This moves most of
+    the processing of file content from script-land into the core,
+    where it belongs. Much of this is an internal change, and at this
+    point the new code has essentially feature-equality with the old
+    one. More script-level changes to come. (Jon Siwek)
+
+2.1-502 | 2013-05-10 19:29:37 -0700
+
+  * Allow default function/hook/event parameters. Addresses #972. (Jon
+    Siwek)
+
+  * Change the endianness parameter of bytestring_to_count() BIF to
+    default to false (big endian). (Jon Siwek)
+
+2.1-500 | 2013-05-10 19:22:24 -0700
+
+  * Fix to prevent merge-hook of SumStat's unique plugin from damaging
+    source data. (Bernhard Amann)
+
+2.1-498 | 2013-05-03 17:44:08 -0700
+
+  * Table lookups return copy of non-const &default vals. This
+    prevents unintentional modifications to the &default value itself.
+    Addresses #981.  (Jon Siwek)
+
+2.1-496 | 2013-05-03 15:54:47 -0700
+
+  * Fix memory leak and unnecessary allocations in OpaqueVal.
+    Addresses #986. (Matthias Vallentin)
+
+2.1-492 | 2013-05-02 12:46:26 -0700
+
+  * Work-around for sumstats framework not propagating updates after
+    intermediate check in cluster environments. (Bernhard Amann)
+
+  * Always apply tcp_connection_attempt. Before this change it was
+    only applied when a connection_attempt() event handler was
+    defined. (Robin Sommer)
+
+  * Fixing coverage.bare-mode-errors test. (Robin Sommer)
+
+2.1-487 | 2013-05-01 18:03:22 -0700
+
+  * Always apply tcp_connection_attempt timer, even if no
+    connection_attempt() event handler is defined. (Robin Sommer)
+
+2.1-486 | 2013-05-01 15:28:45 -0700
+
+  * New framework for computing summary statistics in
+    base/framework/sumstats. This replaces the metrics frameworks, and
+    comes with a number of applications build on top, see NEWS. More
+    documentation to follow. (Seth Hall)
+
+2.1-397 | 2013-04-29 21:19:00 -0700
+
+  * Fixing memory leaks in CompHash implementation. Addresses #987.
+    (Robin Sommer)
+
 2.1-394 | 2013-04-27 15:02:31 -0700
 
   * Fixed a bug in the vulnerable software script and added a test.

DocSourcesList.cmake

@@ -1,144 +0,0 @@
-# DO NOT EDIT
-# This file is auto-generated from the genDocSourcesList.sh script.
-#
-# This is a list of Bro script sources for which to generate reST documentation.
-# It will be included inline in the CMakeLists.txt found in the same directory
-# in order to create Makefile targets that define how to generate reST from
-# a given Bro script.
-#
-# Note: any path prefix of the script (2nd argument of rest_target macro)
-# will be used to derive what path under scripts/ the generated documentation
-# will be placed.
-
-set(psd ${PROJECT_SOURCE_DIR}/scripts)
-
-rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
-rest_target(${psd} base/init-default.bro internal)
-rest_target(${psd} base/init-bare.bro internal)
-
-rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
-rest_target(${psd} base/frameworks/cluster/main.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
-rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
-rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
-rest_target(${psd} base/frameworks/communication/main.bro)
-rest_target(${psd} base/frameworks/control/main.bro)
-rest_target(${psd} base/frameworks/dpd/main.bro)
-rest_target(${psd} base/frameworks/intel/main.bro)
-rest_target(${psd} base/frameworks/logging/main.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
-rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
-rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
-rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
-rest_target(${psd} base/frameworks/notice/actions/drop.bro)
-rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
-rest_target(${psd} base/frameworks/notice/actions/page.bro)
-rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
-rest_target(${psd} base/frameworks/notice/cluster.bro)
-rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
-rest_target(${psd} base/frameworks/notice/main.bro)
-rest_target(${psd} base/frameworks/notice/weird.bro)
-rest_target(${psd} base/frameworks/packet-filter/main.bro)
-rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
-rest_target(${psd} base/frameworks/reporter/main.bro)
-rest_target(${psd} base/frameworks/signatures/main.bro)
-rest_target(${psd} base/frameworks/software/main.bro)
-rest_target(${psd} base/protocols/conn/contents.bro)
-rest_target(${psd} base/protocols/conn/inactivity.bro)
-rest_target(${psd} base/protocols/conn/main.bro)
-rest_target(${psd} base/protocols/dns/consts.bro)
-rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/file-extract.bro)
-rest_target(${psd} base/protocols/ftp/main.bro)
-rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/http/file-extract.bro)
-rest_target(${psd} base/protocols/http/file-hash.bro)
-rest_target(${psd} base/protocols/http/file-ident.bro)
-rest_target(${psd} base/protocols/http/main.bro)
-rest_target(${psd} base/protocols/http/utils.bro)
-rest_target(${psd} base/protocols/irc/dcc-send.bro)
-rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
-rest_target(${psd} base/protocols/smtp/entities.bro)
-rest_target(${psd} base/protocols/smtp/main.bro)
-rest_target(${psd} base/protocols/ssh/main.bro)
-rest_target(${psd} base/protocols/ssl/consts.bro)
-rest_target(${psd} base/protocols/ssl/main.bro)
-rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
-rest_target(${psd} base/protocols/syslog/consts.bro)
-rest_target(${psd} base/protocols/syslog/main.bro)
-rest_target(${psd} base/utils/addrs.bro)
-rest_target(${psd} base/utils/conn-ids.bro)
-rest_target(${psd} base/utils/directions-and-hosts.bro)
-rest_target(${psd} base/utils/files.bro)
-rest_target(${psd} base/utils/numbers.bro)
-rest_target(${psd} base/utils/paths.bro)
-rest_target(${psd} base/utils/patterns.bro)
-rest_target(${psd} base/utils/site.bro)
-rest_target(${psd} base/utils/strings.bro)
-rest_target(${psd} base/utils/thresholds.bro)
-rest_target(${psd} policy/frameworks/communication/listen.bro)
-rest_target(${psd} policy/frameworks/control/controllee.bro)
-rest_target(${psd} policy/frameworks/control/controller.bro)
-rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
-rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
-rest_target(${psd} policy/frameworks/software/version-changes.bro)
-rest_target(${psd} policy/frameworks/software/vulnerable.bro)
-rest_target(${psd} policy/integration/barnyard2/main.bro)
-rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/misc/analysis-groups.bro)
-rest_target(${psd} policy/misc/capture-loss.bro)
-rest_target(${psd} policy/misc/loaded-scripts.bro)
-rest_target(${psd} policy/misc/profiling.bro)
-rest_target(${psd} policy/misc/stats.bro)
-rest_target(${psd} policy/misc/trim-trace-file.bro)
-rest_target(${psd} policy/protocols/conn/known-hosts.bro)
-rest_target(${psd} policy/protocols/conn/known-services.bro)
-rest_target(${psd} policy/protocols/conn/weirds.bro)
-rest_target(${psd} policy/protocols/dns/auth-addl.bro)
-rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
-rest_target(${psd} policy/protocols/ftp/detect.bro)
-rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-MHR.bro)
-rest_target(${psd} policy/protocols/http/detect-intel.bro)
-rest_target(${psd} policy/protocols/http/detect-sqli.bro)
-rest_target(${psd} policy/protocols/http/detect-webapps.bro)
-rest_target(${psd} policy/protocols/http/header-names.bro)
-rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
-rest_target(${psd} policy/protocols/http/software.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
-rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
-rest_target(${psd} policy/protocols/smtp/blocklists.bro)
-rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
-rest_target(${psd} policy/protocols/smtp/software.bro)
-rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
-rest_target(${psd} policy/protocols/ssh/geo-data.bro)
-rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
-rest_target(${psd} policy/protocols/ssh/software.bro)
-rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
-rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
-rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
-rest_target(${psd} policy/protocols/ssl/known-certs.bro)
-rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
-rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
-rest_target(${psd} policy/tuning/defaults/warnings.bro)
-rest_target(${psd} policy/tuning/track-all-assets.bro)
-rest_target(${psd} site/local-manager.bro)
-rest_target(${psd} site/local-proxy.bro)
-rest_target(${psd} site/local-worker.bro)
-rest_target(${psd} site/local.bro)
-rest_target(${psd} test-all-policy.bro)

NEWS

@@ -51,6 +51,40 @@ New Functionality
   can take up to two indices for the start and end index of the
   substring to return (e.g. "mystring[1,3]").
 
+- Functions now support default parameters, e.g.:
+
+        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+
+- The new file analysis framework moves most of the processing of file
+  content from script-land into the core, where it belongs. Much of
+  this is an internal change, the framework comes with the following
+  user-visibible functionality (some of that was already available
+  before, but done differently):
+
+  [TODO: This will probably change with further script updates.]
+
+      - A binary input reader interfaces the input framework with file
+        analysis, allowing to inject files on disk into Bro's
+        processing.
+
+      - Supports for analyzing data transfereed via HTTP range
+        requests.
+
+      - HTTP:
+        * Identify MIME type of message.
+        * Extract message to disk.
+        * Compute MD5 for messages.
+
+      - SMTP:
+        * Identify MIME type of message.
+        * Extract message to disk.
+        * Compute MD5 for messages.
+        * Provide access to start of entity data.
+
+      - FTP data transfers: Identify MIME type; record to disk.
+
+      - IRC DCC transfers: Record to disk.
+
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~
 
@@ -126,6 +160,9 @@ Changed Functionality
 - Removed the byte_len() and length() bif functions. Use the "|...|"
   operator instead.
 
+- The SSH::Login notice has been superseded by an corresponding
+  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+
 Bro 2.1
 -------
 
@@ -209,6 +246,27 @@ New Functionality
   outputs. We do not yet recommend them for production (but welcome
   feedback!)
 
+- Summary statistics framework. [Extend]
+
+- A number of new applications build on top of the summary statistics
+  framework:
+
+    * Scan detection: Detectors for port and address scans return. See
+      policy/misc/scan.bro.
+
+    * Tracerouter detector: policy/misc/detect-traceroute
+
+    * Web application detection/measurement: policy/misc/app-metrics.bro
+
+    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
+
+    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
+      (existed before, but now ported to the new framework)
+
+    * SSH brute-forcing detector feeding the intelligence framework:
+      policy/protocols/ssh/detect-bruteforcing.bro
+
+
 
 Changed Functionality
 ~~~~~~~~~~~~~~~~~~~~~

VERSION

@@ -1 +1 @@
-2.1-394
+2.1-641

aux/binpac

@@ -1 +1 @@
-Subproject commit a4b8dd0b691c3f614537ad8471fc80a82ce7b2df
+Subproject commit f86a3169b8d49189d264cbc1a7507260cd9ff51d

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 70681007546aad6e5648494e882b71adb9165105
+Subproject commit 18c454981e3b0903811c541f6e728d4ef6cee2c5

aux/broccoli

@@ -1 +1 @@
-Subproject commit e64204fec55759c614a276c1933bbff2069a63db
+Subproject commit 8955807b0f4151f5f6aca2e68d353b9b341d9f86

aux/broctl

@@ -1 +1 @@
-Subproject commit 786b83664c6a15faeb153d118310526b7790deae
+Subproject commit 3ea30f7a146343f054a3846a61ee5c67259b2de2

doc/scripts/DocSourcesList.cmake

@@ -39,6 +39,7 @@ rest_target(${psd} base/frameworks/input/readers/ascii.bro)
 rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
 rest_target(${psd} base/frameworks/input/readers/binary.bro)
 rest_target(${psd} base/frameworks/input/readers/raw.bro)
+rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
 rest_target(${psd} base/frameworks/intel/cluster.bro)
 rest_target(${psd} base/frameworks/intel/input.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
@@ -49,9 +50,7 @@ rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
 rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
 rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
 rest_target(${psd} base/frameworks/logging/writers/none.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
+rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -67,6 +66,18 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/sumstats/cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/main.bro)
+rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
 rest_target(${psd} base/frameworks/tunnels/main.bro)
 rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
@@ -110,9 +121,11 @@ rest_target(${psd} base/utils/files.bro)
 rest_target(${psd} base/utils/numbers.bro)
 rest_target(${psd} base/utils/paths.bro)
 rest_target(${psd} base/utils/patterns.bro)
+rest_target(${psd} base/utils/queue.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} base/utils/time.bro)
 rest_target(${psd} base/utils/urls.bro)
 rest_target(${psd} policy/frameworks/communication/listen.bro)
 rest_target(${psd} policy/frameworks/control/controllee.bro)
@@ -128,18 +141,18 @@ rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
 rest_target(${psd} policy/frameworks/intel/smtp.bro)
 rest_target(${psd} policy/frameworks/intel/ssl.bro)
 rest_target(${psd} policy/frameworks/intel/where-locations.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
 rest_target(${psd} policy/integration/collective-intel/main.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
+rest_target(${psd} policy/misc/app-metrics.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
+rest_target(${psd} policy/misc/detect-traceroute/main.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
+rest_target(${psd} policy/misc/scan.bro)
 rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
@@ -147,6 +160,7 @@ rest_target(${psd} policy/protocols/conn/known-services.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
+rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)

doc/scripts/builtins.rst

@@ -459,6 +459,31 @@ The Bro scripting language supports the following built-in types.
 
         print greeting("Dave");
 
+    Function parameters may specify default values as long as they appear
+    last in the parameter list:
+
+    .. code:: bro
+
+        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+
+    If a function was previously declared with default parameters, the
+    default expressions can be omitted when implementing the function
+    body and they will still be used for function calls that lack those
+    arguments.
+
+    .. code:: bro
+
+        function foo(s: string, t: string, u: count)
+            {
+            print s, t, u;
+            }
+
+    And calls to the function may omit the defaults from the argument list:
+
+    .. code:: bro
+
+        foo("test");
+
 .. bro:type:: event
 
     Event handlers are nearly identical in both syntax and semantics to
@@ -597,10 +622,10 @@ scripting language supports the following built-in attributes.
 
 .. bro:attr:: &default
 
-    Uses a default value for a record field or container elements. For
+    Uses a default value for a record field, a function/hook/event
-    example, ``table[int] of string &default="foo" }`` would create a
+    parameter, or container elements.  For example, ``table[int] of
-    table that returns the :bro:type:`string` ``"foo"`` for any
+    string &default="foo" }`` would create a table that returns the
-    non-existing index.
+    :bro:type:`string` ``"foo"`` for any non-existing index.
 
 .. bro:attr:: &redef
 

scripts/base/frameworks/input/__load__.bro

@@ -3,4 +3,4 @@
 @load ./readers/raw
 @load ./readers/benchmark
 @load ./readers/binary
-
+@load ./readers/sqlite

scripts/base/frameworks/input/readers/sqlite.bro

@@ -0,0 +1,17 @@
+##! Interface for the SQLite input reader.
+##!
+##! The defaults are set to match Bro's ASCII output.
+
+module InputSQLite;
+
+export {
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long.
+	const set_separator = Input::set_separator &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Input::unset_field &redef;
+
+	## String to use for empty fields.
+	const empty_field = Input::empty_field &redef;
+}

scripts/base/frameworks/logging/__load__.bro

@@ -2,5 +2,6 @@
 @load ./postprocessors
 @load ./writers/ascii
 @load ./writers/dataseries
+@load ./writers/sqlite
 @load ./writers/elasticsearch
 @load ./writers/none

scripts/base/frameworks/logging/main.bro

@@ -189,6 +189,15 @@ export {
 	## .. bro:see:: Log::add_default_filter Log::remove_default_filter
 	global create_stream: function(id: ID, stream: Stream) : bool;
 
+	## Removes a logging stream completely, stopping all the threads.
+	##
+	## id: The ID enum to be associated with the new logging stream.
+	##
+	## Returns: True if a new stream was successfully removed.
+	##
+	## .. bro:see:: Log:create_stream
+	global remove_stream: function(id: ID) : bool;
+
 	## Enables a previously disabled logging stream.  Disabled streams
 	## will not be written to until they are enabled again.  New streams
 	## are enabled by default.
@@ -442,6 +451,12 @@ function create_stream(id: ID, stream: Stream) : bool
 	return add_default_filter(id);
 	}
 
+function remove_stream(id: ID) : bool
+	{
+	delete active_streams[id];
+	return __remove_stream(id);
+	}
+
 function disable_stream(id: ID) : bool
 	{
 	delete active_streams[id];

scripts/base/frameworks/logging/writers/sqlite.bro

@@ -0,0 +1,17 @@
+##! Interface for the SQLite log writer.  Redefinable options are available
+##! to tweak the output format of the SQLite reader.
+
+module LogSQLite;
+
+export {
+	## Separator between set elements.
+	const set_separator = Log::set_separator &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Log::unset_field &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious.
+	const empty_field = Log::empty_field &redef;
+}
+

scripts/base/frameworks/metrics/cluster.bro

@@ -1,264 +0,0 @@
-##! This implements transparent cluster support for the metrics framework.
-##! Do not load this file directly.  It's only meant to be loaded automatically
-##! and will be depending on if the cluster framework has been enabled.
-##! The goal of this script is to make metric calculation completely and
-##! transparently automated when running on a cluster.
-##! 
-##! Events defined here are not exported deliberately because they are meant
-##! to be an internal implementation detail.
-
-@load base/frameworks/cluster
-@load ./main
-
-module Metrics;
-
-export {
-	## Allows a user to decide how large of result groups the 
-	## workers should transmit values for cluster metric aggregation.
-	const cluster_send_in_groups_of = 50 &redef;
-	
-	## The percent of the full threshold value that needs to be met 
-	## on a single worker for that worker to send the value to its manager in
-	## order for it to request a global view for that value.  There is no
-	## requirement that the manager requests a global view for the index
-	## since it may opt not to if it requested a global view for the index
-	## recently.
-	const cluster_request_global_view_percent = 0.1 &redef;
-	
-	## Event sent by the manager in a cluster to initiate the 
-	## collection of metrics values for a filter.
-	global cluster_filter_request: event(uid: string, id: ID, filter_name: string);
-
-	## Event sent by nodes that are collecting metrics after receiving
-	## a request for the metric filter from the manager.
-	global cluster_filter_response: event(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool);
-
-	## This event is sent by the manager in a cluster to initiate the
-	## collection of a single index value from a filter.  It's typically
-	## used to get intermediate updates before the break interval triggers
-	## to speed detection of a value crossing a threshold.
-	global cluster_index_request: event(uid: string, id: ID, filter_name: string, index: Index);
-
-	## This event is sent by nodes in response to a 
-	## :bro:id:`Metrics::cluster_index_request` event.
-	global cluster_index_response: event(uid: string, id: ID, filter_name: string, index: Index, val: count);
-
-	## This is sent by workers to indicate that they crossed the percent of the 
-	## current threshold by the percentage defined globally in 
-	## :bro:id:`Metrics::cluster_request_global_view_percent`
-	global cluster_index_intermediate_response: event(id: Metrics::ID, filter_name: string, index: Metrics::Index, val: count);
-
-	## This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, id: ID, filter_name: string, data: MetricTable);
-	
-}
-
-
-# This is maintained by managers so they can know what data they requested and
-# when they requested it.
-global requested_results: table[string] of time = table() &create_expire=5mins;
-
-# TODO: The next 4 variables make the assumption that a value never 
-#       takes longer than 5 minutes to transmit from workers to manager.  This needs to 
-#       be tunable or self-tuning.  These should also be restructured to be
-#       maintained within a single variable.
-
-# This variable is maintained by manager nodes as they collect and aggregate 
-# results.
-global filter_results: table[string, ID, string] of MetricTable &create_expire=5mins;
-
-# This variable is maintained by manager nodes to track how many "dones" they
-# collected per collection unique id.  Once the number of results for a uid 
-# matches the number of peer nodes that results should be coming from, the 
-# result is written out and deleted from here.
-# TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &create_expire=5mins &default=0;
-
-# This variable is maintained by managers to track intermediate responses as 
-# they are getting a global view for a certain index.
-global index_requests: table[string, ID, string, Index] of count &create_expire=5mins &default=0;
-
-# This variable is maintained by all hosts for different purposes. Non-managers
-# maintain it to know what indexes they have recently sent as intermediate
-# updates so they don't overwhelm their manager. Managers maintain it so they
-# don't overwhelm workers with intermediate index requests. The count that is
-# yielded is the number of times the percentage threshold has been crossed and
-# an intermediate result has been received. The manager may optionally request
-# the index again before data expires from here if too many workers are crossing
-# the percentage threshold (not implemented yet!).
-global recent_global_view_indexes: table[ID, string, Index] of count &create_expire=5mins &default=0;
-
-# Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
-redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
-
-@if ( Cluster::local_node_type() != Cluster::MANAGER )
-# This is done on all non-manager node types in the event that a metric is 
-# being collected somewhere other than a worker.
-function data_added(filter: Filter, index: Index, val: count)
-	{
-	# If an intermediate update for this value was sent recently, don't send
-	# it again.
-	if ( [filter$id, filter$name, index] in recent_global_view_indexes )
-		return;
-		
-	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
-	# crosses the full threshold then it's a candidate to send as an 
-	# intermediate update.
-	local pct_val = double_to_count(val / cluster_request_global_view_percent);
-	
-	if ( check_notice(filter, index, pct_val) ) 
-		{
-		# kick off intermediate update
-		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val);
-		
-		++recent_global_view_indexes[filter$id, filter$name, index];
-		}
-	}
-
-event Metrics::send_data(uid: string, id: ID, filter_name: string, data: MetricTable)
-	{
-	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
-	
-	local local_data: MetricTable;
-	local num_added = 0;
-	for ( index in data )
-		{
-		local_data[index] = data[index];
-		delete data[index];
-		
-		# Only send cluster_send_in_groups_of at a time.  Queue another
-		# event to send the next group.
-		if ( cluster_send_in_groups_of == ++num_added )
-			break;
-		}
-	
-	local done = F;
-	# If data is empty, this metric is done.
-	if ( |data| == 0 )
-		done = T;
-	
-	event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
-	if ( ! done )
-		event Metrics::send_data(uid, id, filter_name, data);
-	}
-
-event Metrics::cluster_filter_request(uid: string, id: ID, filter_name: string)
-	{
-	#print fmt("WORKER %s: received the cluster_filter_request event.", Cluster::node);
-	
-	# Initiate sending all of the data for the requested filter.
-	event Metrics::send_data(uid, id, filter_name, store[id, filter_name]);
-	
-	# Lookup the actual filter and reset it, the reference to the data
-	# currently stored will be maintained interally by the send_data event.
-	reset(filter_store[id, filter_name]);
-	}
-	
-event Metrics::cluster_index_request(uid: string, id: ID, filter_name: string, index: Index)
-	{
-	local val=0;
-	if ( index in store[id, filter_name] )
-		val = store[id, filter_name][index];
-	
-	# fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val);
-	event Metrics::cluster_index_response(uid, id, filter_name, index, val);
-	}
-
-@endif
-
-
-@if ( Cluster::local_node_type() == Cluster::MANAGER )
-
-# Manager's handle logging.
-event Metrics::log_it(filter: Filter)
-	{
-	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
-	
-	local uid = unique_id("");
-	
-	# Set some tracking variables.
-	requested_results[uid] = network_time();
-	filter_results[uid, filter$id, filter$name] = table();
-	
-	# Request data from peers.
-	event Metrics::cluster_filter_request(uid, filter$id, filter$name);
-	# Schedule the log_it event for the next break period.
-	schedule filter$break_interval { Metrics::log_it(filter) };
-	}
-
-# This is unlikely to be called often, but it's here in case there are metrics
-# being collected by managers.
-function data_added(filter: Filter, index: Index, val: count)
-	{
-	if ( check_notice(filter, index, val) )
-		do_notice(filter, index, val);
-	}
-	
-event Metrics::cluster_index_response(uid: string, id: ID, filter_name: string, index: Index, val: count)
-	{
-	#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
-
-	if ( [uid, id, filter_name, index] !in index_requests )
-		index_requests[uid, id, filter_name, index] = 0;
-	
-	index_requests[uid, id, filter_name, index] += val;
-	local ir = index_requests[uid, id, filter_name, index];
-	
-	++done_with[uid];
-	if ( Cluster::worker_count == done_with[uid] )
-		{
-		if ( check_notice(filter_store[id, filter_name], index, ir) )
-			do_notice(filter_store[id, filter_name], index, ir);
-		delete done_with[uid];
-		delete index_requests[uid, id, filter_name, index];
-		}
-	}
-
-# Managers handle intermediate updates here.
-event Metrics::cluster_index_intermediate_response(id: ID, filter_name: string, index: Index, val: count)
-	{
-	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
-	#print fmt("MANAGER: requesting index data for %s", index2str(index));
-	
-	local uid = unique_id("");
-	event Metrics::cluster_index_request(uid, id, filter_name, index);
-	++recent_global_view_indexes[id, filter_name, index];
-	}
-
-event Metrics::cluster_filter_response(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool)
-	{
-	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
-	
-	local local_data = filter_results[uid, id, filter_name];
-	for ( index in data )
-		{
-		if ( index !in local_data )
-			local_data[index] = 0;
-		local_data[index] += data[index];
-		}
-	
-	# Mark another worker as being "done" for this uid.
-	if ( done )
-		++done_with[uid];
-	
-	# If the data has been collected from all peers, we are done and ready to log.
-	if ( Cluster::worker_count == done_with[uid] )
-		{
-		local ts = network_time();
-		# Log the time this was initially requested if it's available.
-		if ( uid in requested_results )
-			{
-			ts = requested_results[uid];
-			delete requested_results[uid];
-			}
-		
-		write_log(ts, filter_store[id, filter_name], local_data);
-		
-		# Clean up
-		delete filter_results[uid, id, filter_name];
-		delete done_with[uid];
-		}
-	}
-
-@endif

scripts/base/frameworks/metrics/main.bro

@@ -1,320 +0,0 @@
-##! The metrics framework provides a way to count and measure data.  
-
-@load base/frameworks/notice
-
-module Metrics;
-
-export {
-	## The metrics logging stream identifier.
-	redef enum Log::ID += { LOG };
-	
-	## Identifiers for metrics to collect.
-	type ID: enum {
-		## Blank placeholder value.
-		NOTHING,
-	};
-	
-	## The default interval used for "breaking" metrics and writing the 
-	## current value to the logging stream.
-	const default_break_interval = 15mins &redef;
-	
-	## This is the interval for how often threshold based notices will happen 
-	## after they have already fired.
-	const renotice_interval = 1hr &redef;
-	
-	## Represents a thing which is having metrics collected for it.  An instance
-	## of this record type and a :bro:type:`Metrics::ID` together represent a 
-	## single measurement.
-	type Index: record {
-		## Host is the value to which this metric applies.
-		host:         addr &optional;
-		
-		## A non-address related metric or a sub-key for an address based metric.
-		## An example might be successful SSH connections by client IP address
-		## where the client string would be the index value.
-		## Another example might be number of HTTP requests to a particular
-		## value in a Host header.  This is an example of a non-host based
-		## metric since multiple IP addresses could respond for the same Host
-		## header value.
-		str:        string &optional;
-		
-		## The CIDR block that this metric applies to.  This is typically
-		## only used internally for host based aggregation.
-		network:      subnet &optional;
-	} &log;
-	
-	## The record type that is used for logging metrics.
-	type Info: record {
-		## Timestamp at which the metric was "broken".
-		ts:           time   &log;
-		## What measurement the metric represents.
-		metric_id:    ID     &log;
-		## The name of the filter being logged.  :bro:type:`Metrics::ID` values
-		## can have multiple filters which represent different perspectives on
-		## the data so this is necessary to understand the value.
-		filter_name:  string &log;
-		## What the metric value applies to.
-		index:        Index  &log;
-		## The simple numeric value of the metric.
-		value:        count  &log;
-	};
-	
-    # TODO: configure a metrics filter logging stream to log the current
-	#       metrics configuration in case someone is looking through
-	#       old logs and the configuration has changed since then.
-	
-	## Filters define how the data from a metric is aggregated and handled.  
-	## Filters can be used to set how often the measurements are cut or "broken"
-	## and logged or how the data within them is aggregated.  It's also 
-	## possible to disable logging and use filters for thresholding.
-	type Filter: record {
-		## The :bro:type:`Metrics::ID` that this filter applies to.
-		id:                ID                      &optional;
-		## The name for this filter so that multiple filters can be
-		## applied to a single metrics to get a different view of the same
-		## metric data being collected (different aggregation, break, etc).
-		name:              string                  &default="default";
-		## A predicate so that you can decide per index if you would like
-		## to accept the data being inserted.
-		pred:              function(index: Index): bool &optional;
-		## Global mask by which you'd like to aggregate traffic.
-		aggregation_mask:  count                   &optional;
-		## This is essentially a mapping table between addresses and subnets.
-		aggregation_table: table[subnet] of subnet &optional;
-		## The interval at which this filter should be "broken" and written
-		## to the logging stream.  The counters are also reset to zero at 
-		## this time so any threshold based detection needs to be set to a 
-		## number that should be expected to happen within this period.
-		break_interval:    interval                &default=default_break_interval;
-		## This determines if the result of this filter is sent to the metrics
-		## logging stream.  One use for the logging framework is as an internal
-		## thresholding and statistics gathering utility that is meant to
-		## never log but rather to generate notices and derive data.
-		log:               bool                    &default=T;
-		## If this and a $notice_threshold value are set, this notice type
-		## will be generated by the metrics framework.
-		note:              Notice::Type            &optional;
-		## A straight threshold for generating a notice.
-		notice_threshold:  count                   &optional;
-		## A series of thresholds at which to generate notices.
-		notice_thresholds: vector of count         &optional;
-		## How often this notice should be raised for this filter.  It 
-		## will be generated everytime it crosses a threshold, but if the 
-		## $break_interval is set to 5mins and this is set to 1hr the notice
-		## only be generated once per hour even if something crosses the
-		## threshold in every break interval.
-		notice_freq:       interval                &optional;
-	};
-	
-	## Function to associate a metric filter with a metric ID.
-	## 
-	## id: The metric ID that the filter should be associated with.
-	##
-	## filter: The record representing the filter configuration.
-	global add_filter: function(id: ID, filter: Filter);
-	
-	## Add data into a :bro:type:`Metrics::ID`.  This should be called when
-	## a script has measured some point value and is ready to increment the
-	## counters.
-	##
-	## id: The metric ID that the data represents.
-	##
-	## index: The metric index that the value is to be added to.
-	##
-	## increment: How much to increment the counter by.
-	global add_data: function(id: ID, index: Index, increment: count);
-	
-	## Helper function to represent a :bro:type:`Metrics::Index` value as 
-	## a simple string
-	## 
-	## index: The metric index that is to be converted into a string.
-	##
-	## Returns: A string reprentation of the metric index.
-	global index2str: function(index: Index): string;
-	
-	## Event that is used to "finish" metrics and adapt the metrics
-	## framework for clustered or non-clustered usage.
-	##
-	## ..note: This is primarily intended for internal use.
-	global log_it: event(filter: Filter);
-	
-	## Event to access metrics records as they are passed to the logging framework.
-	global log_metrics: event(rec: Info);
-	
-	## Type to store a table of metrics values.  Interal use only!
-	type MetricTable: table[Index] of count &default=0;
-}
-
-redef record Notice::Info += {
-	metric_index: Index &log &optional;
-};
-
-global metric_filters: table[ID] of vector of Filter = table();
-global filter_store: table[ID, string] of Filter = table();
-
-# This is indexed by metric ID and stream filter name.
-global store: table[ID, string] of MetricTable = table() &default=table();
-
-# This function checks if a threshold has been crossed and generates a 
-# notice if it has.  It is also used as a method to implement 
-# mid-break-interval threshold crossing detection for cluster deployments.
-global check_notice: function(filter: Filter, index: Index, val: count): bool;
-
-# This is hook for watching thresholds being crossed.  It is called whenever
-# index values are updated and the new val is given as the `val` argument.
-global data_added: function(filter: Filter, index: Index, val: count);
-
-# This stores the current threshold index for filters using the
-# $notice_threshold and $notice_thresholds elements.
-global thresholds: table[ID, string, Index] of count = {} &create_expire=renotice_interval &default=0;
-
-event bro_init() &priority=5
-	{
-	Log::create_stream(Metrics::LOG, [$columns=Info, $ev=log_metrics]);
-	}
-
-function index2str(index: Index): string
-	{
-	local out = "";
-	if ( index?$host )
-		out = fmt("%shost=%s", out, index$host);
-	if ( index?$network )
-		out = fmt("%s%snetwork=%s", out, |out|==0 ? "" : ", ", index$network);
-	if ( index?$str )
-		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", index$str);
-	return fmt("metric_index(%s)", out);
-	}
-	
-function write_log(ts: time, filter: Filter, data: MetricTable)
-	{
-	for ( index in data )
-		{
-		local val = data[index];
-		local m: Info = [$ts=ts,
-		                 $metric_id=filter$id,
-		                 $filter_name=filter$name,
-		                 $index=index,
-		                 $value=val];
-		
-		if ( filter$log )
-			Log::write(Metrics::LOG, m);
-		}
-	}
-
-
-function reset(filter: Filter)
-	{
-	store[filter$id, filter$name] = table();
-	}
-
-function add_filter(id: ID, filter: Filter)
-	{
-	if ( filter?$aggregation_table && filter?$aggregation_mask )
-		{
-		print "INVALID Metric filter: Defined $aggregation_table and $aggregation_mask.";
-		return;
-		}
-	if ( [id, filter$name] in store )
-		{
-		print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
-		return;
-		}
-	if ( filter?$notice_threshold && filter?$notice_thresholds )
-		{
-		print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds";
-		return;
-		}
-	
-	if ( ! filter?$id )
-		filter$id = id;
-	
-	if ( id !in metric_filters )
-		metric_filters[id] = vector();
-	metric_filters[id][|metric_filters[id]|] = filter;
-
-	filter_store[id, filter$name] = filter;
-	store[id, filter$name] = table();
-	
-	schedule filter$break_interval { Metrics::log_it(filter) };
-	}
-	
-function add_data(id: ID, index: Index, increment: count)
-	{
-	if ( id !in metric_filters )
-		return;
-	
-	local filters = metric_filters[id];
-	
-	# Try to add the data to all of the defined filters for the metric.
-	for ( filter_id in filters )
-		{
-		local filter = filters[filter_id];
-		
-		# If this filter has a predicate, run the predicate and skip this
-		# index if the predicate return false.
-		if ( filter?$pred && ! filter$pred(index) )
-			next;
-		
-		if ( index?$host )
-			{
-			if ( filter?$aggregation_mask )
-				{
-				index$network = mask_addr(index$host, filter$aggregation_mask);
-				delete index$host;
-				}
-			else if ( filter?$aggregation_table )
-				{
-				# Don't add the data if the aggregation table doesn't include 
-				# the given host address.
-				if ( index$host !in filter$aggregation_table )
-					return;
-				index$network = filter$aggregation_table[index$host];
-				delete index$host;
-				}
-			}
-		
-		local metric_tbl = store[id, filter$name];
-		if ( index !in metric_tbl )
-			metric_tbl[index] = 0;
-		metric_tbl[index] += increment;
-		
-		data_added(filter, index, metric_tbl[index]);
-		}
-	}
-
-function check_notice(filter: Filter, index: Index, val: count): bool
-	{
-	if ( (filter?$notice_threshold &&
-	      [filter$id, filter$name, index] !in thresholds &&
-	      val >= filter$notice_threshold) ||
-	     (filter?$notice_thresholds &&
-	      |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
-	      val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
-		return T;
-	else
-		return F;
-	}
-		
-function do_notice(filter: Filter, index: Index, val: count)
-	{
-	# We include $peer_descr here because the a manager count have actually 
-	# generated the notice even though the current remote peer for the event 
-	# calling this could be a worker if this is running as a cluster.
-	local n: Notice::Info = [$note=filter$note, 
-	                         $n=val, 
-	                         $metric_index=index, 
-	                         $peer_descr=peer_description];
-	n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
-	if ( index?$str )
-		n$sub = index$str;
-	if ( index?$host )
-		n$src = index$host;
-	# TODO: not sure where to put the network yet.
-	
-	NOTICE(n);
-	
-	# This just needs set to some value so that it doesn't refire the 
-	# notice until it expires from the table or it crosses the next 
-	# threshold in the case of vectors of thresholds.
-	++thresholds[filter$id, filter$name, index];
-	}

scripts/base/frameworks/metrics/non-cluster.bro

@@ -1,21 +0,0 @@
-@load ./main
-
-module Metrics;
-
-event Metrics::log_it(filter: Filter)
-	{
-	local id = filter$id;
-	local name = filter$name;
-	
-	write_log(network_time(), filter, store[id, name]);
-	reset(filter);
-	
-	schedule filter$break_interval { Metrics::log_it(filter) };
-	}
-	
-	
-function data_added(filter: Filter, index: Index, val: count)
-	{
-	if ( check_notice(filter, index, val) )
-		do_notice(filter, index, val);
-	}

scripts/base/frameworks/sumstats/__load__.bro

@@ -1,4 +1,5 @@
 @load ./main
+@load ./plugins
 
 # The cluster framework must be loaded first.
 @load base/frameworks/cluster

scripts/base/frameworks/sumstats/cluster.bro

@@ -0,0 +1,346 @@
+##! This implements transparent cluster support for the SumStats framework.
+##! Do not load this file directly.  It's only meant to be loaded automatically
+##! and will be depending on if the cluster framework has been enabled.
+##! The goal of this script is to make sumstats calculation completely and
+##! transparently automated when running on a cluster.
+
+@load base/frameworks/cluster
+@load ./main
+
+module SumStats;
+
+export {
+	## Allows a user to decide how large of result groups the workers should transmit
+	## values for cluster stats aggregation.
+	const cluster_send_in_groups_of = 50 &redef;
+
+	## The percent of the full threshold value that needs to be met on a single worker
+	## for that worker to send the value to its manager in order for it to request a
+	## global view for that value.  There is no requirement that the manager requests
+	## a global view for the key since it may opt not to if it requested a global view
+	## for the key recently.
+	const cluster_request_global_view_percent = 0.2 &redef;
+
+	## This is to deal with intermediate update overload.  A manager will only allow
+	## this many intermediate update requests to the workers to be inflight at any
+	## given time.  Requested intermediate updates are currently thrown out and not
+	## performed.  In practice this should hopefully have a minimal effect.
+	const max_outstanding_global_views = 10 &redef;
+
+	## Intermediate updates can cause overload situations on very large clusters. This
+	## option may help reduce load and correct intermittent problems. The goal for this
+	## option is also meant to be temporary.
+	const enable_intermediate_updates = T &redef;
+
+	## Event sent by the manager in a cluster to initiate the collection of values for
+	## a sumstat.
+	global cluster_ss_request: event(uid: string, ssid: string);
+
+	## Event sent by nodes that are collecting sumstats after receiving a request for
+	## the sumstat from the manager.
+	global cluster_ss_response: event(uid: string, ssid: string, data: ResultTable, done: bool);
+
+	## This event is sent by the manager in a cluster to initiate the collection of
+	## a single key value from a sumstat.  It's typically used to get intermediate
+	## updates before the break interval triggers to speed detection of a value
+	## crossing a threshold.
+	global cluster_key_request: event(uid: string, ssid: string, key: Key);
+
+	## This event is sent by nodes in response to a
+	## :bro:id:`SumStats::cluster_key_request` event.
+	global cluster_key_response: event(uid: string, ssid: string, key: Key, result: Result);
+
+	## This is sent by workers to indicate that they crossed the percent
+	## of the current threshold by the percentage defined globally in
+	## :bro:id:`SumStats::cluster_request_global_view_percent`
+	global cluster_key_intermediate_response: event(ssid: string, key: SumStats::Key);
+
+	## This event is scheduled internally on workers to send result chunks.
+	global send_data: event(uid: string, ssid: string, data: ResultTable);
+
+	## This event is generated when a threshold is crossed.
+	global cluster_threshold_crossed: event(ssid: string, key: SumStats::Key, thold: Thresholding);
+}
+
+# Add events to the cluster framework to make this work.
+redef Cluster::manager2worker_events += /SumStats::cluster_(ss_request|key_request|threshold_crossed)/;
+redef Cluster::manager2worker_events += /SumStats::thresholds_reset/;
+redef Cluster::worker2manager_events += /SumStats::cluster_(ss_response|key_response|key_intermediate_response)/;
+
+@if ( Cluster::local_node_type() != Cluster::MANAGER )
+# This variable is maintained to know what keys have recently sent as
+# intermediate updates so they don't overwhelm their manager. The count that is
+# yielded is the number of times the percentage threshold has been crossed and
+# an intermediate result has been received.
+global recent_global_view_keys: table[string, Key] of count &create_expire=1min &default=0;
+
+event bro_init() &priority=-100
+	{
+	# The manager is the only host allowed to track these.
+	stats_store = table();
+	reducer_store = table();
+	}
+
+# This is done on all non-manager node types in the event that a sumstat is
+# being collected somewhere other than a worker.
+function data_added(ss: SumStat, key: Key, result: Result)
+	{
+	# If an intermediate update for this value was sent recently, don't send
+	# it again.
+	if ( [ss$id, key] in recent_global_view_keys )
+		return;
+
+	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
+	# crosses the full threshold then it's a candidate to send as an
+	# intermediate update.
+	if ( enable_intermediate_updates &&
+	     check_thresholds(ss, key, result, cluster_request_global_view_percent) )
+		{
+		# kick off intermediate update
+		event SumStats::cluster_key_intermediate_response(ss$id, key);
+		++recent_global_view_keys[ss$id, key];
+		}
+	}
+
+event SumStats::send_data(uid: string, ssid: string, data: ResultTable)
+	{
+	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
+
+	local local_data: ResultTable = table();
+	local num_added = 0;
+	for ( key in data )
+		{
+		local_data[key] = data[key];
+		delete data[key];
+
+		# Only send cluster_send_in_groups_of at a time.  Queue another
+		# event to send the next group.
+		if ( cluster_send_in_groups_of == ++num_added )
+			break;
+		}
+
+	local done = F;
+	# If data is empty, this sumstat is done.
+	if ( |data| == 0 )
+		done = T;
+
+	# Note: copy is needed to compensate serialization caching issue. This should be
+	# changed to something else later. 
+	event SumStats::cluster_ss_response(uid, ssid, copy(local_data), done);
+	if ( ! done )
+		schedule 0.01 sec { SumStats::send_data(uid, ssid, data) };
+	}
+
+event SumStats::cluster_ss_request(uid: string, ssid: string)
+	{
+	#print fmt("WORKER %s: received the cluster_ss_request event for %s.", Cluster::node, id);
+
+	# Initiate sending all of the data for the requested stats.
+	if ( ssid in result_store )
+		event SumStats::send_data(uid, ssid, result_store[ssid]);
+	else
+		event SumStats::send_data(uid, ssid, table());
+
+	# Lookup the actual sumstats and reset it, the reference to the data
+	# currently stored will be maintained internally by the send_data event.
+	if ( ssid in stats_store )
+		reset(stats_store[ssid]);
+	}
+
+event SumStats::cluster_key_request(uid: string, ssid: string, key: Key)
+	{
+	if ( ssid in result_store && key in result_store[ssid] )
+		{
+		#print fmt("WORKER %s: received the cluster_key_request event for %s=%s.", Cluster::node, key2str(key), data);
+
+		# Note: copy is needed to compensate serialization caching issue. This should be
+		# changed to something else later. 
+		event SumStats::cluster_key_response(uid, ssid, key, copy(result_store[ssid][key]));
+		}
+	else
+		{
+		# We need to send an empty response if we don't have the data so that the manager
+		# can know that it heard back from all of the workers.
+		event SumStats::cluster_key_response(uid, ssid, key, table());
+		}
+	}
+
+event SumStats::cluster_threshold_crossed(ssid: string, key: SumStats::Key, thold: Thresholding)
+	{
+	if ( ssid !in threshold_tracker )
+		threshold_tracker[ssid] = table();
+
+	threshold_tracker[ssid][key] = thold;
+	}
+
+event SumStats::thresholds_reset(ssid: string)
+	{
+	threshold_tracker[ssid] = table();
+	}
+
+@endif
+
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+
+# This variable is maintained by manager nodes as they collect and aggregate
+# results.
+# Index on a uid.
+global stats_results: table[string] of ResultTable &read_expire=1min;
+
+# This variable is maintained by manager nodes to track how many "dones" they
+# collected per collection unique id.  Once the number of results for a uid
+# matches the number of peer nodes that results should be coming from, the
+# result is written out and deleted from here.
+# Indexed on a uid.
+# TODO: add an &expire_func in case not all results are received.
+global done_with: table[string] of count &read_expire=1min &default=0;
+
+# This variable is maintained by managers to track intermediate responses as
+# they are getting a global view for a certain key.
+# Indexed on a uid.
+global key_requests: table[string] of Result &read_expire=1min;
+
+# This variable is maintained by managers to prevent overwhelming communication due
+# to too many intermediate updates.  Each sumstat is tracked separately so that
+# one won't overwhelm and degrade other quieter sumstats.
+# Indexed on a sumstat id.
+global outstanding_global_views: table[string] of count &default=0;
+
+const zero_time = double_to_time(0.0);
+# Managers handle logging.
+event SumStats::finish_epoch(ss: SumStat)
+	{
+	if ( network_time() > zero_time )
+		{
+		#print fmt("%.6f MANAGER: breaking %s sumstat for %s sumstat", network_time(), ss$name, ss$id);
+		local uid = unique_id("");
+
+		if ( uid in stats_results )
+			delete stats_results[uid];
+		stats_results[uid] = table();
+
+		# Request data from peers.
+		event SumStats::cluster_ss_request(uid, ss$id);
+		}
+
+	# Schedule the next finish_epoch event.
+	schedule ss$epoch { SumStats::finish_epoch(ss) };
+	}
+
+# This is unlikely to be called often, but it's here in
+# case there are sumstats being collected by managers.
+function data_added(ss: SumStat, key: Key, result: Result)
+	{
+	if ( check_thresholds(ss, key, result, 1.0) )
+		{
+		threshold_crossed(ss, key, result);
+		event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+		}
+	}
+
+event SumStats::cluster_key_response(uid: string, ssid: string, key: Key, result: Result)
+	{
+	#print fmt("%0.6f MANAGER: receiving key data from %s - %s=%s", network_time(), get_event_peer()$descr, key2str(key), result);
+
+	# We only want to try and do a value merge if there are actually measured datapoints
+	# in the Result.
+	if ( uid in key_requests )
+		key_requests[uid] = compose_results(key_requests[uid], result);
+	else
+		key_requests[uid] = result;
+
+	# Mark that a worker is done.
+	++done_with[uid];
+
+	#print fmt("worker_count:%d :: done_with:%d", Cluster::worker_count, done_with[uid]);
+	if ( Cluster::worker_count == done_with[uid] )
+		{
+		local ss = stats_store[ssid];
+		local ir = key_requests[uid];
+		if ( check_thresholds(ss, key, ir, 1.0) )
+			{
+			threshold_crossed(ss, key, ir);
+			event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+			}
+
+		delete done_with[uid];
+		delete key_requests[uid];
+		# Check that there is an outstanding view before subtracting.
+		if ( outstanding_global_views[ssid] > 0 )
+			--outstanding_global_views[ssid];
+		}
+	}
+
+# Managers handle intermediate updates here.
+event SumStats::cluster_key_intermediate_response(ssid: string, key: Key)
+	{
+	#print fmt("MANAGER: receiving intermediate key data from %s", get_event_peer()$descr);
+	#print fmt("MANAGER: requesting key data for %s", key2str(key));
+
+	if ( ssid in outstanding_global_views &&
+	     |outstanding_global_views[ssid]| > max_outstanding_global_views )
+		{
+		# Don't do this intermediate update.  Perhaps at some point in the future
+		# we will queue and randomly select from these ignored intermediate
+		# update requests.
+		return;
+		}
+
+	++outstanding_global_views[ssid];
+
+	local uid = unique_id("");
+	event SumStats::cluster_key_request(uid, ssid, key);
+	}
+
+event SumStats::cluster_ss_response(uid: string, ssid: string, data: ResultTable, done: bool)
+	{
+	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
+
+	# Mark another worker as being "done" for this uid.
+	if ( done )
+		++done_with[uid];
+
+	local local_data = stats_results[uid];
+	local ss = stats_store[ssid];
+
+	for ( key in data )
+		{
+		if ( key in local_data )
+			local_data[key] = compose_results(local_data[key], data[key]);
+		else
+			local_data[key] = data[key];
+
+		# If a stat is done being collected, thresholds for each key
+		# need to be checked so we're doing it here to avoid doubly
+		# iterating over each key.
+		if ( Cluster::worker_count == done_with[uid] )
+			{
+			if ( check_thresholds(ss, key, local_data[key], 1.0) )
+				{
+				threshold_crossed(ss, key, local_data[key]);
+				event SumStats::cluster_threshold_crossed(ss$id, key, threshold_tracker[ss$id][key]);
+				}
+			}
+		}
+
+	# If the data has been collected from all peers, we are done and ready to finish.
+	if ( Cluster::worker_count == done_with[uid] )
+		{
+		if ( ss?$epoch_finished )
+			ss$epoch_finished(local_data);
+
+		# Clean up
+		delete stats_results[uid];
+		delete done_with[uid];
+		# Not sure I need to reset the sumstat on the manager.
+		reset(ss);
+		}
+	}
+
+event remote_connection_handshake_done(p: event_peer) &priority=5
+	{
+	send_id(p, "SumStats::stats_store");
+	send_id(p, "SumStats::reducer_store");
+	}
+@endif

scripts/base/frameworks/sumstats/main.bro

@@ -0,0 +1,436 @@
+##! The summary statistics framework provides a way to
+##! summarize large streams of data into simple reduced
+##! measurements.
+
+module SumStats;
+
+export {
+	## The various calculations are all defined as plugins.
+	type Calculation: enum {
+		PLACEHOLDER
+	};
+
+	## Represents a thing which is having summarization
+	## results collected for it.
+	type Key: record {
+		## A non-address related summarization or a sub-key for
+		## an address based summarization. An example might be
+		## successful SSH connections by client IP address
+		## where the client string would be the key value.
+		## Another example might be number of HTTP requests to
+		## a particular value in a Host header.  This is an
+		## example of a non-host based metric since multiple
+		## IP addresses could respond for the same Host
+		## header value.
+		str:  string &optional;
+
+		## Host is the value to which this metric applies.
+		host: addr &optional;
+	};
+
+	## Represents data being added for a single observation.
+	## Only supply a single field at a time!
+	type Observation: record {
+		## Count value.
+		num:  count  &optional;
+		## Double value.
+		dbl:  double &optional;
+		## String value.
+		str:  string &optional;
+	};
+
+	type Reducer: record {
+		## Observation stream identifier for the reducer
+		## to attach to.
+		stream:         string;
+
+		## The calculations to perform on the data points.
+		apply:          set[Calculation];
+
+		## A predicate so that you can decide per key if you
+		## would like to accept the data being inserted.
+		pred:           function(key: SumStats::Key, obs: SumStats::Observation): bool &optional;
+
+		## A function to normalize the key.  This can be used to aggregate or
+		## normalize the entire key.
+		normalize_key:  function(key: SumStats::Key): Key &optional;
+	};
+
+	## Value calculated for an observation stream fed into a reducer.
+	## Most of the fields are added by plugins.
+	type ResultVal: record {
+		## The time when the first observation was added to
+		## this result value.
+		begin:  time;
+
+		## The time when the last observation was added to
+		## this result value.
+		end:    time;
+
+		## The number of observations received.
+		num:    count &default=0;
+	};
+
+	## Type to store results for multiple reducers.
+	type Result: table[string] of ResultVal;
+
+	## Type to store a table of sumstats results indexed
+	## by keys.
+	type ResultTable: table[Key] of Result;
+
+	## SumStats represent an aggregation of reducers along with
+	## mechanisms to handle various situations like the epoch ending
+	## or thresholds being crossed.
+	##
+	## It's best to not access any global state outside
+	## of the variables given to the callbacks because there
+	## is no assurance provided as to where the callbacks
+	## will be executed on clusters.
+	type SumStat: record {
+		## The interval at which this filter should be "broken"
+		## and the '$epoch_finished' callback called.  The
+		## results are also reset at this time so any threshold
+		## based detection needs to be set to a
+		## value that should be expected to happen within
+		## this epoch.
+		epoch:              interval;
+
+		## The reducers for the SumStat
+		reducers:           set[Reducer];
+
+		## Provide a function to calculate a value from the
+		## :bro:see:`Result` structure which will be used
+		## for thresholding.
+		## This is required if a $threshold value is given.
+		threshold_val:      function(key: SumStats::Key, result: SumStats::Result): count &optional;
+
+		## The threshold value for calling the
+		## $threshold_crossed callback.
+		threshold:          count             &optional;
+
+		## A series of thresholds for calling the
+		## $threshold_crossed callback.
+		threshold_series:   vector of count   &optional;
+
+		## A callback that is called when a threshold is crossed.
+		threshold_crossed:  function(key: SumStats::Key, result: SumStats::Result) &optional;
+
+		## A callback with the full collection of Results for
+		## this SumStat.
+		epoch_finished:    function(rt: SumStats::ResultTable) &optional;
+	};
+
+	## Create a summary statistic.
+	global create: function(ss: SumStats::SumStat);
+
+	## Add data into an observation stream. This should be
+	## called when a script has measured some point value.
+	##
+	## id: The observation stream identifier that the data
+	##     point represents.
+	##
+	## key: The key that the value is related to.
+	##
+	## obs: The data point to send into the stream.
+	global observe: function(id: string, key: SumStats::Key, obs: SumStats::Observation);
+
+	## This record is primarily used for internal threshold tracking.
+	type Thresholding: record {
+		# Internal use only.  Indicates if a simple threshold was already crossed.
+		is_threshold_crossed: bool &default=F;
+
+		# Internal use only.  Current key for threshold series.
+		threshold_series_index: count &default=0;
+	};
+
+	## This event is generated when thresholds are reset for a SumStat.
+	##
+	## ssid: SumStats ID that thresholds were reset for.
+	global thresholds_reset: event(ssid: string);
+
+	## Helper function to represent a :bro:type:`SumStats::Key` value as
+	## a simple string.
+	##
+	## key: The metric key that is to be converted into a string.
+	##
+	## Returns: A string representation of the metric key.
+	global key2str: function(key: SumStats::Key): string;
+}
+
+redef record Reducer += {
+	# Internal use only.  Provides a reference back to the related SumStats by it's ID.
+	sid: string &optional;
+};
+
+# Internal use only.  For tracking thresholds per sumstat and key.
+global threshold_tracker: table[string] of table[Key] of Thresholding &optional;
+
+redef record SumStat += {
+	# Internal use only (mostly for cluster coherency).
+	id: string &optional;
+};
+
+# Store of sumstats indexed on the sumstat id.
+global stats_store: table[string] of SumStat = table();
+
+# Store of reducers indexed on the data point stream id.
+global reducer_store: table[string] of set[Reducer] = table();
+
+# Store of results indexed on the measurement id.
+global result_store: table[string] of ResultTable = table();
+
+# Store of threshold information.
+global thresholds_store: table[string, Key] of bool = table();
+
+# This is called whenever key values are updated and the new val is given as the
+# `val` argument. It's only prototyped here because cluster and non-cluster have
+# separate  implementations.
+global data_added: function(ss: SumStat, key: Key, result: Result);
+
+# Prototype the hook point for plugins to do calculations.
+global observe_hook: hook(r: Reducer, val: double, data: Observation, rv: ResultVal);
+
+# Prototype the hook point for plugins to initialize any result values.
+global init_resultval_hook: hook(r: Reducer, rv: ResultVal);
+
+# Prototype the hook point for plugins to merge Results.
+global compose_resultvals_hook: hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal);
+
+# Event that is used to "finish" measurements and adapt the measurement
+# framework for clustered or non-clustered usage.
+global finish_epoch: event(ss: SumStat);
+
+function key2str(key: Key): string
+	{
+	local out = "";
+	if ( key?$host )
+		out = fmt("%shost=%s", out, key$host);
+	if ( key?$str )
+		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", key$str);
+	return fmt("sumstats_key(%s)", out);
+	}
+
+function init_resultval(r: Reducer): ResultVal
+	{
+	local rv: ResultVal = [$begin=network_time(), $end=network_time()];
+	hook init_resultval_hook(r, rv);
+	return rv;
+	}
+
+function compose_resultvals(rv1: ResultVal, rv2: ResultVal): ResultVal
+	{
+	local result: ResultVal;
+
+	result$begin = (rv1$begin < rv2$begin) ? rv1$begin : rv2$begin;
+	result$end = (rv1$end > rv2$end) ? rv1$end : rv2$end;
+	result$num = rv1$num + rv2$num;
+
+	# Run the plugin composition hooks.
+	hook compose_resultvals_hook(result, rv1, rv2);
+	return result;
+	}
+
+function compose_results(r1: Result, r2: Result): Result
+	{
+	local result: Result = table();
+
+	if ( |r1| > |r2| )
+		{
+		for ( data_id in r1 )
+			{
+			if ( data_id in r2 )
+				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
+			else
+				result[data_id] = r1[data_id];
+			}
+		}
+	else
+		{
+		for ( data_id in r2 )
+			{
+			if ( data_id in r1 )
+				result[data_id] = compose_resultvals(r1[data_id], r2[data_id]);
+			else
+				result[data_id] = r2[data_id];
+			}
+		}
+
+	return result;
+	}
+
+
+function reset(ss: SumStat)
+	{
+	if ( ss$id in result_store )
+		delete result_store[ss$id];
+
+	result_store[ss$id] = table();
+
+	if ( ss?$threshold || ss?$threshold_series )
+		{
+		threshold_tracker[ss$id] = table();
+		event SumStats::thresholds_reset(ss$id);
+		}
+	}
+
+function create(ss: SumStat)
+	{
+	if ( (ss?$threshold || ss?$threshold_series) && ! ss?$threshold_val )
+		{
+		Reporter::error("SumStats given a threshold with no $threshold_val function");
+		}
+
+	if ( ! ss?$id )
+		ss$id=unique_id("");
+	threshold_tracker[ss$id] = table();
+	stats_store[ss$id] = ss;
+
+	for ( reducer in ss$reducers )
+		{
+		reducer$sid = ss$id;
+		if ( reducer$stream !in reducer_store )
+			reducer_store[reducer$stream] = set();
+		add reducer_store[reducer$stream][reducer];
+		}
+
+	reset(ss);
+	schedule ss$epoch { SumStats::finish_epoch(ss) };
+	}
+
+function observe(id: string, key: Key, obs: Observation)
+	{
+	if ( id !in reducer_store )
+		return;
+
+	# Try to add the data to all of the defined reducers.
+	for ( r in reducer_store[id] )
+		{
+		if ( r?$normalize_key )
+			key = r$normalize_key(copy(key));
+
+		# If this reducer has a predicate, run the predicate
+		# and skip this key if the predicate return false.
+		if ( r?$pred && ! r$pred(key, obs) )
+			next;
+
+		local ss = stats_store[r$sid];
+
+		# If there is a threshold and no epoch_finished callback
+		# we don't need to continue counting since the data will
+		# never be accessed.  This was leading
+		# to some state management issues when measuring
+		# uniqueness.
+		# NOTE: this optimization could need removed in the
+		#       future if on demand access is provided to the
+		#       SumStats results.
+		if ( ! ss?$epoch_finished &&
+		     r$sid in threshold_tracker &&
+		     key in threshold_tracker[r$sid] &&
+		     ( ss?$threshold &&
+		       threshold_tracker[r$sid][key]$is_threshold_crossed ) ||
+		     ( ss?$threshold_series &&
+		       threshold_tracker[r$sid][key]$threshold_series_index+1 == |ss$threshold_series| ) )
+			next;
+
+		if ( r$sid !in result_store )
+			result_store[ss$id] = table();
+		local results = result_store[r$sid];
+
+		if ( key !in results )
+			results[key] = table();
+		local result = results[key];
+
+		if ( id !in result )
+			result[id] = init_resultval(r);
+		local result_val = result[id];
+
+		++result_val$num;
+		# Continually update the $end field.
+		result_val$end=network_time();
+
+		# If a string was given, fall back to 1.0 as the value.
+		local val = 1.0;
+		if ( obs?$num || obs?$dbl )
+			val = obs?$dbl ? obs$dbl : obs$num;
+
+		hook observe_hook(r, val, obs, result_val);
+		data_added(ss, key, result);
+		}
+	}
+
+# This function checks if a threshold has been crossed.  It is also used as a method to implement
+# mid-break-interval threshold crossing detection for cluster deployments.
+function check_thresholds(ss: SumStat, key: Key, result: Result, modify_pct: double): bool
+	{
+	if ( ! (ss?$threshold || ss?$threshold_series) )
+		return F;
+
+	# Add in the extra ResultVals to make threshold_vals easier to write.
+	if ( |ss$reducers| != |result| )
+		{
+		for ( reducer in ss$reducers )
+			{
+			if ( reducer$stream !in result )
+				result[reducer$stream] = init_resultval(reducer);
+			}
+		}
+
+	local watch = ss$threshold_val(key, result);
+
+	if ( modify_pct < 1.0 && modify_pct > 0.0 )
+		watch = double_to_count(floor(watch/modify_pct));
+
+	if ( ss$id !in threshold_tracker )
+		threshold_tracker[ss$id] = table();
+	local t_tracker = threshold_tracker[ss$id];
+
+	if ( key !in t_tracker )
+		{
+		local ttmp: Thresholding;
+		t_tracker[key] = ttmp;
+		}
+	local tt = t_tracker[key];
+
+	if ( ss?$threshold && ! tt$is_threshold_crossed && watch >= ss$threshold )
+		{
+		# Value crossed the threshold.
+		return T;
+		}
+
+	if ( ss?$threshold_series &&
+	     |ss$threshold_series| >= tt$threshold_series_index &&
+	     watch >= ss$threshold_series[tt$threshold_series_index] )
+		{
+		# A threshold series was given and the value crossed the next
+		# value in the series.
+		return T;
+		}
+
+	return F;
+	}
+
+function threshold_crossed(ss: SumStat, key: Key, result: Result)
+	{
+	# If there is no callback, there is no point in any of this.
+	if ( ! ss?$threshold_crossed )
+		return;
+
+	# Add in the extra ResultVals to make threshold_crossed callbacks easier to write.
+	if ( |ss$reducers| != |result| )
+		{
+		for ( reducer in ss$reducers )
+			{
+			if ( reducer$stream !in result )
+				result[reducer$stream] = init_resultval(reducer);
+			}
+		}
+
+	ss$threshold_crossed(key, result);
+	local tt = threshold_tracker[ss$id][key];
+	tt$is_threshold_crossed = T;
+
+	# Bump up to the next threshold series index if a threshold series is being used.
+	if ( ss?$threshold_series )
+		++tt$threshold_series_index;
+	}
+

scripts/base/frameworks/sumstats/non-cluster.bro

@@ -0,0 +1,24 @@
+@load ./main
+
+module SumStats;
+
+event SumStats::finish_epoch(ss: SumStat)
+	{
+	if ( ss$id in result_store )
+		{
+		local data = result_store[ss$id];
+		if ( ss?$epoch_finished )
+			ss$epoch_finished(data);
+
+		reset(ss);
+		}
+
+	schedule ss$epoch { SumStats::finish_epoch(ss) };
+	}
+
+
+function data_added(ss: SumStat, key: Key, result: Result)
+	{
+	if ( check_thresholds(ss, key, result, 1.0) )
+		threshold_crossed(ss, key, result);
+	}

scripts/base/frameworks/sumstats/plugins/__load__.bro

@@ -0,0 +1,9 @@
+@load ./average
+@load ./last
+@load ./max
+@load ./min
+@load ./sample
+@load ./std-dev
+@load ./sum
+@load ./unique
+@load ./variance

scripts/base/frameworks/sumstats/plugins/average.bro

@@ -0,0 +1,36 @@
+@load base/frameworks/sumstats/main
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Calculate the average of the values.
+		AVERAGE
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the average of all values.
+		average: double &optional;
+	};
+}
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( AVERAGE in r$apply )
+		{
+		if ( ! rv?$average )
+			rv$average = val;
+		else
+			rv$average += (val - rv$average) / rv$num;
+		}
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$average && rv2?$average )
+		result$average = ((rv1$average*rv1$num) + (rv2$average*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$average )
+		result$average = rv1$average;
+	else if ( rv2?$average )
+		result$average = rv2$average;
+	}

scripts/base/frameworks/sumstats/plugins/last.bro

@@ -0,0 +1,54 @@
+@load base/frameworks/sumstats
+@load base/utils/queue
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Keep last X observations in a queue
+		LAST
+	};
+
+	redef record Reducer += {
+		## number of elements to keep.
+		num_last_elements: count &default=0;
+	};
+
+	redef record ResultVal += {
+		## This is the queue where elements are maintained.  Use the
+		## :bro:see:`SumStats::get_elements` function to get a vector of the current element values.
+		last_elements: Queue::Queue &optional;
+	};
+
+	## Get a vector of element values from a ResultVal.
+	global get_last: function(rv: ResultVal): vector of Observation;
+}
+
+function get_last(rv: ResultVal): vector of Observation
+	{
+	local s: vector of Observation = vector();
+	if ( rv?$last_elements )
+		Queue::get_vector(rv$last_elements, s);
+	return s;
+	}
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( LAST in r$apply && r$num_last_elements > 0 )
+		{
+		if ( ! rv?$last_elements )
+			rv$last_elements = Queue::init([$max_len=r$num_last_elements]);
+		Queue::put(rv$last_elements, obs);
+		}
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	# Merge $samples
+	if ( rv1?$last_elements && rv2?$last_elements )
+		result$last_elements = Queue::merge(rv1$last_elements, rv2$last_elements);
+	else if ( rv1?$last_elements )
+		result$last_elements = rv1$last_elements;
+	else if ( rv2?$last_elements )
+		result$last_elements = rv2$last_elements;
+	}

scripts/base/frameworks/sumstats/plugins/max.bro

@@ -0,0 +1,38 @@
+@load base/frameworks/sumstats/main
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Find the maximum value.
+		MAX
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the maximum value given.
+		max: double &optional;
+	};
+}
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( MAX in r$apply )
+		{
+		if ( ! rv?$max )
+			rv$max = val;
+		else if ( val > rv$max )
+			rv$max = val;
+		}
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$max && rv2?$max )
+		result$max = (rv1$max > rv2$max) ? rv1$max : rv2$max;
+	else if ( rv1?$max )
+		result$max = rv1$max;
+	else if ( rv2?$max )
+		result$max = rv2$max;
+	}
+
+

scripts/base/frameworks/sumstats/plugins/min.bro

@@ -0,0 +1,36 @@
+@load base/frameworks/sumstats/main
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Find the minimum value.
+		MIN
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the minimum value given.
+		min: double &optional;
+	};
+}
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( MIN in r$apply )
+		{
+		if ( ! rv?$min )
+			rv$min = val;
+		else if ( val < rv$min )
+			rv$min = val;
+		}
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$min && rv2?$min )
+		result$min = (rv1$min < rv2$min) ? rv1$min : rv2$min;
+	else if ( rv1?$min )
+		result$min = rv1$min;
+	else if ( rv2?$min )
+		result$min = rv2$min;
+	}

scripts/base/frameworks/sumstats/plugins/sample.bro

@@ -0,0 +1,120 @@
+@load base/frameworks/sumstats/main
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Get uniquely distributed random samples from the observation stream.
+		SAMPLE
+	};
+
+	redef record Reducer += {
+		## A number of sample Observations to collect.
+		num_samples: count &default=0;
+	};
+
+	redef record ResultVal += {
+		## This is the vector in which the samples are maintained.
+		samples: vector of Observation &default=vector();
+
+		## Number of total observed elements.
+		sample_elements: count &default=0;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  This is not meant to be publically available
+	# and just a copy of num_samples from the Reducer. Needed for availability
+	# in the compose hook.
+	num_samples: count &default=0;
+};
+
+hook init_resultval_hook(r: Reducer, rv: ResultVal)
+	{
+	if ( SAMPLE in r$apply )
+		rv$num_samples = r$num_samples;
+	}
+
+function sample_add_sample(obs:Observation, rv: ResultVal)
+	{
+	++rv$sample_elements;
+
+	if ( |rv$samples| < rv$num_samples )
+		rv$samples[|rv$samples|] = obs;
+	else
+		{
+		local ra = rand(rv$sample_elements);
+		if ( ra < rv$num_samples )
+			rv$samples[ra] = obs;
+		}
+
+	}
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( SAMPLE in r$apply )
+		{
+		sample_add_sample(obs, rv);
+		}
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1$num_samples != rv2$num_samples )
+		{
+		Reporter::error("Merging sample sets with differing sizes is not supported");
+		return;
+		}
+
+	local num_samples = rv1$num_samples;
+	result$num_samples = num_samples;
+
+	if ( |rv1$samples| > num_samples || |rv2$samples| > num_samples )
+		{
+		Reporter::error("Sample vector with too many elements. Aborting.");
+		return;
+		}
+
+
+	if ( |rv1$samples| != num_samples && |rv2$samples| < num_samples )
+		{
+		if ( |rv1$samples| != rv1$sample_elements || |rv2$samples| < rv2$sample_elements )
+			{
+			Reporter::error("Mismatch in sample element size and tracking. Aborting merge");
+			return;
+			}
+
+		for ( i in rv1$samples )
+			sample_add_sample(rv1$samples[i], result);
+
+		for ( i in rv2$samples)
+			sample_add_sample(rv2$samples[i], result);
+		}
+	else
+		{
+		local other_vector: vector of Observation;
+		local othercount: count;
+		
+		if ( rv1$sample_elements > rv2$sample_elements )
+			{
+			result$samples = copy(rv1$samples);
+			other_vector = rv2$samples;
+			othercount = rv2$sample_elements;
+			}
+		else
+			{
+			result$samples = copy(rv2$samples);
+			other_vector = rv1$samples;
+			othercount = rv1$sample_elements;
+			}
+
+		local totalcount = rv1$sample_elements + rv2$sample_elements;
+		result$sample_elements = totalcount;
+
+		for ( i in other_vector )
+			{
+			if ( rand(totalcount) <= othercount )
+				result$samples[i] = other_vector[i];
+			}
+		}
+	}

scripts/base/frameworks/sumstats/plugins/std-dev.bro

@@ -0,0 +1,34 @@
+@load base/frameworks/sumstats/main
+@load ./variance
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Find the standard deviation of the values.
+		STD_DEV
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the standard deviation.
+		std_dev: double &default=0.0;
+	};
+}
+
+function calc_std_dev(rv: ResultVal)
+	{
+	if ( rv?$variance )
+		rv$std_dev = sqrt(rv$variance);
+	}
+
+# This depends on the variance plugin which uses priority -5
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-10
+	{
+	if ( STD_DEV in r$apply )
+		calc_std_dev(rv);
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-10
+	{
+	calc_std_dev(result);
+	}

scripts/base/frameworks/sumstats/plugins/sum.bro

@@ -0,0 +1,51 @@
+@load base/frameworks/sumstats/main
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Sums the values given.  For string values,
+		## this will be the number of strings given.
+		SUM
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this tracks the sum of all values.
+		sum: double &default=0.0;
+	};
+
+	type threshold_function: function(key: SumStats::Key, result: SumStats::Result): count;
+	global sum_threshold: function(data_id: string): threshold_function;
+}
+
+function sum_threshold(data_id: string): threshold_function
+	{
+	return function(key: SumStats::Key, result: SumStats::Result): count
+		{
+		print fmt("data_id: %s", data_id);
+		print result;
+		return double_to_count(result[data_id]$sum);
+		};
+	}
+
+hook init_resultval_hook(r: Reducer, rv: ResultVal)
+	{
+	if ( SUM in r$apply && ! rv?$sum )
+		rv$sum = 0;
+	}
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( SUM in r$apply )
+		rv$sum += val;
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$sum || rv2?$sum )
+		{
+		result$sum = rv1?$sum ? rv1$sum : 0;
+		if ( rv2?$sum )
+			result$sum += rv2$sum;
+		}
+	}

scripts/base/frameworks/sumstats/plugins/unique.bro

@@ -0,0 +1,53 @@
+@load base/frameworks/sumstats/main
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Calculate the number of unique values.
+		UNIQUE
+	};
+
+	redef record ResultVal += {
+		## If cardinality is being tracked, the number of unique
+		## items is tracked here.
+		unique: count &default=0;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  This is not meant to be publically available
+	# because we don't want to trust that we can inspect the values
+	# since we will like move to a probalistic data structure in the future.
+	# TODO: in the future this will optionally be a hyperloglog structure
+	unique_vals: set[Observation] &optional;
+};
+
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal)
+	{
+	if ( UNIQUE in r$apply )
+		{
+		if ( ! rv?$unique_vals )
+			rv$unique_vals=set();
+		add rv$unique_vals[obs];
+		rv$unique = |rv$unique_vals|;
+		}
+	}
+
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal)
+	{
+	if ( rv1?$unique_vals || rv2?$unique_vals )
+		{
+		if ( rv1?$unique_vals )
+			result$unique_vals = copy(rv1$unique_vals);
+
+		if ( rv2?$unique_vals )
+			if ( ! result?$unique_vals )
+				result$unique_vals = copy(rv2$unique_vals);
+			else
+				for ( val2 in rv2$unique_vals )
+					add result$unique_vals[copy(val2)];
+
+		result$unique = |result$unique_vals|;
+		}
+	}

scripts/base/frameworks/sumstats/plugins/variance.bro

@@ -0,0 +1,69 @@
+@load base/frameworks/sumstats/main
+@load ./average
+
+module SumStats;
+
+export {
+	redef enum Calculation += {
+		## Find the variance of the values.
+		VARIANCE
+	};
+
+	redef record ResultVal += {
+		## For numeric data, this calculates the variance.
+		variance: double &optional;
+	};
+}
+
+redef record ResultVal += {
+	# Internal use only.  Used for incrementally calculating variance.
+	prev_avg: double &optional;
+
+	# Internal use only.  For calculating incremental variance.
+	var_s: double &default=0.0;
+};
+
+function calc_variance(rv: ResultVal)
+	{
+	rv$variance = (rv$num > 1) ? rv$var_s/(rv$num-1) : 0.0;
+	}
+
+# Reduced priority since this depends on the average
+hook observe_hook(r: Reducer, val: double, obs: Observation, rv: ResultVal) &priority=-5
+	{
+	if ( VARIANCE in r$apply )
+		{
+		if ( rv$num > 1 )
+			rv$var_s += ((val - rv$prev_avg) * (val - rv$average));
+
+		calc_variance(rv);
+		rv$prev_avg = rv$average;
+		}
+	}
+
+# Reduced priority since this depends on the average
+hook compose_resultvals_hook(result: ResultVal, rv1: ResultVal, rv2: ResultVal) &priority=-5
+	{
+	if ( rv1?$var_s && rv1?$average &&
+	     rv2?$var_s && rv2?$average )
+		{
+		local rv1_avg_sq = (rv1$average - result$average);
+		rv1_avg_sq = rv1_avg_sq*rv1_avg_sq;
+		local rv2_avg_sq = (rv2$average - result$average);
+		rv2_avg_sq = rv2_avg_sq*rv2_avg_sq;
+		result$var_s = rv1$num*(rv1$var_s/rv1$num + rv1_avg_sq) + rv2$num*(rv2$var_s/rv2$num + rv2_avg_sq);
+		}
+	else if ( rv1?$var_s )
+		result$var_s = rv1$var_s;
+	else if ( rv2?$var_s )
+		result$var_s = rv2$var_s;
+
+	if ( rv1?$prev_avg && rv2?$prev_avg )
+		result$prev_avg = ((rv1$prev_avg*rv1$num) + (rv2$prev_avg*rv2$num))/(rv1$num+rv2$num);
+	else if ( rv1?$prev_avg )
+		result$prev_avg = rv1$prev_avg;
+	else if ( rv2?$prev_avg )
+		result$prev_avg = rv2$prev_avg;
+
+	calc_variance(result);
+	}

scripts/base/init-default.bro

@@ -12,8 +12,10 @@
 @load base/utils/numbers
 @load base/utils/paths
 @load base/utils/patterns
+@load base/utils/queue
 @load base/utils/strings
 @load base/utils/thresholds
+@load base/utils/time
 @load base/utils/urls
 
 # This has some deep interplay between types and BiFs so it's 
@@ -27,9 +29,9 @@
 @load base/frameworks/communication
 @load base/frameworks/control
 @load base/frameworks/cluster
-@load base/frameworks/metrics
 @load base/frameworks/intel
 @load base/frameworks/reporter
+@load base/frameworks/sumstats
 @load base/frameworks/tunnels
 
 @load base/protocols/conn

scripts/base/protocols/ftp/main.bro

@@ -70,10 +70,10 @@ export {
 		data_channel:     ExpectedDataChannel &log &optional;
 
 		## Current working directory that this session is in.  By making
-		## the default value '/.', we can indicate that unless something
+		## the default value '.', we can indicate that unless something
 		## more concrete is discovered that the existing but unknown
 		## directory is ok to use.
-		cwd:                string  &default="/.";
+		cwd:                string  &default=".";
 		
 		## Command that is currently waiting for a response.
 		cmdarg:             CmdArg  &optional;
@@ -187,7 +187,13 @@ function ftp_message(s: Info)
 		
 		local arg = s$cmdarg$arg;
 		if ( s$cmdarg$cmd in file_cmds )
-			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), build_path_compressed(s$cwd, arg));
+			{
+			local comp_path = build_path_compressed(s$cwd, arg);
+			if ( comp_path[0] != "/" )
+				comp_path = cat("/", comp_path);
+
+			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), comp_path);
+			}
 		
 		s$ts=s$cmdarg$ts;
 		s$command=s$cmdarg$cmd;
@@ -264,17 +270,14 @@ event ftp_request(c: connection, command: string, arg: string) &priority=5
 
 event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &priority=5
 	{
-	# TODO: figure out what to do with continued FTP response (not used much)
-	#if ( cont_resp ) return;
-
-	local id = c$id;
 	set_ftp_session(c);
-	
 	c$ftp$cmdarg = get_pending_cmd(c$ftp$pending_commands, code, msg);
-	
 	c$ftp$reply_code = code;
 	c$ftp$reply_msg = msg;
 
+	# TODO: figure out what to do with continued FTP response (not used much)
+	if ( cont_resp ) return;
+	
 	# TODO: do some sort of generic clear text login processing here.
 	local response_xyz = parse_ftp_reply_code(code);
 	#if ( response_xyz$x == 2 &&  # successful
@@ -302,9 +305,9 @@ event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool) &prior
 			c$ftp$passive=T;
 			
 			if ( code == 229 && data$h == [::] )
-				data$h = id$resp_h;
+				data$h = c$id$resp_h;
 
-			add_expected_data_channel(c$ftp, [$passive=T, $orig_h=id$orig_h,
+			add_expected_data_channel(c$ftp, [$passive=T, $orig_h=c$id$orig_h,
 			                          $resp_h=data$h, $resp_p=data$p]);
 			}
 		else

scripts/base/protocols/ssh/main.bro

@@ -5,6 +5,7 @@
 ##! Requires that :bro:id:`use_conn_size_analyzer` is set to T!  The heuristic
 ##! is not attempted if the connection size analyzer isn't enabled.
 
+@load base/protocols/conn
 @load base/frameworks/notice
 @load base/utils/site
 @load base/utils/thresholds
@@ -17,12 +18,6 @@ export {
 	## The SSH protocol logging stream identifier.
 	redef enum Log::ID += { LOG };
 
-	redef enum Notice::Type += { 
-		## Indicates that a heuristically detected "successful" SSH 
-		## authentication occurred.
-		Login 
-	};
-
 	type Info: record {
 		## Time when the SSH connection began.
 		ts:              time         &log;
@@ -30,9 +25,9 @@ export {
 		uid:             string       &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
-		## Indicates if the login was heuristically guessed to be "success"
+		## Indicates if the login was heuristically guessed to be "success",
-		## or "failure".
+		## "failure", or "undetermined".
-		status:          string       &log &optional;
+		status:          string       &log &default="undetermined";
 		## Direction of the connection.  If the client was a local host
 		## logging into an external host, this would be OUTBOUND. INBOUND
 		## would be set for the opposite situation.
@@ -54,12 +49,12 @@ export {
 
 	## The size in bytes of data sent by the server at which the SSH
 	## connection is presumed to be successful.
-	const authentication_data_size = 5500 &redef;
+	const authentication_data_size = 4000 &redef;
 
 	## If true, we tell the event engine to not look at further data
 	## packets after the initial SSH handshake. Helps with performance
 	## (especially with large file transfers) but precludes some
-	## kinds of analyses (e.g., tracking connection size).
+	## kinds of analyses.
 	const skip_processing_after_detection = F &redef;
 
 	## Event that is generated when the heuristic thinks that a login
@@ -104,55 +99,61 @@ function set_session(c: connection)
 
 function check_ssh_connection(c: connection, done: bool)
 	{
-	# If done watching this connection, just return.
+	# If already done watching this connection, just return.
 	if ( c$ssh$done )
 		return;
 
-	# Make sure conn_size_analyzer is active by checking 
+	if ( done )
-	# resp$num_bytes_ip.  In general it should always be active though.
+		{
-	if ( ! c$resp?$num_bytes_ip )
+		# If this connection is done, then we can look to see if
-		return;
+		# this matches the conditions for a failed login.  Failed
+		# logins are only detected at connection state removal.
 
-	# Remove the IP and TCP header length from the total size.
+		if ( # Require originators to have sent at least 50 bytes.
-	# TODO: Fix for IPv6.  This whole approach also seems to break in some 
+		     c$orig$size > 50 &&
-	#       cases where there are more header bytes than num_bytes_ip.
+		     # Responders must be below 4000 bytes.
-	local header_bytes = c$resp$num_pkts*32 + c$resp$num_pkts*20;
+		     c$resp$size < 4000 &&
-	local server_bytes = c$resp$num_bytes_ip;
+		     # Responder must have sent fewer than 40 packets.
-	if ( server_bytes >= header_bytes )
+		     c$resp$num_pkts < 40 &&
-		server_bytes = server_bytes - header_bytes;
+		     # If there was a content gap we can't reliably do this heuristic.
-	else
+		     c?$conn && c$conn$missed_bytes == 0)# &&
-		server_bytes = c$resp$size;
+		     # Only "normal" connections can count.
-	
+		     #c$conn?$conn_state && c$conn$conn_state in valid_states )
-	# If this is still a live connection and the byte count has not crossed 
-	# the threshold, just return and let the rescheduled check happen later.
-	if ( ! done && server_bytes < authentication_data_size )
-		return;
-
-	# Make sure the server has sent back more than 50 bytes to filter out
-	# hosts that are just port scanning.  Nothing is ever logged if the server
-	# doesn't send back at least 50 bytes.
-	if ( server_bytes < 50 )
-		return;
-
-	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
-	c$ssh$resp_size = server_bytes;
-	
-	if ( server_bytes < authentication_data_size )
 			{
 			c$ssh$status = "failure";
 			event SSH::heuristic_failed_login(c);
 			}
-	else
+
+		if ( c$resp$size > authentication_data_size )
 			{
-		# presumed successful login
 			c$ssh$status = "success";
 			event SSH::heuristic_successful_login(c);
 			}
+		}
+	else
+		{
+		# If this connection is still being tracked, then it's possible
+		# to watch for it to be a successful connection.
+		if ( c$resp$size > authentication_data_size )
+			{
+			c$ssh$status = "success";
+			event SSH::heuristic_successful_login(c);
+			}
+		else
+			# This connection must be tracked longer.  Let the scheduled
+			# check happen again.
+			return;
+		}
+
+	# Set the direction for the log.
+	c$ssh$direction = Site::is_local_addr(c$id$orig_h) ? OUTBOUND : INBOUND;
 
 	# Set the "done" flag to prevent the watching event from rescheduling
 	# after detection is done.
 	c$ssh$done=T;
 
+	Log::write(SSH::LOG, c$ssh);
+
 	if ( skip_processing_after_detection )
 		{
 		# Stop watching this connection, we don't care about it anymore.
@@ -161,18 +162,6 @@ function check_ssh_connection(c: connection, done: bool)
 		}
 	}
 
-event SSH::heuristic_successful_login(c: connection) &priority=-5
-	{
-	NOTICE([$note=Login, 
-	        $msg="Heuristically detected successful SSH login.",
-	        $conn=c]);
-	
-	Log::write(SSH::LOG, c$ssh);
-	}
-event SSH::heuristic_failed_login(c: connection) &priority=-5
-	{
-	Log::write(SSH::LOG, c$ssh);
-	}
 
 event connection_state_remove(c: connection) &priority=-5
 	{

scripts/base/utils/paths.bro

@@ -19,7 +19,7 @@ function extract_path(input: string): string
 	}
 
 ## Compresses a given path by removing '..'s and the parent directory it
-## references and also removing '/'s.
+## references and also removing dual '/'s and extraneous '/./'s.
 ## dir: a path string, either relative or absolute
 ## Returns: a compressed version of the input path
 function compress_path(dir: string): string
@@ -41,7 +41,7 @@ function compress_path(dir: string): string
 		return compress_path(dir);
 		}
 
-	const multislash_sep = /(\/){2,}/;
+	const multislash_sep = /(\/\.?){2,}/;
 	parts = split_all(dir, multislash_sep);
 	for ( i in parts )
 		if ( i % 2 == 0 )

scripts/base/utils/queue.bro

@@ -0,0 +1,143 @@
+##! A FIFO queue.
+
+module Queue;
+
+export {
+	## Settings for initializing the queue.
+	type Settings: record {
+		## If a maximum length is set for the queue
+		## it will maintain itself at that
+		## maximum length automatically.
+		max_len: count &optional;
+	};
+
+	## The internal data structure for the queue.
+	type Queue: record {};
+
+	## Initialize a queue record structure.
+	##
+	## s: A :bro:record:`Settings` record configuring the queue.
+	##
+	## Returns: An opaque queue record.
+	global init:       function(s: Settings): Queue;
+
+	## Put a string onto the beginning of a queue.
+	##
+	## q: The queue to put the value into.
+	##
+	## val: The value to insert into the queue.
+	global put:       function(q: Queue, val: any);
+
+	## Get a string from the end of a queue.
+	##
+	## q: The queue to get the string from.
+	##
+	## Returns: The value gotten from the queue.
+	global get:        function(q: Queue): any;
+
+	## Merge two queue's together.  If any settings are applied
+	## to the queues, the settings from q1 are used for the new
+	## merged queue.
+	##
+	## q1: The first queue.  Settings are taken from here.
+	##
+	## q2: The second queue.
+	##
+	## Returns: A new queue from merging the other two together.
+	global merge:      function(q1: Queue, q2: Queue): Queue;
+
+	## Get the number of items in a queue.
+	##
+	## q: The queue.
+	##
+	## Returns: The length of the queue.
+	global len:     function(q: Queue): count;
+
+	## Get the contents of the queue as a vector.
+	##
+	## q: The queue.
+	##
+	## ret: A vector containing the
+	##      current contents of q as the type of ret.
+	global get_vector: function(q: Queue, ret: vector of any);
+
+}
+
+redef record Queue += {
+	# Indicator for if the queue was appropriately initialized.
+	initialized: bool                   &default=F;
+	# The values are stored here.
+	vals:        table[count] of any &optional;
+	# Settings for the queue.
+	settings:    Settings               &optional;
+	# The top value in the vals table.
+	top:         count                  &default=0;
+	# The bottom value in the vals table.
+	bottom:      count                  &default=0;
+	# The number of bytes in the queue.
+	size:        count                  &default=0;
+};
+
+function init(s: Settings): Queue
+	{
+	local q: Queue;
+	q$vals=table();
+	q$settings = copy(s);
+	q$initialized=T;
+	return q;
+	}
+
+function put(q: Queue, val: any)
+	{
+	if ( q$settings?$max_len && len(q) >= q$settings$max_len )
+		get(q);
+	q$vals[q$top] = val;
+	++q$top;
+	}
+
+function get(q: Queue): any
+	{
+	local ret = q$vals[q$bottom];
+	delete q$vals[q$bottom];
+	++q$bottom;
+	return ret;
+	}
+
+function merge(q1: Queue, q2: Queue): Queue
+	{
+	local ret = init(q1$settings);
+	local i = q1$bottom;
+	local j = q2$bottom;
+	for ( ignored_val in q1$vals )
+		{
+		if ( i in q1$vals )
+			put(ret, q1$vals[i]);
+		if ( j in q2$vals )
+			put(ret, q2$vals[j]);
+		++i;
+		++j;
+		}
+	return ret;
+	}
+
+function len(q: Queue): count
+	{
+	return |q$vals|;
+	}
+
+function get_vector(q: Queue, ret: vector of any)
+	{
+	local i = q$bottom;
+	local j = 0;
+	# Really dumb hack, this is only to provide
+	# the iteration for the correct number of
+	# values in q$vals.
+	for ( ignored_val in q$vals )
+		{
+		if ( i >= q$top )
+			break;
+
+		ret[j] = q$vals[i];
+		++j; ++i;
+		}
+	}

scripts/base/utils/time.bro

@@ -0,0 +1,9 @@
+
+## Given an interval, returns a string of the form 3m34s to
+## give a minimalized human readable string for the minutes
+## and seconds represented by the interval.
+function duration_to_mins_secs(dur: interval): string
+	{
+	local dur_count = double_to_count(interval_to_double(dur));
+	return fmt("%dm%ds", dur_count/60, dur_count%60);
+	}

scripts/policy/frameworks/metrics/conn-example.bro

@@ -1,25 +0,0 @@
-##! An example of using the metrics framework to collect connection metrics 
-##! aggregated into /24 CIDR ranges.
-
-@load base/frameworks/metrics
-@load base/utils/site
-
-redef enum Metrics::ID += { 
-	CONNS_ORIGINATED, 
-	CONNS_RESPONDED 
-};
-
-event bro_init()
-	{
-	Metrics::add_filter(CONNS_ORIGINATED, [$aggregation_mask=24, $break_interval=1mins]);
-	
-	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter(CONNS_RESPONDED,  [$aggregation_table=Site::local_nets_table, $break_interval=1mins]);
-	}
-
-event connection_established(c: connection)
-	{
-	Metrics::add_data(CONNS_ORIGINATED, [$host=c$id$orig_h], 1);
-	Metrics::add_data(CONNS_RESPONDED,  [$host=c$id$resp_h], 1);
-	}
-	

scripts/policy/frameworks/metrics/http-example.bro

@@ -1,37 +0,0 @@
-##! Provides an example of aggregating and limiting collection down to 
-##! only local networks.  Additionally, the status code for the response from
-##! the request is added into the metric.
-
-@load base/frameworks/metrics
-@load base/protocols/http
-@load base/utils/site
-
-redef enum Metrics::ID += {
-	## Measures HTTP requests indexed on both the request host and the response
-	## code from the server.
-	HTTP_REQUESTS_BY_STATUS_CODE,
-
-	## Currently unfinished and not working.
-	HTTP_REQUESTS_BY_HOST_HEADER,
-};
-
-event bro_init()
-	{
-	# TODO: these are waiting on a fix with table vals + records before they will work.
-	#Metrics::add_filter(HTTP_REQUESTS_BY_HOST_HEADER, 
-	#                    [$pred(index: Metrics::Index) = { return Site::is_local_addr(index$host); },
-	#                     $aggregation_mask=24,
-	#                     $break_interval=1min]);
-	
-	# Site::local_nets must be defined in order for this to actually do anything.
-	Metrics::add_filter(HTTP_REQUESTS_BY_STATUS_CODE, [$aggregation_table=Site::local_nets_table,
-	                                                   $break_interval=1min]);
-	}
-
-event HTTP::log_http(rec: HTTP::Info)
-	{
-	if ( rec?$host )
-		Metrics::add_data(HTTP_REQUESTS_BY_HOST_HEADER, [$str=rec$host], 1);
-	if ( rec?$status_code )
-		Metrics::add_data(HTTP_REQUESTS_BY_STATUS_CODE, [$host=rec$id$orig_h, $str=fmt("%d", rec$status_code)], 1);
-	}

scripts/policy/frameworks/metrics/ssl-example.bro

@@ -1,28 +0,0 @@
-##! Provides an example of using the metrics framework to collect the number
-##! of times a specific server name indicator value is seen in SSL session
-##! establishments.  Names ending in google.com are being filtered out as an
-##! example of the predicate based filtering in metrics filters.
-
-@load base/frameworks/metrics
-@load base/protocols/ssl
-
-redef enum Metrics::ID += {
-	SSL_SERVERNAME,
-};
-
-event bro_init()
-	{
-	Metrics::add_filter(SSL_SERVERNAME, 
-		[$name="no-google-ssl-servers",
-		 $pred(index: Metrics::Index) = { 
-		    return (/google\.com$/ !in index$str); 
-		 },
-		 $break_interval=10secs
-		]);
-	}
-
-event SSL::log_ssl(rec: SSL::Info)
-	{
-	if ( rec?$server_name )
-		Metrics::add_data(SSL_SERVERNAME, [$str=rec$server_name], 1);
-	}

scripts/policy/misc/app-metrics.bro

@@ -0,0 +1,109 @@
+@load base/protocols/http
+@load base/protocols/ssl
+@load base/frameworks/sumstats
+
+module AppStats;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The name of the "app", like "facebook" or "netflix".
+		app:        string &log;
+		## The number of unique local hosts using the app.
+		uniq_hosts: count  &log;
+		## The number of hits to the app in total.
+		hits:       count  &log;
+		## The total number of bytes received by users of the app.
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 15mins &redef;
+}
+
+redef record connection += {
+	resp_hostname: string &optional;
+};
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(AppStats::LOG, [$columns=Info]);
+
+	local r1: SumStats::Reducer = [$stream="apps.bytes", $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="apps.hits",  $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_finished(data: SumStats::ResultTable) =
+	                  	{
+	                  	local l: Info;
+	                  	l$ts = network_time();
+	                  	l$ts_delta = break_interval;
+	                  	for ( key in data )
+	                  		{
+	                  		local result = data[key];
+	                  		l$app        = key$str;
+	                  		l$bytes      = double_to_count(floor(result["apps.bytes"]$sum));
+	                  		l$hits       = result["apps.hits"]$num;
+	                  		l$uniq_hosts = result["apps.hits"]$unique;
+	                  		Log::write(LOG, l);
+	                  		}
+	                  	}]);
+	}
+
+function add_sumstats(id: conn_id, hostname: string, size: count)
+	{
+	if ( /\.youtube\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="youtube"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="youtube"], [$str=cat(id$orig_h)]);
+		}
+	else if ( /(\.facebook\.com|\.fbcdn\.net)$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="facebook"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="facebook"], [$str=cat(id$orig_h)]);
+		}
+	else if ( /\.google\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="google"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="google"], [$str=cat(id$orig_h)]);
+		}
+	else if ( /\.nflximg\.com$/ in hostname && size > 200*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="netflix"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="netflix"], [$str=cat(id$orig_h)]);
+		}
+	else if ( /\.(pandora|p-cdn)\.com$/ in hostname && size > 512*1024 )
+		{
+		SumStats::observe("apps.bytes", [$str="pandora"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="pandora"], [$str=cat(id$orig_h)]);
+		}
+	else if ( /\.gmail\.com$/ in hostname && size > 20 )
+		{
+		SumStats::observe("apps.bytes", [$str="gmail"], [$num=size]);
+		SumStats::observe("apps.hits",  [$str="gmail"], [$str=cat(id$orig_h)]);
+		}
+}
+
+
+event ssl_established(c: connection)
+	{
+	if ( c?$ssl && c$ssl?$server_name )
+		c$resp_hostname = c$ssl$server_name;
+	}
+
+event connection_finished(c: connection)
+	{
+	if ( c?$resp_hostname )
+		add_sumstats(c$id, c$resp_hostname, c$resp$size);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if( rec?$host )
+		add_sumstats(rec$id, rec$host, rec$response_body_len);
+	}

scripts/policy/misc/capture-loss.bro

@@ -8,7 +8,6 @@
 ##! for a sequence number that's above a gap).
 
 @load base/frameworks/notice
-@load base/frameworks/metrics
 
 module CaptureLoss;
 

scripts/policy/misc/detect-traceroute/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/policy/misc/detect-traceroute/detect-low-ttls.sig

@@ -0,0 +1,9 @@
+signature traceroute-detector-ipv4 {
+	header ip[8] < 10
+	event "match"
+}
+
+signature traceroute-detector-ipv6 {
+	header ip6[7] < 10
+	event "match"
+}

scripts/policy/misc/detect-traceroute/main.bro

@@ -0,0 +1,98 @@
+##! This script detects a large number of ICMP Time Exceeded messages heading toward
+##! hosts that have sent low TTL packets. It generates a notice when the number of
+##! ICMP Time Exceeded messages for a source-destination pair exceeds a
+##! threshold.
+@load base/frameworks/sumstats
+@load base/frameworks/signatures
+@load-sigs ./detect-low-ttls.sig
+
+redef Signatures::ignored_ids += /traceroute-detector.*/;
+
+module Traceroute;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	redef enum Notice::Type += {
+		## Indicates that a host was seen running traceroutes.  For more
+		## detail about specific traceroutes that we run, refer to the
+		## traceroute.log.
+		Detected
+	};
+
+	## By default this script requires that any host detected running traceroutes
+	## first send low TTL packets (TTL < 10) to the traceroute destination host.
+	## Changing this this setting to `F` will relax the detection a bit by
+	## solely relying on ICMP time-exceeded messages to detect traceroute.
+	const require_low_ttl_packets = T &redef;
+
+	## Defines the threshold for ICMP Time Exceeded messages for a src-dst pair.
+	## This threshold only comes into play after a host is found to be
+	## sending low ttl packets.
+	const icmp_time_exceeded_threshold = 3 &redef;
+
+	## Interval at which to watch for the
+	## :bro:id:`ICMPTimeExceeded::icmp_time_exceeded_threshold` variable to be crossed.
+	## At the end of each interval the counter is reset.
+	const icmp_time_exceeded_interval = 3min &redef;
+
+	## The log record for the traceroute log.
+	type Info: record {
+		## Timestamp
+		ts:    time &log;
+		## Address initiaing the traceroute.
+		src:   addr &log;
+		## Destination address of the traceroute.
+		dst:   addr &log;
+		## Protocol used for the traceroute.
+		proto: string &log;
+	};
+
+	global log_traceroute: event(rec: Traceroute::Info);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Traceroute::LOG, [$columns=Info, $ev=log_traceroute]);
+
+	local r1: SumStats::Reducer = [$stream="traceroute.time_exceeded", $apply=set(SumStats::UNIQUE)];
+	local r2: SumStats::Reducer = [$stream="traceroute.low_ttl_packet", $apply=set(SumStats::SUM)];
+	SumStats::create([$epoch=icmp_time_exceeded_interval,
+	                  $reducers=set(r1, r2),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	# Give a threshold value of zero depending on if the host
+	                  	# sends a low ttl packet.
+	                  	if ( require_low_ttl_packets && result["traceroute.low_ttl_packet"]$sum == 0 )
+	                  		return 0;
+	                  	else
+	                  		return result["traceroute.time_exceeded"]$unique;
+	                  	},
+	                  $threshold=icmp_time_exceeded_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local parts = split_n(key$str, /-/, F, 2);
+	                  	local src = to_addr(parts[1]);
+	                  	local dst = to_addr(parts[2]);
+	                  	local proto = parts[3];
+	                  	Log::write(LOG, [$ts=network_time(), $src=src, $dst=dst, $proto=proto]);
+	                  	NOTICE([$note=Traceroute::Detected,
+	                  	        $msg=fmt("%s seems to be running traceroute using %s", src, proto),
+	                  	        $src=src,
+	                  	        $identifier=cat(src,proto)]);
+	                  	}]);
+	}
+
+# Low TTL packets are detected with a signature.
+event signature_match(state: signature_state, msg: string, data: string)
+	{
+	if ( state$sig_id == /traceroute-detector.*/ )
+		{
+		SumStats::observe("traceroute.low_ttl_packet", [$str=cat(state$conn$id$orig_h,"-",state$conn$id$resp_h,"-",get_port_transport_proto(state$conn$id$resp_p))], [$num=1]);
+		}
+	}
+
+event icmp_time_exceeded(c: connection, icmp: icmp_conn, code: count, context: icmp_context)
+	{
+	SumStats::observe("traceroute.time_exceeded", [$str=cat(context$id$orig_h,"-",context$id$resp_h,"-",get_port_transport_proto(context$id$resp_p))], [$str=cat(c$id$orig_h)]);
+	}

scripts/policy/misc/loaded-scripts.bro

@@ -1,4 +1,5 @@
 ##! Log the loaded scripts.
+@load base/utils/paths
 
 module LoadedScripts;
 
@@ -34,5 +35,5 @@ event bro_init() &priority=5
 
 event bro_script_loaded(path: string, level: count)
 	{
-	Log::write(LoadedScripts::LOG, [$name=cat(depth[level], path)]);
+	Log::write(LoadedScripts::LOG, [$name=cat(depth[level], compress_path(path))]);
 	}

scripts/policy/misc/scan.bro

@@ -0,0 +1,193 @@
+##! TCP Scan detection
+##!
+##! ..Authors: Sheharbano Khattak
+##!            Seth Hall
+##!            All the authors of the old scan.bro
+
+@load base/frameworks/notice
+@load base/frameworks/sumstats
+
+@load base/utils/time
+
+module Scan;
+
+export {
+	redef enum Notice::Type += {
+		## Address scans detect that a host appears to be scanning some number of
+		## destinations on a single port. This notice is generated when more than
+		## :bro:id:`addr_scan_threshold` unique hosts are seen over the previous
+		## :bro:id:`addr_scan_interval` time range.
+		Address_Scan,
+
+		## Port scans detect that an attacking host appears to be scanning a
+		## single victim host on several ports.  This notice is generated when
+		## an attacking host attempts to connect to :bro:id:`port_scan_threshold`
+		## unique ports on a single host over the previous
+		## :bro:id:`port_scan_interval` time range.
+		Port_Scan,
+	};
+
+	## Failed connection attempts are tracked over this time interval for the address
+	## scan detection.  A higher interval will detect slower scanners, but may also
+	## yield more false positives.
+	const addr_scan_interval = 5min &redef;
+
+	## Failed connection attempts are tracked over this time interval for the port scan
+	## detection.  A higher interval will detect slower scanners, but may also yield
+	## more false positives.
+	const port_scan_interval = 5min &redef;
+
+	## The threshold of a unique number of hosts a scanning host has to have failed
+	## connections with on a single port.
+	const addr_scan_threshold = 25 &redef;
+
+	## The threshold of a number of unique ports a scanning host has to have failed
+	## connections with on a single victim host.
+	const port_scan_threshold = 15 &redef;
+
+	## Custom thresholds based on service for address scan.  This is primarily
+	## useful for setting reduced thresholds for specific ports.
+	const addr_scan_custom_thresholds: table[port] of count &redef;
+
+	global Scan::addr_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
+	global Scan::port_scan_policy: hook(scanner: addr, victim: addr, scanned_port: port);
+}
+
+event bro_init() &priority=5
+	{
+	local r1: SumStats::Reducer = [$stream="scan.addr.fail", $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=addr_scan_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return double_to_count(result["scan.addr.fail"]$unique);
+	                  	},
+	                  #$threshold_func=check_addr_scan_threshold,
+	                  $threshold=addr_scan_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["scan.addr.fail"];
+	                  	local side = Site::is_local_addr(key$host) ? "local" : "remote";
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local message=fmt("%s scanned at least %d unique hosts on port %s in %s", key$host, r$unique, key$str, dur);
+	                  	NOTICE([$note=Address_Scan,
+	                  	        $src=key$host,
+	                  	        $p=to_port(key$str),
+	                  	        $sub=side,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+
+	# Note: port scans are tracked similar to: table[src_ip, dst_ip] of set(port);
+	local r2: SumStats::Reducer = [$stream="scan.port.fail", $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=port_scan_interval,
+	                  $reducers=set(r2),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return double_to_count(result["scan.port.fail"]$unique);
+	                  	},
+	                  $threshold=port_scan_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["scan.port.fail"];
+	                  	local side = Site::is_local_addr(key$host) ? "local" : "remote";
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local message = fmt("%s scanned at least %d unique ports of host %s in %s", key$host, r$unique, key$str, dur);
+	                  	NOTICE([$note=Port_Scan,
+	                  	        $src=key$host,
+	                  	        $dst=to_addr(key$str),
+	                  	        $sub=side,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+	}
+
+function add_sumstats(id: conn_id, reverse: bool)
+	{
+	local scanner      = id$orig_h;
+	local victim       = id$resp_h;
+	local scanned_port = id$resp_p;
+
+	if ( reverse )
+		{
+		scanner      = id$resp_h;
+		victim       = id$orig_h;
+		scanned_port = id$orig_p;
+		}
+
+	if ( hook Scan::addr_scan_policy(scanner, victim, scanned_port) )
+		SumStats::observe("scan.addr.fail", [$host=scanner, $str=cat(scanned_port)], [$str=cat(victim)]);
+
+	if ( hook Scan::port_scan_policy(scanner, victim, scanned_port) )
+		SumStats::observe("scan.port.fail", [$host=scanner, $str=cat(victim)], [$str=cat(scanned_port)]);
+	}
+
+function is_failed_conn(c: connection): bool
+	{
+	# Sr || ( (hR || ShR) && (data not sent in any direction) )
+	if ( (c$orig$state == TCP_SYN_SENT && c$resp$state == TCP_RESET) ||
+	     (((c$orig$state == TCP_RESET && c$resp$state == TCP_SYN_ACK_SENT) ||
+	       (c$orig$state == TCP_RESET && c$resp$state == TCP_ESTABLISHED && "S" in c$history )
+	      ) && /[Dd]/ !in c$history )
+	   )
+		return T;
+	return F;
+	}
+
+function is_reverse_failed_conn(c: connection): bool
+	{
+	# reverse scan i.e. conn dest is the scanner
+	# sR || ( (Hr || sHr) && (data not sent in any direction) )
+	if ( (c$resp$state == TCP_SYN_SENT && c$orig$state == TCP_RESET) ||
+	     (((c$resp$state == TCP_RESET && c$orig$state == TCP_SYN_ACK_SENT) ||
+	       (c$resp$state == TCP_RESET && c$orig$state == TCP_ESTABLISHED && "s" in c$history )
+	      ) && /[Dd]/ !in c$history )
+	   )
+		return T;
+	return F;
+	}
+
+## Generated for an unsuccessful connection attempt. This
+## event is raised when an originator unsuccessfully attempted
+## to establish a connection. “Unsuccessful” is defined as at least
+## tcp_attempt_delay seconds having elapsed since the originator first sent a
+## connection establishment packet to the destination without seeing a reply.
+event connection_attempt(c: connection)
+	{
+	local is_reverse_scan = F;
+	if ( "H" in c$history )
+		is_reverse_scan = T;
+
+	add_sumstats(c$id, is_reverse_scan);
+	}
+
+## Generated for a rejected TCP connection. This event is raised when an originator
+## attempted to setup a TCP connection but the responder replied with a RST packet
+## denying it.
+event connection_rejected(c: connection)
+	{
+	local is_reverse_scan = F;
+	if ( "s" in c$history )
+		is_reverse_scan = T;
+
+	add_sumstats(c$id, is_reverse_scan);
+	}
+
+## Generated when an endpoint aborted a TCP connection. The event is raised when
+## one endpoint of an *established* TCP connection aborted by sending a RST packet.
+event connection_reset(c: connection)
+	{
+	if ( is_failed_conn(c) )
+		add_sumstats(c$id, F);
+	else if ( is_reverse_failed_conn(c) )
+		add_sumstats(c$id, T);
+	}
+
+## Generated for each still-open connection when Bro terminates.
+event connection_pending(c: connection)
+	{
+	if ( is_failed_conn(c) )
+		add_sumstats(c$id, F);
+	else if ( is_reverse_failed_conn(c) )
+		add_sumstats(c$id, T);
+	}

scripts/policy/protocols/ftp/detect-bruteforcing.bro

@@ -0,0 +1,59 @@
+##! FTP brute-forcing detector, triggering when too  many rejected usernames or
+##! failed passwords have occured from a single address.
+
+@load base/protocols/ftp
+@load base/frameworks/sumstats
+
+@load base/utils/time
+
+module FTP;
+
+export {
+	redef enum Notice::Type += {
+		## Indicates a host bruteforcing FTP logins by watching for too many
+		## rejected usernames or failed passwords.
+		Bruteforcing
+	};
+
+	## How many rejected usernames or passwords are required before being
+	## considered to be bruteforcing.
+	const bruteforce_threshold = 20 &redef;
+
+	## The time period in which the threshold needs to be crossed before
+	## being reset.
+	const bruteforce_measurement_interval = 15mins &redef;
+}
+
+
+event bro_init()
+	{
+	local r1: SumStats::Reducer = [$stream="ftp.failed_auth", $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$epoch=bruteforce_measurement_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return result["ftp.failed_auth"]$num;
+	                  	},
+	                  $threshold=bruteforce_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["ftp.failed_auth"];
+	                  	local dur = duration_to_mins_secs(r$end-r$begin);
+	                  	local plural = r$unique>1 ? "s" : "";
+	                  	local message = fmt("%s had %d failed logins on %d FTP server%s in %s", key$host, r$num, r$unique, plural, dur);
+	                  	NOTICE([$note=FTP::Bruteforcing,
+	                  	        $src=key$host,
+	                  	        $msg=message,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
+	}
+
+event ftp_reply(c: connection, code: count, msg: string, cont_resp: bool)
+	{
+	local cmd = c$ftp$cmdarg$cmd;
+	if ( cmd == "USER" || cmd == "PASS" )
+		{
+		if ( FTP::parse_ftp_reply_code(code)$x == 5 )
+			SumStats::observe("ftp.failed_auth", [$host=c$id$orig_h], [$str=cat(c$id$resp_h)]);
+		}
+	}

scripts/policy/protocols/http/detect-sqli.bro

@@ -1,7 +1,7 @@
 ##! SQL injection attack detection in HTTP.
 
 @load base/frameworks/notice
-@load base/frameworks/metrics
+@load base/frameworks/sumstats
 @load base/protocols/http
 
 module HTTP;
@@ -15,13 +15,6 @@ export {
 		SQL_Injection_Victim,
 	};
 
-	redef enum Metrics::ID += {
-		## Metric to track SQL injection attackers.
-		SQLI_ATTACKER,
-		## Metrics to track SQL injection victims.
-		SQLI_VICTIM,
-	};
-
 	redef enum Tags += {
 		## Indicator of a URI based SQL injection attack.
 		URI_SQLI,
@@ -42,6 +35,11 @@ export {
 	## At the end of each interval the counter is reset.
 	const sqli_requests_interval = 5min &redef;
 
+	## Collecting samples will add extra data to notice emails
+	## by collecting some sample SQL injection url paths.  Disable
+	## sample collection by setting this value to 0.
+	const collect_SQLi_samples = 5 &redef;
+
 	## Regular expression is used to match URI based SQL injections.
 	const match_sql_injection_uri =
 		  /[\?&][^[:blank:]\x00-\x37\|]+?=[\-[:alnum:]%]+([[:blank:]\x00-\x37]|\/\*.*?\*\/)*['"]?([[:blank:]\x00-\x37]|\/\*.*?\*\/|\)?;)+.*?([hH][aA][vV][iI][nN][gG]|[uU][nN][iI][oO][nN]|[eE][xX][eE][cC]|[sS][eE][lL][eE][cC][tT]|[dD][eE][lL][eE][tT][eE]|[dD][rR][oO][pP]|[dD][eE][cC][lL][aA][rR][eE]|[cC][rR][eE][aA][tT][eE]|[iI][nN][sS][eE][rR][tT])([[:blank:]\x00-\x37]|\/\*.*?\*\/)+/
@@ -52,20 +50,54 @@ export {
 		| /\/\*![[:digit:]]{5}.*?\*\// &redef;
 }
 
+function format_sqli_samples(samples: vector of SumStats::Observation): string
+	{
+	local ret = "SQL Injection samples\n---------------------";
+	for ( i in samples )
+		ret += "\n" + samples[i]$str;
+	return ret;
+	}
+
 event bro_init() &priority=3
 	{
 	# Add filters to the metrics so that the metrics framework knows how to
 	# determine when it looks like an actual attack and how to respond when
 	# thresholds are crossed.
+	local r1: SumStats::Reducer = [$stream="http.sqli.attacker", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples];
+	SumStats::create([$epoch=sqli_requests_interval,
+	                  $reducers=set(r1),
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	return double_to_count(result["http.sqli.attacker"]$sum);
+	                  	},
+	                  $threshold=sqli_requests_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["http.sqli.attacker"];
+	                  	NOTICE([$note=SQL_Injection_Attacker,
+	                  	        $msg="An SQL injection attacker was discovered!",
+	                  	        $email_body_sections=vector(format_sqli_samples(r$samples)),
+	                  	        $src=key$host,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
 
-	Metrics::add_filter(SQLI_ATTACKER, [$log=F,
+	local r2: SumStats::Reducer = [$stream="http.sqli.victim", $apply=set(SumStats::SUM, SumStats::SAMPLE), $num_samples=collect_SQLi_samples];
-	                                   $notice_threshold=sqli_requests_threshold,
+	SumStats::create([$epoch=sqli_requests_interval,
-	                                   $break_interval=sqli_requests_interval,
+	                  $reducers=set(r2),
-	                                   $note=SQL_Injection_Attacker]);
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
-	Metrics::add_filter(SQLI_VICTIM, [$log=F,
+	                  	{
-	                                 $notice_threshold=sqli_requests_threshold,
+	                  	return double_to_count(result["http.sqli.victim"]$sum);
-	                                 $break_interval=sqli_requests_interval,
+	                  	},
-	                                 $note=SQL_Injection_Victim]);
+	                  $threshold=sqli_requests_threshold,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["http.sqli.victim"];
+	                  	NOTICE([$note=SQL_Injection_Victim,
+	                  	        $msg="An SQL injection victim was discovered!",
+	                  	        $email_body_sections=vector(format_sqli_samples(r$samples)),
+	                  	        $src=key$host,
+	                  	        $identifier=cat(key$host)]);
+	                  	}]);
 	}
 
 event http_request(c: connection, method: string, original_URI: string,
@@ -75,7 +107,7 @@ event http_request(c: connection, method: string, original_URI: string,
 		{
 		add c$http$tags[URI_SQLI];
 
-		Metrics::add_data(SQLI_ATTACKER, [$host=c$id$orig_h], 1);
+		SumStats::observe("http.sqli.attacker", [$host=c$id$orig_h], [$str=original_URI]);
-		Metrics::add_data(SQLI_VICTIM, [$host=c$id$resp_h], 1);
+		SumStats::observe("http.sqli.victim",   [$host=c$id$resp_h], [$str=original_URI]);
 		}
 	}

scripts/policy/protocols/ssh/detect-bruteforcing.bro

@@ -2,7 +2,7 @@
 ##! bruteforcing over SSH.
 
 @load base/protocols/ssh
-@load base/frameworks/metrics
+@load base/frameworks/sumstats
 @load base/frameworks/notice
 @load base/frameworks/intel
 
@@ -20,9 +20,9 @@ export {
 		Login_By_Password_Guesser,
 	};
 
-	redef enum Metrics::ID  += {
+	redef enum Intel::Where += {
-		## Metric is to measure failed logins.
+		## An indicator of the login for the intel framework.
-		FAILED_LOGIN,
+		SSH::SUCCESSFUL_LOGIN,
 	};
 
 	## The number of failed SSH connections before a host is designated as
@@ -38,33 +38,40 @@ export {
 	## heuristic fails and this acts as the whitelist.  The index represents
 	## client subnets and the yield value represents server subnets.
 	const ignore_guessers: table[subnet] of subnet &redef;
-
-	## Tracks hosts identified as guessing passwords.
-	global password_guessers: set[addr] 
-		&read_expire=guessing_timeout+1hr &synchronized &redef;
 }
 
 event bro_init()
 	{
-	Metrics::add_filter(FAILED_LOGIN, [$name="detect-bruteforcing", $log=F,
+	local r1: SumStats::Reducer = [$stream="ssh.login.failure", $apply=set(SumStats::SUM)];
-	                                   $note=Password_Guessing,
+	SumStats::create([$epoch=guessing_timeout,
-	                                   $notice_threshold=password_guesses_limit,
+	                  $reducers=set(r1),
-	                                   $notice_freq=1hr,
+	                  $threshold_val(key: SumStats::Key, result: SumStats::Result) =
-	                                   $break_interval=guessing_timeout]);
+	                  	{
+	                  	return double_to_count(result["ssh.login.failure"]$sum);
+	                  	},
+	                  $threshold=password_guesses_limit,
+	                  $threshold_crossed(key: SumStats::Key, result: SumStats::Result) =
+	                  	{
+	                  	local r = result["ssh.login.failure"];
+	                  	# Generate the notice.
+	                  	NOTICE([$note=Password_Guessing,
+	                  	        $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
+	                  	        $src=key$host,
+	                  	        $identifier=cat(key$host)]);
+	                  	# Insert the guesser into the intel framework.
+	                  	Intel::insert([$host=key$host,
+	                  	               $meta=[$source="local",
+	                  	                      $desc=fmt("Bro observed %d apparently failed SSH connections.", r$num)]]);
+	                  	}]);
 	}
 
 event SSH::heuristic_successful_login(c: connection)
 	{
 	local id = c$id;
 
-	# TODO: This is out for the moment pending some more additions to the 
+	Intel::seen([$host=id$orig_h,
-	#       metrics framework.
+	             $conn=c,
-	#if ( id$orig_h in password_guessers )
+	             $where=SSH::SUCCESSFUL_LOGIN]);
-	#	{
-	#	NOTICE([$note=Login_By_Password_Guesser,
-	#	        $conn=c,
-	#	        $msg=fmt("Successful SSH login by password guesser %s", id$orig_h)]);
-	#	}
 	}
 
 event SSH::heuristic_failed_login(c: connection)
@@ -75,5 +82,5 @@ event SSH::heuristic_failed_login(c: connection)
 	# be ignored.
 	if ( ! (id$orig_h in ignore_guessers &&
 	        id$resp_h in ignore_guessers[id$orig_h]) )
-		Metrics::add_data(FAILED_LOGIN, [$host=id$orig_h], 1);
+		SumStats::observe("ssh.login.failure", [$host=id$orig_h], [$num=1]);
 	}

scripts/site/local.bro

@@ -8,6 +8,9 @@
 # Apply the default tuning scripts for common tuning settings.
 @load tuning/defaults
 
+# Load the scan detection script.
+@load misc/scan
+
 # Generate notices when vulnerable versions of software are discovered.
 # The default is to only monitor software found in the address space defined
 # as "local".  Refer to the software framework's documentation for more 

scripts/test-all-policy.bro

@@ -24,9 +24,6 @@
 @load frameworks/intel/smtp.bro
 @load frameworks/intel/ssl.bro
 @load frameworks/intel/where-locations.bro
-@load frameworks/metrics/conn-example.bro
-@load frameworks/metrics/http-example.bro
-@load frameworks/metrics/ssl-example.bro
 @load frameworks/software/version-changes.bro
 @load frameworks/software/vulnerable.bro
 @load integration/barnyard2/__load__.bro
@@ -35,9 +32,13 @@
 @load integration/collective-intel/__load__.bro
 @load integration/collective-intel/main.bro
 @load misc/analysis-groups.bro
+@load misc/app-metrics.bro
 @load misc/capture-loss.bro
+@load misc/detect-traceroute/__load__.bro
+@load misc/detect-traceroute/main.bro
 @load misc/loaded-scripts.bro
 @load misc/profiling.bro
+@load misc/scan.bro
 @load misc/stats.bro
 @load misc/trim-trace-file.bro
 @load protocols/conn/known-hosts.bro
@@ -45,6 +46,7 @@
 @load protocols/conn/weirds.bro
 @load protocols/dns/auth-addl.bro
 @load protocols/dns/detect-external-names.bro
+@load protocols/ftp/detect-bruteforcing.bro
 @load protocols/ftp/detect.bro
 @load protocols/ftp/software.bro
 @load protocols/http/detect-MHR.bro
@@ -76,3 +78,5 @@
 @load tuning/defaults/warnings.bro
 @load tuning/logs-to-elasticsearch.bro
 @load tuning/track-all-assets.bro
+
+redef LogElasticSearch::server_host = "";

src/CMakeLists.txt

@@ -231,6 +231,10 @@ binpac_target(file_analysis/analyzers/pe.pac
 
 find_package (Threads)
 
+# Avoid CMake warning about "3rdparty" looking like a number.
+cmake_policy(PUSH)
+cmake_policy(SET CMP0012 NEW)
+
 # This macro stores associated headers for any C/C++ source files given
 # as arguments (past _var) as a list in the CMake variable named "_var".
 macro(COLLECT_HEADERS _var)
@@ -250,6 +254,8 @@ macro(COLLECT_HEADERS _var)
     endforeach ()
 endmacro(COLLECT_HEADERS _var)
 
+cmake_policy(POP)
+
 # define a command that's used to run the make_dbg_constants.pl script
 # building the bro binary depends on the outputs of this script
 add_custom_command(OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/DebugCmdConstants.h
@@ -442,6 +448,7 @@ set(bro_SRCS
     logging/WriterFrontend.cc
     logging/writers/Ascii.cc
     logging/writers/DataSeries.cc
+    logging/writers/SQLite.cc
     logging/writers/ElasticSearch.cc
     logging/writers/None.cc
 
@@ -452,6 +459,7 @@ set(bro_SRCS
     input/readers/Raw.cc
     input/readers/Benchmark.cc
     input/readers/Binary.cc
+    input/readers/SQLite.cc
 
     file_analysis/Manager.cc
     file_analysis/File.cc
@@ -464,6 +472,8 @@ set(bro_SRCS
     file_analysis/DataEvent.cc
     file_analysis/analyzers/PE.cc
 
+    3rdparty/sqlite3.c
+
     nb_dns.c
     digest.h
 )
@@ -472,7 +482,7 @@ collect_headers(bro_HEADERS ${bro_SRCS})
 
 add_executable(bro ${bro_SRCS} ${bro_HEADERS})
 
-target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT})
+target_link_libraries(bro ${brodeps} ${CMAKE_THREAD_LIBS_INIT} ${CMAKE_DL_LIBS})
 
 install(TARGETS bro DESTINATION bin)
 install(FILES ${INSTALL_BIF_OUTPUTS} DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)

src/CompHash.cc

@@ -181,17 +181,25 @@ char* CompositeHash::SingleValHash(int type_check, char* kp0,
 				Val* key = lv->Index(i);
 				if ( ! (kp1 = SingleValHash(type_check, kp1, key->Type(), key,
 				                            false)) )
+					{
+					Unref(lv);
 					return 0;
+					}
 
 				if ( ! v->Type()->IsSet() )
 					{
 					Val* val = tv->Lookup(key);
 					if ( ! (kp1 = SingleValHash(type_check, kp1, val->Type(),
 								    val, false)) )
+						{
+						Unref(lv);
 						return 0;
 						}
 					}
 				}
+
+			Unref(lv);
+			}
 			break;
 
 		case TYPE_VECTOR:
@@ -454,15 +462,26 @@ int CompositeHash::SingleTypeKeySize(BroType* bt, const Val* v,
 				Val* key = lv->Index(i);
 				sz = SingleTypeKeySize(key->Type(), key, type_check, sz, false,
 				                       calc_static_size);
-				if ( ! sz ) return 0;
+				if ( ! sz )
+					{
+					Unref(lv);
+					return 0;
+					}
+
 				if ( ! bt->IsSet() )
 					{
 					Val* val = tv->Lookup(key);
 					sz = SingleTypeKeySize(val->Type(), val, type_check, sz,
 					                       false, calc_static_size);
-					if ( ! sz ) return 0;
+					if ( ! sz )
+						{
+						Unref(lv);
+						return 0;
 						}
 					}
+				}
+
+			Unref(lv);
 
 			break;
 			}
@@ -830,7 +849,10 @@ const char* CompositeHash::RecoverOneVal(const HashKey* k, const char* kp0,
 				}
 
 			for ( int i = 0; i < n; ++i )
+				{
 				tv->Assign(keys[i], t->IsSet() ? 0 : values[i]);
+				Unref(keys[i]);
+				}
 
 			pval = tv;
 			}

src/Expr.cc

@@ -5478,6 +5478,50 @@ int check_and_promote_exprs(ListExpr*& elements, TypeList* types)
 	return 1;
 	}
 
+int check_and_promote_args(ListExpr*& args, RecordType* types)
+	{
+	expr_list& el = args->Exprs();
+	int ntypes = types->NumFields();
+
+	// give variadic BIFs automatic pass
+	if ( ntypes == 1 && types->FieldDecl(0)->type->Tag() == TYPE_ANY )
+		return 1;
+
+	if ( el.length() < ntypes )
+		{
+		expr_list def_elements;
+
+		// Start from rightmost parameter, work backward to fill in missing
+		// arguments using &default expressions.
+		for ( int i = ntypes - 1; i >= el.length(); --i )
+			{
+			TypeDecl* td = types->FieldDecl(i);
+			Attr* def_attr = td->attrs ? td->attrs->FindAttr(ATTR_DEFAULT) : 0;
+
+			if ( ! def_attr )
+				{
+				types->Error("parameter mismatch", args);
+				return 0;
+				}
+
+			def_elements.insert(def_attr->AttrExpr());
+			}
+
+		loop_over_list(def_elements, i)
+			el.append(def_elements[i]->Ref());
+		}
+
+	TypeList* tl = new TypeList();
+
+	for ( int i = 0; i < types->NumFields(); ++i )
+		tl->Append(types->FieldType(i)->Ref());
+
+	int rval = check_and_promote_exprs(args, tl);
+	Unref(tl);
+
+	return rval;
+	}
+
 int check_and_promote_exprs_to_type(ListExpr*& elements, BroType* type)
 	{
 	expr_list& el = elements->Exprs();

src/Expr.h

@@ -1082,13 +1082,14 @@ Expr* get_assign_expr(Expr* op1, Expr* op2, int is_init);
 // match, promote it as necessary (modifying the ref parameter accordingly)
 // and return 1.
 //
-// The second and third forms are for promoting a list of
+// The second, third, and fourth forms are for promoting a list of
 // expressions (which is updated in place) to either match a list of
 // types or a single type.
 //
 // Note, the type is not "const" because it can be ref'd.
 extern int check_and_promote_expr(Expr*& e, BroType* t);
 extern int check_and_promote_exprs(ListExpr*& elements, TypeList* types);
+extern int check_and_promote_args(ListExpr*& args, RecordType* types);
 extern int check_and_promote_exprs_to_type(ListExpr*& elements, BroType* type);
 
 // Returns a fully simplified form of the expression.  Note that passed

src/FileAnalyzer.cc

@@ -5,16 +5,10 @@
 #include "Reporter.h"
 #include "util.h"
 
-magic_t File_Analyzer::magic = 0;
-magic_t File_Analyzer::magic_mime = 0;
-
 File_Analyzer::File_Analyzer(AnalyzerTag::Tag tag, Connection* conn)
 : TCP_ApplicationAnalyzer(tag, conn)
 	{
 	buffer_len = 0;
-
-	bro_init_magic(&magic, MAGIC_NONE);
-	bro_init_magic(&magic_mime, MAGIC_MIME);
 	}
 
 void File_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
@@ -49,19 +43,13 @@ void File_Analyzer::Done()
 
 void File_Analyzer::Identify()
 	{
-	const char* descr = 0;
+	const char* desc = bro_magic_buffer(magic_desc_cookie, buffer, buffer_len);
-	const char* mime = 0;
+	const char* mime = bro_magic_buffer(magic_mime_cookie, buffer, buffer_len);
-
-	if ( magic )
-		descr = bro_magic_buffer(magic, buffer, buffer_len);
-
-	if ( magic_mime )
-		mime = bro_magic_buffer(magic_mime, buffer, buffer_len);
 
 	val_list* vl = new val_list;
 	vl->append(BuildConnVal());
 	vl->append(new StringVal(buffer_len, buffer));
-	vl->append(new StringVal(descr ? descr : "<unknown>"));
+	vl->append(new StringVal(desc ? desc : "<unknown>"));
 	vl->append(new StringVal(mime ? mime : "<unknown>"));
 	ConnectionEvent(file_transferred, vl);
 	}

src/FileAnalyzer.h

@@ -6,7 +6,6 @@
 #include "TCP.h"
 
 #include <string>
-#include <magic.h>
 
 class File_Analyzer : public TCP_ApplicationAnalyzer {
 public:
@@ -31,9 +30,6 @@ protected:
 	static const int BUFFER_SIZE = 1024;
 	char buffer[BUFFER_SIZE];
 	int buffer_len;
-
-	static magic_t magic;
-	static magic_t magic_mime;
 };
 
 class IRC_Data : public File_Analyzer {

src/HTTP.cc

@@ -299,10 +299,12 @@ void HTTP_Entity::SubmitData(int len, const char* buf)
 			                  http_message->MyHTTP_Analyzer()->GetTag(),
 			                  http_message->MyHTTP_Analyzer()->Conn(),
 			                  http_message->IsOrig());
+
 		file_mgr->DataIn(reinterpret_cast<const u_char*>(buf), len, offset,
 		                 http_message->MyHTTP_Analyzer()->GetTag(),
 		                 http_message->MyHTTP_Analyzer()->Conn(),
 		                 http_message->IsOrig());
+
 		offset += len;
 		}
 	else
@@ -312,6 +314,7 @@ void HTTP_Entity::SubmitData(int len, const char* buf)
 			                  http_message->MyHTTP_Analyzer()->GetTag(),
 			                  http_message->MyHTTP_Analyzer()->Conn(),
 			                  http_message->IsOrig());
+
 		file_mgr->DataIn(reinterpret_cast<const u_char*>(buf), len,
 		                 http_message->MyHTTP_Analyzer()->GetTag(),
 		                 http_message->MyHTTP_Analyzer()->Conn(),
@@ -407,6 +410,7 @@ void HTTP_Entity::SubmitHeader(MIME_Header* h)
 				              instance_length) )
 					instance_length = 0;
 				}
+
 			is_partial_content = true;
 			offset = f;
 			content_length = len;
@@ -642,6 +646,7 @@ void HTTP_Message::EndEntity(MIME_Entity* entity)
 	// SubmitAllHeaders (through EndOfData).
 	if ( entity == top_level )
 		Done();
+
 	else if ( is_orig || MyHTTP_Analyzer()->HTTP_ReplyCode() != 206 )
 		file_mgr->EndOfFile(MyHTTP_Analyzer()->GetTag(),
 		                    MyHTTP_Analyzer()->Conn(), is_orig);
@@ -904,6 +909,7 @@ void HTTP_Analyzer::Done()
 		}
 
 	file_mgr->EndOfFile(GetTag(), Conn(), true);
+
 	/* TODO: this might be nice to have, but reply code is cleared by now.
 	if ( HTTP_ReplyCode() != 206 )
 		// multipart/byteranges may span multiple connections

src/MIME.cc

@@ -1034,12 +1034,14 @@ MIME_Mail::~MIME_Mail()
 void MIME_Mail::BeginEntity(MIME_Entity* /* entity */)
 	{
 	cur_entity_len = 0;
+
 	if ( mime_begin_entity )
 		{
 		val_list* vl = new val_list;
 		vl->append(analyzer->BuildConnVal());
 		analyzer->ConnectionEvent(mime_begin_entity, vl);
 		}
+
 	buffer_start = data_start = 0;
 	ASSERT(entity_content.size() == 0);
 	}
@@ -1131,8 +1133,8 @@ void MIME_Mail::SubmitData(int len, const char* buf)
 	// is_orig param not available, doesn't matter as long as it's consistent
 	file_mgr->DataIn(reinterpret_cast<const u_char*>(buf), len,
 	                 analyzer->GetTag(), analyzer->Conn(), false);
-	cur_entity_len += len;
 
+	cur_entity_len += len;
 	buffer_start = (buf + len) - (char*)data_buffer->Bytes();
 	}
 

src/NetVar.cc

@@ -240,6 +240,11 @@ TableType* record_field_table;
 
 StringVal* cmd_line_bpf_filter;
 
+OpaqueType* md5_type;
+OpaqueType* sha1_type;
+OpaqueType* sha256_type;
+OpaqueType* entropy_type;
+
 #include "const.bif.netvar_def"
 #include "types.bif.netvar_def"
 #include "event.bif.netvar_def"
@@ -300,6 +305,11 @@ void init_general_global_var()
 
 	cmd_line_bpf_filter =
 		internal_val("cmd_line_bpf_filter")->AsStringVal();
+
+	md5_type = new OpaqueType("md5");
+	sha1_type = new OpaqueType("sha1");
+	sha256_type = new OpaqueType("sha256");
+	entropy_type = new OpaqueType("entropy");
 	}
 
 void init_net_var()

src/NetVar.h

@@ -244,6 +244,12 @@ extern TableType* record_field_table;
 
 extern StringVal* cmd_line_bpf_filter;
 
+class OpaqueType;
+extern OpaqueType* md5_type;
+extern OpaqueType* sha1_type;
+extern OpaqueType* sha256_type;
+extern OpaqueType* entropy_type;
+
 // Initializes globals that don't pertain to network/event analysis.
 extern void init_general_global_var();
 

src/OpaqueVal.cc

@@ -1,4 +1,5 @@
 #include "OpaqueVal.h"
+#include "NetVar.h"
 #include "Reporter.h"
 #include "Serializer.h"
 
@@ -72,6 +73,10 @@ bool HashVal::DoUnserialize(UnserialInfo* info)
 	return UNSERIALIZE(&valid);
 	}
 
+MD5Val::MD5Val() : HashVal(md5_type)
+	{
+	}
+
 void MD5Val::digest(val_list& vlist, u_char result[MD5_DIGEST_LENGTH])
 	{
 	MD5_CTX h;
@@ -189,6 +194,10 @@ bool MD5Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+SHA1Val::SHA1Val() : HashVal(sha1_type)
+	{
+	}
+
 void SHA1Val::digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH])
 	{
 	SHA_CTX h;
@@ -297,6 +306,10 @@ bool SHA1Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+SHA256Val::SHA256Val() : HashVal(sha256_type)
+	{
+	}
+
 void SHA256Val::digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH])
 	{
 	SHA256_CTX h;
@@ -410,6 +423,9 @@ bool SHA256Val::DoUnserialize(UnserialInfo* info)
 	return true;
 	}
 
+EntropyVal::EntropyVal() : OpaqueVal(entropy_type)
+	{
+	}
 
 bool EntropyVal::Feed(const void* data, size_t size)
 	{

src/OpaqueVal.h

@@ -36,7 +36,7 @@ public:
 			 u_char key[MD5_DIGEST_LENGTH],
 			 u_char result[MD5_DIGEST_LENGTH]);
 
-	MD5Val() : HashVal(new OpaqueType("md5"))	{ Unref(Type()); }
+	MD5Val();
 
 protected:
 	friend class Val;
@@ -55,7 +55,7 @@ class SHA1Val : public HashVal {
 public:
 	static void digest(val_list& vlist, u_char result[SHA_DIGEST_LENGTH]);
 
-	SHA1Val() : HashVal(new OpaqueType("sha1")) { Unref(Type()); }
+	SHA1Val();
 
 protected:
 	friend class Val;
@@ -74,7 +74,7 @@ class SHA256Val : public HashVal {
 public:
 	static void digest(val_list& vlist, u_char result[SHA256_DIGEST_LENGTH]);
 
-	SHA256Val() : HashVal(new OpaqueType("sha256"))	{ Unref(Type()); }
+	SHA256Val();
 
 protected:
 	friend class Val;
@@ -91,7 +91,7 @@ private:
 
 class EntropyVal : public OpaqueVal {
 public:
-	EntropyVal() : OpaqueVal(new OpaqueType("entropy"))	{ }
+	EntropyVal();
 
 	bool Feed(const void* data, size_t size);
 	bool Get(double *r_ent, double *r_chisq, double *r_mean,

src/SMTP.cc

@@ -90,6 +90,7 @@ void SMTP_Analyzer::Undelivered(int seq, int len, bool is_orig)
 		// ongoing mail transaction.
 		if ( mail )
 			mail->Undelivered(len);
+
 		EndData();
 		}
 

src/TCP.cc

@@ -566,7 +566,7 @@ void TCP_Analyzer::UpdateInactiveState(double t,
 			else
 				endpoint->SetState(TCP_ENDPOINT_SYN_SENT);
 
-			if ( connection_attempt )
+			if ( tcp_attempt_delay )
 				ADD_ANALYZER_TIMER(&TCP_Analyzer::AttemptTimer,
 					t + tcp_attempt_delay, 1,
 					TIMER_TCP_ATTEMPT);
@@ -1497,24 +1497,7 @@ void TCP_Analyzer::ExpireTimer(double t)
 
 		if ( resp->state == TCP_ENDPOINT_INACTIVE )
 			{
-			if ( (orig->state == TCP_ENDPOINT_SYN_SENT ||
+			if ( orig->state == TCP_ENDPOINT_INACTIVE )
-			      orig->state == TCP_ENDPOINT_SYN_ACK_SENT) )
-				{
-				if ( ! connection_attempt )
-					{
-					// Time out the connection attempt,
-					// since the AttemptTimer isn't going
-					// to do it for us, and we don't want
-					// to clog the data structures with
-					// old, failed attempts.
-					Event(connection_timeout);
-					is_active = 0;
-					sessions->Remove(Conn());
-					return;
-					}
-				}
-
-			else if ( orig->state == TCP_ENDPOINT_INACTIVE )
 				{
 				// Nothing ever happened on this connection.
 				// This can occur when we see a trashed

src/Type.cc

@@ -671,9 +671,25 @@ FuncType::FuncType(RecordType* arg_args, BroType* arg_yield, function_flavor arg
 
 	arg_types = new TypeList();
 
+	bool has_default_arg = false;
+
 	for ( int i = 0; i < args->NumFields(); ++i )
+		{
+		const TypeDecl* td = args->FieldDecl(i);
+
+		if ( td->attrs && td->attrs->FindAttr(ATTR_DEFAULT) )
+			has_default_arg = true;
+
+		else if ( has_default_arg )
+			{
+			const char* err_str = fmt("required parameter '%s' must precede "
+			                          "default parameters", td->id);
+			args->Error(err_str);
+			}
+
 		arg_types->Append(args->FieldType(i)->Ref());
 		}
+	}
 
 string FuncType::FlavorString() const
 	{
@@ -708,7 +724,7 @@ BroType* FuncType::YieldType()
 
 int FuncType::MatchesIndex(ListExpr*& index) const
 	{
-	return check_and_promote_exprs(index, arg_types) ?
+	return check_and_promote_args(index, args) ?
 			MATCHES_INDEX_SCALAR : DOES_NOT_MATCH_INDEX;
 	}
 

src/Val.cc

@@ -1749,7 +1749,7 @@ Val* TableVal::Default(Val* index)
 
 	if ( def_val->Type()->Tag() != TYPE_FUNC ||
 	     same_type(def_val->Type(), Type()->YieldType()) )
-		return def_val->Ref();
+		return def_attr->AttrExpr()->IsConst() ? def_val->Ref() : def_val->Clone();
 
 	const Func* f = def_val->AsFunc();
 	val_list* vl = new val_list();

src/Var.cc

@@ -318,6 +318,30 @@ void add_type(ID* id, BroType* t, attr_list* attr, int /* is_event */)
 		id->SetAttrs(new Attributes(attr, tnew, false));
 	}
 
+static void transfer_arg_defaults(RecordType* args, RecordType* recv)
+	{
+	for ( int i = 0; i < args->NumFields(); ++i )
+		{
+		TypeDecl* args_i = args->FieldDecl(i);
+		TypeDecl* recv_i = recv->FieldDecl(i);
+
+		Attr* def = args_i->attrs ? args_i->attrs->FindAttr(ATTR_DEFAULT) : 0;
+
+		if ( ! def )
+			continue;
+
+		if ( ! recv_i->attrs )
+			{
+			attr_list* a = new attr_list();
+			a->append(def);
+			recv_i->attrs = new Attributes(a, recv_i->type, true);
+			}
+
+		else if ( ! recv_i->attrs->FindAttr(ATTR_DEFAULT) )
+			recv_i->attrs->AddAttr(def);
+		}
+	}
+
 void begin_func(ID* id, const char* module_name, function_flavor flavor,
 		int is_redef, FuncType* t)
 	{
@@ -335,6 +359,11 @@ void begin_func(ID* id, const char* module_name, function_flavor flavor,
 		{
 		if ( ! same_type(id->Type(), t) )
 			id->Type()->Error("incompatible types", t);
+
+		// If a previous declaration of the function had &default params,
+		// automatically transfer any that are missing (convenience so that
+		// implementations don't need to specify the &default expression again).
+		transfer_arg_defaults(id->Type()->AsFuncType()->Args(), t->Args());
 		}
 
 	else if ( is_redef )

src/bif_arg.cc

@@ -30,11 +30,13 @@ BuiltinFuncArg::BuiltinFuncArg(const char* arg_name, int arg_type)
 	type_str = "";
 	}
 
-BuiltinFuncArg::BuiltinFuncArg(const char* arg_name, const char* arg_type_str)
+BuiltinFuncArg::BuiltinFuncArg(const char* arg_name, const char* arg_type_str,
+                               const char* arg_attr_str)
 	{
 	name = arg_name;
 	type = TYPE_OTHER;
 	type_str = arg_type_str;
+	attr_str = arg_attr_str;
 
 	for ( int i = 0; builtin_func_arg_type[i].bif_type[0] != '\0'; ++i )
 		if ( ! strcmp(builtin_func_arg_type[i].bif_type, arg_type_str) )
@@ -46,7 +48,8 @@ BuiltinFuncArg::BuiltinFuncArg(const char* arg_name, const char* arg_type_str)
 
 void BuiltinFuncArg::PrintBro(FILE* fp)
 	{
-	fprintf(fp, "%s: %s%s", name, builtin_func_arg_type[type].bro_type, type_str);
+	fprintf(fp, "%s: %s%s %s", name, builtin_func_arg_type[type].bro_type,
+	        type_str, attr_str);
 	}
 
 void BuiltinFuncArg::PrintCDef(FILE* fp, int n)

src/bif_arg.h

@@ -25,7 +25,13 @@ extern const char* builtin_func_arg_type_bro_name[];
 class BuiltinFuncArg {
 public:
 	BuiltinFuncArg(const char* arg_name, int arg_type);
-	BuiltinFuncArg(const char* arg_name, const char* arg_type_str);
+	BuiltinFuncArg(const char* arg_name, const char* arg_type_str,
+	               const char* arg_attr_str = "");
+
+	void SetAttrStr(const char* arg_attr_str)
+		{
+		attr_str = arg_attr_str;
+		};
 
 	const char* Name() const	{ return name; }
 	int Type() const		{ return type; }
@@ -39,6 +45,7 @@ protected:
 	const char* name;
 	int type;
 	const char* type_str;
+	const char* attr_str;
 };
 
 #endif

src/bro.bif

@@ -849,11 +849,7 @@ extern "C" {
 ## Returns: The MIME type of *data*, or "<unknown>" if there was an error.
 function identify_data%(data: string, return_mime: bool%): string
 	%{
-	static magic_t magic_mime = 0;
+	magic_t* magic = return_mime ? &magic_mime_cookie : &magic_desc_cookie;
-	static magic_t magic_descr = 0;
-
-	magic_t* magic = return_mime ? &magic_mime : &magic_descr;
-	bro_init_magic(magic, return_mime ? MAGIC_MIME : MAGIC_NONE);
 
 	if( ! *magic )
 		return new StringVal("<unknown>");
@@ -2473,7 +2469,7 @@ function bytestring_to_double%(s: string%): double
 ##
 ## Returns: The value contained in *s*, or 0 if the conversion failed.
 ##
-function bytestring_to_count%(s: string, is_le: bool%): count
+function bytestring_to_count%(s: string, is_le: bool &default=F%): count
 	%{
 #ifdef HOST_BIGENDIAN
 	static const bool host_bigendian = true;

src/builtin-func.l

@@ -26,6 +26,7 @@ int check_c_mode(int t)
 %}
 
 WS	[ \t]+
+OWS	[ \t]*
  /* Note, bifcl only accepts a single "::" in IDs while the policy
     layer acceptes multiple. (But the policy layer doesn't have 
 	a hierachy. */
@@ -101,7 +102,13 @@ HEX	[0-9a-fA-F]+
 	return TOK_ID;
 	}
 
-&{ID}	{
+  /*
+  Hacky way to pass along arbitrary attribute expressions since the BIF parser
+  has little understanding of valid Bro expressions.  With this pattern, the
+  attribute expression should stop when it reaches another attribute, another
+  function argument, or the end of the function declaration.
+  */
+&{ID}({OWS}={OWS}[^&%;,]+)?	{
 	int t = check_c_mode(TOK_ATTR);
 
 	if ( t == TOK_ATTR )

src/builtin-func.y

@@ -277,7 +277,7 @@ void print_event_c_body(FILE *fp)
 
 %left ',' ':'
 
-%type <str> TOK_C_TOKEN TOK_ID TOK_CSTR TOK_WS TOK_COMMENT TOK_ATTR TOK_INT opt_ws type
+%type <str> TOK_C_TOKEN TOK_ID TOK_CSTR TOK_WS TOK_COMMENT TOK_ATTR TOK_INT opt_ws type attr_list opt_attr_list
 %type <val> TOK_ATOM TOK_BOOL
 
 %union	{
@@ -375,7 +375,8 @@ type_def_types: TOK_RECORD
 			{ set_definition_type(TYPE_DEF, "Table"); }
 	;
 
-event_def:	event_prefix opt_ws plain_head opt_attr end_of_head ';'
+event_def:	event_prefix opt_ws plain_head opt_attr_list
+			{ fprintf(fp_bro_init, "%s", $4); } end_of_head ';'
 			{
 			print_event_c_prototype(fp_func_h, true);
 			print_event_c_prototype(fp_func_def, false);
@@ -458,20 +459,17 @@ const_def:	TOK_CONST opt_ws TOK_ID opt_ws ':' opt_ws TOK_ID opt_ws ';'
 				accessor);
 			}
 
-
+attr_list:
-/* Currently support only boolean and string values */
+		attr_list TOK_ATTR
-opt_attr_init:	/* nothing */
+			{ $$ = concat($1, $2); }
-	|	'=' opt_ws TOK_BOOL opt_ws
+	|
-			{
+		TOK_ATTR
-			fprintf(fp_bro_init, "=%s%c%s", $2, ($3) ? 'T' : 'F', $4);
-			}
-	|	'=' opt_ws TOK_CSTR opt_ws
-			{ fprintf(fp_bro_init, "=%s%s%s", $2, $3, $4); }
 	;
 
-opt_attr:	/* nothing */
+opt_attr_list:
-	|	opt_attr TOK_ATTR { fprintf(fp_bro_init, "%s", $2); }
+		attr_list
-		opt_ws opt_attr_init
+	|	/* nothing */
+		{ $$ = ""; }
 	;
 
 func_prefix:	TOK_FUNCTION
@@ -579,9 +577,10 @@ args:		args_1
 			{ /* empty, to avoid yacc complaint about type clash */ }
 	;
 
-args_1:		args_1 ',' opt_ws arg opt_ws
+args_1:		args_1 ',' opt_ws arg opt_ws opt_attr_list
-	|	opt_ws arg opt_ws
+			{ if ( ! args.empty() ) args[args.size()-1]->SetAttrStr($6); }
-			{ /* empty */ }
+	|	opt_ws arg opt_ws opt_attr_list
+			{ if ( ! args.empty() ) args[args.size()-1]->SetAttrStr($4); }
 	;
 
 // TODO: Migrate all other compound types to this rule. Once the BiF language

src/event.bif

@@ -7000,7 +7000,7 @@ event event_queue_flush_point%(%);
 ## .. bro:see:: set_file_handle
 event get_file_handle%(tag: count, c: connection, is_orig: bool%);
 
-## Indicates that a analysis of a new file has begun. The analysis can be
+## Indicates that an analysis of a new file has begun. The analysis can be
 ## augmented at this time via :bro:see:`FileAnalysis::add_analyzer`.
 ##
 ## f: The file.

src/file_analysis.bif

@@ -6,6 +6,27 @@ module FileAnalysis;
 #include "file_analysis/Manager.h"
 %%}
 
+type AnalyzerArgs: record;
+
+## An enumeration of various file analysis actions that can be taken.
+enum Analyzer %{
+
+	## Extract a file to local filesystem
+	ANALYZER_EXTRACT,
+
+	## Calculate an MD5 digest of the file's contents.
+	ANALYZER_MD5,
+
+	## Calculate an SHA1 digest of the file's contents.
+	ANALYZER_SHA1,
+
+	## Calculate an SHA256 digest of the file's contents.
+	ANALYZER_SHA256,
+
+	## Deliver the file contents to the script-layer in an event.
+	ANALYZER_DATA_EVENT,
+%}
+
 ## :bro:see:`FileAnalysis::postpone_timeout`.
 function FileAnalysis::__postpone_timeout%(file_id: string%): bool
 	%{

src/file_analysis/Analyzer.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_ANALYZER_H
 #define FILE_ANALYSIS_ANALYZER_H
 
@@ -15,7 +17,6 @@ class File;
  */
 class Analyzer {
 public:
-
 	virtual ~Analyzer()
 		{
 		DBG_LOG(DBG_FILE_ANALYSIS, "Destroy file analyzer %d", tag);
@@ -83,13 +84,13 @@ public:
 		}
 
 protected:
-
 	Analyzer(RecordVal* arg_args, File* arg_file)
 	    : tag(file_analysis::Analyzer::ArgsTag(arg_args)),
 	      args(arg_args->Ref()->AsRecordVal()),
 	      file(arg_file)
 		{}
 
+private:
 	FA_Tag tag;
 	RecordVal* args;
 	File* file;

src/file_analysis/AnalyzerSet.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include "AnalyzerSet.h"
 #include "File.h"
 #include "Analyzer.h"
@@ -41,6 +43,7 @@ AnalyzerSet::~AnalyzerSet()
 		delete mod;
 		mod_queue.pop();
 		}
+
 	delete analyzer_hash;
 	}
 
@@ -110,6 +113,7 @@ bool AnalyzerSet::Remove(FA_Tag tag, HashKey* key)
 	{
 	file_analysis::Analyzer* a =
 	    (file_analysis::Analyzer*) analyzer_map.Remove(key);
+
 	delete key;
 
 	if ( ! a )
@@ -121,6 +125,7 @@ bool AnalyzerSet::Remove(FA_Tag tag, HashKey* key)
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Remove analyzer %d for file id %s", a->Tag(),
 	        file->GetID().c_str());
+
 	delete a;
 	return true;
 	}
@@ -145,6 +150,7 @@ HashKey* AnalyzerSet::GetKey(const RecordVal* args) const
 	HashKey* key = analyzer_hash->ComputeHash(args, 1);
 	if ( ! key )
 		reporter->InternalError("AnalyzerArgs type mismatch");
+
 	return key;
 	}
 
@@ -174,7 +180,8 @@ void AnalyzerSet::Insert(file_analysis::Analyzer* a, HashKey* key)
 
 void AnalyzerSet::DrainModifications()
 	{
-	if ( mod_queue.empty() ) return;
+	if ( mod_queue.empty() )
+		return;
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Start analyzer mod queue flush of file id %s",
 	        file->GetID().c_str());

src/file_analysis/AnalyzerSet.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_ANALYZERSET_H
 #define FILE_ANALYSIS_ANALYZERSET_H
 
@@ -20,7 +22,6 @@ declare(PDict,Analyzer);
  */
 class AnalyzerSet {
 public:
-
 	AnalyzerSet(File* arg_file);
 
 	~AnalyzerSet();
@@ -57,12 +58,12 @@ public:
 		{ return analyzer_map.NextEntry(c); }
 
 protected:
-
 	HashKey* GetKey(const RecordVal* args) const;
 	file_analysis::Analyzer* InstantiateAnalyzer(RecordVal* args) const;
 	void Insert(file_analysis::Analyzer* a, HashKey* key);
 	bool Remove(FA_Tag tag, HashKey* key);
 
+private:
 	File* file;
 	CompositeHash* analyzer_hash;                /**< AnalyzerArgs hashes. */
 	PDict(file_analysis::Analyzer) analyzer_map; /**< Indexed by AnalyzerArgs. */

src/file_analysis/DataEvent.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include <string>
 
 #include "DataEvent.h"
@@ -17,10 +19,8 @@ file_analysis::Analyzer* DataEvent::Instantiate(RecordVal* args, File* file)
 	{
 	using BifType::Record::FileAnalysis::AnalyzerArgs;
 
-	const char* chunk_field = "chunk_event";
+	int chunk_off = AnalyzerArgs->FieldOffset("chunk_event");
-	const char* stream_field = "stream_event";
+	int stream_off = AnalyzerArgs->FieldOffset("stream_event");
-	int chunk_off = AnalyzerArgs->FieldOffset(chunk_field);
-	int stream_off = AnalyzerArgs->FieldOffset(stream_field);
 
 	Val* chunk_val = args->Lookup(chunk_off);
 	Val* stream_val = args->Lookup(stream_off);
@@ -44,7 +44,7 @@ bool DataEvent::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 	if ( ! chunk_event ) return true;
 
 	val_list* args = new val_list;
-	args->append(file->GetVal()->Ref());
+	args->append(GetFile()->GetVal()->Ref());
 	args->append(new StringVal(new BroString(data, len, 0)));
 	args->append(new Val(offset, TYPE_COUNT));
 
@@ -58,7 +58,7 @@ bool DataEvent::DeliverStream(const u_char* data, uint64 len)
 	if ( ! stream_event ) return true;
 
 	val_list* args = new val_list;
-	args->append(file->GetVal()->Ref());
+	args->append(GetFile()->GetVal()->Ref());
 	args->append(new StringVal(new BroString(data, len, 0)));
 
 	mgr.QueueEvent(stream_event, args);

src/file_analysis/DataEvent.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_DATAEVENT_H
 #define FILE_ANALYSIS_DATAEVENT_H
 
@@ -14,18 +16,17 @@ namespace file_analysis {
  */
 class DataEvent : public file_analysis::Analyzer {
 public:
-
-	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
-
 	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
 
 	virtual bool DeliverStream(const u_char* data, uint64 len);
 
-protected:
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
 
+protected:
 	DataEvent(RecordVal* args, File* file,
 	          EventHandlerPtr ce, EventHandlerPtr se);
 
+private:
 	EventHandlerPtr chunk_event;
 	EventHandlerPtr stream_event;
 };

src/file_analysis/Extract.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include <string>
 
 #include "Extract.h"
@@ -28,17 +30,18 @@ Extract::~Extract()
 file_analysis::Analyzer* Extract::Instantiate(RecordVal* args, File* file)
 	{
 	using BifType::Record::FileAnalysis::AnalyzerArgs;
-	const char* field = "extract_filename";
+	Val* v = args->Lookup(AnalyzerArgs->FieldOffset("extract_filename"));
-	Val* v = args->Lookup(AnalyzerArgs->FieldOffset(field));
 
-	if ( ! v ) return 0;
+	if ( ! v )
+		return 0;
 
 	return new Extract(args, file, v->AsString()->CheckString());
 	}
 
 bool Extract::DeliverChunk(const u_char* data, uint64 len, uint64 offset)
 	{
-	if ( ! fd ) return false;
+	if ( ! fd )
+		return false;
 
 	safe_pwrite(fd, data, len, offset);
 	return true;

src/file_analysis/Extract.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_EXTRACT_H
 #define FILE_ANALYSIS_EXTRACT_H
 
@@ -14,17 +16,16 @@ namespace file_analysis {
  */
 class Extract : public file_analysis::Analyzer {
 public:
-
-	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
-
 	virtual ~Extract();
 
 	virtual bool DeliverChunk(const u_char* data, uint64 len, uint64 offset);
 
-protected:
+	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file);
 
+protected:
 	Extract(RecordVal* args, File* file, const string& arg_filename);
 
+private:
 	string filename;
 	int fd;
 };

src/file_analysis/File.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include <string>
 #include <openssl/md5.h>
 
@@ -49,13 +51,12 @@ int File::bof_buffer_size_idx = -1;
 int File::bof_buffer_idx = -1;
 int File::mime_type_idx = -1;
 
-magic_t File::magic_mime = 0;
-
 string File::salt;
 
 void File::StaticInit()
 	{
-	if ( id_idx != -1 ) return;
+	if ( id_idx != -1 )
+		return;
 
 	id_idx = Idx("id");
 	parent_id_idx = Idx("parent_id");
@@ -72,8 +73,6 @@ void File::StaticInit()
 	bof_buffer_idx = Idx("bof_buffer");
 	mime_type_idx = Idx("mime_type");
 
-	bro_init_magic(&magic_mime, MAGIC_MIME);
-
 	salt = BifConst::FileAnalysis::salt->CheckString();
 	}
 
@@ -131,7 +130,8 @@ double File::GetLastActivityTime() const
 
 void File::UpdateConnectionFields(Connection* conn)
 	{
-	if ( ! conn ) return;
+	if ( ! conn )
+		return;
 
 	Val* conns = val->Lookup(conns_idx);
 
@@ -140,7 +140,8 @@ void File::UpdateConnectionFields(Connection* conn)
 	if ( ! conns )
 		{
 		is_first = true;
-		val->Assign(conns_idx, conns = empty_connection_table());
+		conns = empty_connection_table();
+		val->Assign(conns_idx, conns);
 		}
 
 	Val* idx = get_conn_id_val(conn);
@@ -182,6 +183,7 @@ int File::Idx(const string& field)
 	int rval = fa_file_type->FieldOffset(field.c_str());
 	if ( rval < 0 )
 		reporter->InternalError("Unknown fa_file field: %s", field.c_str());
+
 	return rval;
 	}
 
@@ -209,9 +211,12 @@ void File::SetTotalBytes(uint64 size)
 bool File::IsComplete() const
 	{
 	Val* total = val->Lookup(total_bytes_idx);
-	if ( ! total ) return false;
+	if ( ! total )
+		return false;
+
 	if ( LookupFieldDefaultCount(seen_bytes_idx) >= total->AsCount() )
 		return true;
+
 	return false;
 	}
 
@@ -232,7 +237,8 @@ bool File::RemoveAnalyzer(const RecordVal* args)
 
 bool File::BufferBOF(const u_char* data, uint64 len)
 	{
-	if ( bof_buffer.full || bof_buffer.replayed ) return false;
+	if ( bof_buffer.full || bof_buffer.replayed )
+		return false;
 
 	uint64 desired_size = LookupFieldDefaultCount(bof_buffer_size_idx);
 
@@ -250,7 +256,7 @@ bool File::BufferBOF(const u_char* data, uint64 len)
 
 bool File::DetectMIME(const u_char* data, uint64 len)
 	{
-	const char* mime = bro_magic_buffer(magic_mime, data, len);
+	const char* mime = bro_magic_buffer(magic_mime_cookie, data, len);
 
 	if ( mime )
 		{
@@ -268,7 +274,9 @@ bool File::DetectMIME(const u_char* data, uint64 len)
 
 void File::ReplayBOF()
 	{
-	if ( bof_buffer.replayed ) return;
+	if ( bof_buffer.replayed )
+		return;
+
 	bof_buffer.replayed = true;
 
 	if ( bof_buffer.chunks.empty() )
@@ -314,9 +322,7 @@ void File::DataIn(const u_char* data, uint64 len, uint64 offset)
 
 	// TODO: check reassembly requirement based on buffer size in record
 	if ( need_reassembly )
-		{
+		reporter->InternalError("file_analyzer::File TODO: reassembly not yet supported");
-		// TODO
-		}
 
 	// TODO: reassembly overflow stuff, increment overflow count, eval trigger
 
@@ -327,7 +333,8 @@ void File::DataIn(const u_char* data, uint64 len)
 	{
 	analyzers.DrainModifications();
 
-	if ( BufferBOF(data, len) ) return;
+	if ( BufferBOF(data, len) )
+		return;
 
 	if ( missed_bof )
 		{
@@ -360,7 +367,8 @@ void File::DataIn(const u_char* data, uint64 len)
 
 void File::EndOfFile()
 	{
-	if ( done ) return;
+	if ( done )
+		return;
 
 	analyzers.DrainModifications();
 
@@ -420,7 +428,8 @@ bool File::FileEventAvailable(EventHandlerPtr h)
 
 void File::FileEvent(EventHandlerPtr h)
 	{
-	if ( ! FileEventAvailable(h) ) return;
+	if ( ! FileEventAvailable(h) )
+		return;
 
 	val_list* vl = new val_list();
 	vl->append(val->Ref());

src/file_analysis/File.h

@@ -1,9 +1,10 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_FILE_H
 #define FILE_ANALYSIS_FILE_H
 
 #include <string>
 #include <vector>
-#include <magic.h>
 
 #include "AnalyzerTags.h"
 #include "Conn.h"
@@ -18,10 +19,7 @@ namespace file_analysis {
  * Wrapper class around \c fa_file record values from script layer.
  */
 class File {
-friend class Manager;
-
 public:
-
 	~File();
 
 	/**
@@ -128,6 +126,7 @@ public:
 	void FileEvent(EventHandlerPtr h, val_list* vl);
 
 protected:
+	friend class Manager;
 
 	/**
 	 * Constructor; only file_analysis::Manager should be creating these.
@@ -176,6 +175,17 @@ protected:
 	 */
 	bool DetectMIME(const u_char* data, uint64 len);
 
+	/**
+	 * @return the field offset in #val record corresponding to \a field_name.
+	 */
+	static int Idx(const string& field_name);
+
+	/**
+	 * Initializes static member.
+	 */
+	static void StaticInit();
+
+private:
 	FileID id;                 /**< A pretty hash that likely identifies file */
 	string unique;             /**< A string that uniquely identifies file */
 	RecordVal* val;            /**< \c fa_file from script layer. */
@@ -197,18 +207,6 @@ protected:
 		BroString::CVec chunks;
 	} bof_buffer;              /**< Beginning of file buffer. */
 
-	/**
-	 * @return the field offset in #val record corresponding to \a field_name.
-	 */
-	static int Idx(const string& field_name);
-
-	/**
-	 * Initializes static member.
-	 */
-	static void StaticInit();
-
-	static magic_t magic_mime;
-
 	static string salt;
 
 	static int id_idx;

src/file_analysis/FileID.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_FILEID_H
 #define FILE_ANALYSIS_FILEID_H
 

src/file_analysis/FileTimer.cc

@@ -1,9 +1,10 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include "Manager.h"
 #include "File.h"
 
 using namespace file_analysis;
 
-
 FileTimer::FileTimer(double t, const FileID& id, double interval)
     : Timer(t + interval, TIMER_FILE_ANALYSIS_INACTIVITY), file_id(id)
 	{
@@ -15,7 +16,8 @@ void FileTimer::Dispatch(double t, int is_expire)
 	{
 	File* file = file_mgr->Lookup(file_id);
 
-	if ( ! file ) return;
+	if ( ! file )
+		return;
 
 	double last_active = file->GetLastActivityTime();
 	double inactive_time = t > last_active ? t - last_active : 0.0;

src/file_analysis/FileTimer.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_FILETIMER_H
 #define FILE_ANALYSIS_FILETIMER_H
 
@@ -12,7 +14,6 @@ namespace file_analysis {
  */
 class FileTimer : public Timer {
 public:
-
 	FileTimer(double t, const FileID& id, double interval);
 
 	/**
@@ -21,8 +22,7 @@ public:
 	 */
 	void Dispatch(double t, int is_expire);
 
-protected:
+private:
-
 	FileID file_id;
 };
 

src/file_analysis/Hash.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include <string>
 
 #include "Hash.h"
@@ -19,7 +21,8 @@ Hash::~Hash()
 
 bool Hash::DeliverStream(const u_char* data, uint64 len)
 	{
-	if ( ! hash->IsValid() ) return false;
+	if ( ! hash->IsValid() )
+		return false;
 
 	if ( ! fed )
 		fed = len > 0;
@@ -41,10 +44,11 @@ bool Hash::Undelivered(uint64 offset, uint64 len)
 
 void Hash::Finalize()
 	{
-	if ( ! hash->IsValid() || ! fed ) return;
+	if ( ! hash->IsValid() || ! fed )
+		return;
 
 	val_list* vl = new val_list();
-	vl->append(file->GetVal()->Ref());
+	vl->append(GetFile()->GetVal()->Ref());
 	vl->append(new StringVal(kind));
 	vl->append(hash->Get());
 

src/file_analysis/Hash.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_HASH_H
 #define FILE_ANALYSIS_HASH_H
 
@@ -15,7 +17,6 @@ namespace file_analysis {
  */
 class Hash : public file_analysis::Analyzer {
 public:
-
 	virtual ~Hash();
 
 	virtual bool DeliverStream(const u_char* data, uint64 len);
@@ -25,11 +26,11 @@ public:
 	virtual bool Undelivered(uint64 offset, uint64 len);
 
 protected:
-
 	Hash(RecordVal* args, File* file, HashVal* hv, const char* kind);
 
 	void Finalize();
 
+private:
 	HashVal* hash;
 	bool fed;
 	const char* kind;
@@ -37,12 +38,10 @@ protected:
 
 class MD5 : public Hash {
 public:
-
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return file_hash ? new MD5(args, file) : 0; }
 
 protected:
-
 	MD5(RecordVal* args, File* file)
 		: Hash(args, file, new MD5Val(), "md5")
 		{}
@@ -50,12 +49,10 @@ protected:
 
 class SHA1 : public Hash {
 public:
-
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return file_hash ? new SHA1(args, file) : 0; }
 
 protected:
-
 	SHA1(RecordVal* args, File* file)
 		: Hash(args, file, new SHA1Val(), "sha1")
 		{}
@@ -63,12 +60,10 @@ protected:
 
 class SHA256 : public Hash {
 public:
-
 	static file_analysis::Analyzer* Instantiate(RecordVal* args, File* file)
 		{ return file_hash ? new SHA256(args, file) : 0; }
 
 protected:
-
 	SHA256(RecordVal* args, File* file)
 		: Hash(args, file, new SHA256Val(), "sha256")
 		{}

src/file_analysis/Manager.cc

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #include <vector>
 #include <string>
 
@@ -25,6 +27,7 @@ void Manager::Terminate()
 	vector<FileID> keys;
 	for ( IDMap::iterator it = id_map.begin(); it != id_map.end(); ++it )
 		keys.push_back(it->first);
+
 	for ( size_t i = 0; i < keys.size(); ++i )
 		Timeout(keys[i], true);
 	}
@@ -37,7 +40,8 @@ void Manager::SetHandle(const string& handle)
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
                      AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) ) return;
+	if ( IsDisabled(tag) )
+		return;
 
 	GetFileHandle(tag, conn, is_orig);
 	DataIn(data, len, offset, GetFile(current_handle, conn, tag, is_orig));
@@ -52,7 +56,8 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
 void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
                      File* file)
 	{
-	if ( ! file ) return;
+	if ( ! file )
+		return;
 
 	file->DataIn(data, len, offset);
 
@@ -63,8 +68,11 @@ void Manager::DataIn(const u_char* data, uint64 len, uint64 offset,
 void Manager::DataIn(const u_char* data, uint64 len, AnalyzerTag::Tag tag,
                      Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) ) return;
+	if ( IsDisabled(tag) )
+		return;
+
 	GetFileHandle(tag, conn, is_orig);
+
 	// Sequential data input shouldn't be going over multiple conns, so don't
 	// do the check to update connection set.
 	DataIn(data, len, GetFile(current_handle, conn, tag, is_orig, false));
@@ -77,7 +85,8 @@ void Manager::DataIn(const u_char* data, uint64 len, const string& unique)
 
 void Manager::DataIn(const u_char* data, uint64 len, File* file)
 	{
-	if ( ! file ) return;
+	if ( ! file )
+		return;
 
 	file->DataIn(data, len);
 
@@ -93,7 +102,8 @@ void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn)
 
 void Manager::EndOfFile(AnalyzerTag::Tag tag, Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) ) return;
+	if ( IsDisabled(tag) )
+		return;
 
 	GetFileHandle(tag, conn, is_orig);
 	EndOfFile(current_handle);
@@ -107,7 +117,8 @@ void Manager::EndOfFile(const string& unique)
 void Manager::Gap(uint64 offset, uint64 len, AnalyzerTag::Tag tag,
                   Connection* conn, bool is_orig)
 	{
-	if ( IsDisabled(tag) ) return;
+	if ( IsDisabled(tag) )
+		return;
 
 	GetFileHandle(tag, conn, is_orig);
 	Gap(offset, len, GetFile(current_handle, conn, tag, is_orig));
@@ -120,7 +131,8 @@ void Manager::Gap(uint64 offset, uint64 len, const string& unique)
 
 void Manager::Gap(uint64 offset, uint64 len, File* file)
 	{
-	if ( ! file ) return;
+	if ( ! file )
+		return;
 
 	file->Gap(offset, len);
 	}
@@ -128,7 +140,8 @@ void Manager::Gap(uint64 offset, uint64 len, File* file)
 void Manager::SetSize(uint64 size, AnalyzerTag::Tag tag, Connection* conn,
                       bool is_orig)
 	{
-	if ( IsDisabled(tag) ) return;
+	if ( IsDisabled(tag) )
+		return;
 
 	GetFileHandle(tag, conn, is_orig);
 	SetSize(size, GetFile(current_handle, conn, tag, is_orig));
@@ -141,7 +154,8 @@ void Manager::SetSize(uint64 size, const string& unique)
 
 void Manager::SetSize(uint64 size, File* file)
 	{
-	if ( ! file ) return;
+	if ( ! file )
+		return;
 
 	file->SetTotalBytes(size);
 
@@ -153,7 +167,8 @@ bool Manager::PostponeTimeout(const FileID& file_id) const
 	{
 	File* file = Lookup(file_id);
 
-	if ( ! file ) return false;
+	if ( ! file )
+		return false;
 
 	file->postpone_timeout = true;
 	return true;
@@ -163,7 +178,8 @@ bool Manager::SetTimeoutInterval(const FileID& file_id, double interval) const
 	{
 	File* file = Lookup(file_id);
 
-	if ( ! file ) return false;
+	if ( ! file )
+		return false;
 
 	file->SetTimeoutInterval(interval);
 	return true;
@@ -173,7 +189,8 @@ bool Manager::AddAnalyzer(const FileID& file_id, RecordVal* args) const
 	{
 	File* file = Lookup(file_id);
 
-	if ( ! file ) return false;
+	if ( ! file )
+		return false;
 
 	return file->AddAnalyzer(args);
 	}
@@ -182,7 +199,8 @@ bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const
 	{
 	File* file = Lookup(file_id);
 
-	if ( ! file ) return false;
+	if ( ! file )
+		return false;
 
 	return file->RemoveAnalyzer(args);
 	}
@@ -190,8 +208,11 @@ bool Manager::RemoveAnalyzer(const FileID& file_id, const RecordVal* args) const
 File* Manager::GetFile(const string& unique, Connection* conn,
                        AnalyzerTag::Tag tag, bool is_orig, bool update_conn)
 	{
-	if ( unique.empty() ) return 0;
+	if ( unique.empty() )
-	if ( IsIgnored(unique) ) return 0;
+		return 0;
+
+	if ( IsIgnored(unique) )
+		return 0;
 
 	File* rval = str_map[unique];
 
@@ -208,11 +229,14 @@ File* Manager::GetFile(const string& unique, Connection* conn,
 
 		id_map[id] = rval;
 		rval->ScheduleInactivityTimer();
-		if ( IsIgnored(unique) ) return 0;
+
+		if ( IsIgnored(unique) )
+			return 0;
 		}
 	else
 		{
 		rval->UpdateLastActivityTime();
+
 		if ( update_conn )
 			rval->UpdateConnectionFields(conn);
 		}
@@ -224,7 +248,8 @@ File* Manager::Lookup(const FileID& file_id) const
 	{
 	IDMap::const_iterator it = id_map.find(file_id);
 
-	if ( it == id_map.end() ) return 0;
+	if ( it == id_map.end() )
+		return 0;
 
 	return it->second;
 	}
@@ -233,7 +258,8 @@ void Manager::Timeout(const FileID& file_id, bool is_terminating)
 	{
 	File* file = Lookup(file_id);
 
-	if ( ! file ) return;
+	if ( ! file )
+		return;
 
 	file->postpone_timeout = false;
 
@@ -258,7 +284,8 @@ bool Manager::IgnoreFile(const FileID& file_id)
 	{
 	IDMap::iterator it = id_map.find(file_id);
 
-	if ( it == id_map.end() ) return false;
+	if ( it == id_map.end() )
+		return false;
 
 	DBG_LOG(DBG_FILE_ANALYSIS, "Ignore FileID %s", file_id.c_str());
 
@@ -271,7 +298,8 @@ bool Manager::RemoveFile(const string& unique)
 	{
 	StrMap::iterator it = str_map.find(unique);
 
-	if ( it == str_map.end() ) return false;
+	if ( it == str_map.end() )
+		return false;
 
 	it->second->EndOfFile();
 
@@ -297,7 +325,8 @@ void Manager::GetFileHandle(AnalyzerTag::Tag tag, Connection* c, bool is_orig)
 	{
 	current_handle.clear();
 
-	if ( ! get_file_handle ) return;
+	if ( ! get_file_handle )
+		return;
 
 	val_list* vl = new val_list();
 	vl->append(new Val(tag, TYPE_COUNT));
@@ -317,7 +346,8 @@ bool Manager::IsDisabled(AnalyzerTag::Tag tag)
 	Val* yield = disabled->Lookup(index);
 	Unref(index);
 
-	if ( ! yield ) return false;
+	if ( ! yield )
+		return false;
 
 	bool rval = yield->AsBool();
 	Unref(yield);

src/file_analysis/Manager.h

@@ -1,3 +1,5 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
 #ifndef FILE_ANALYSIS_MANAGER_H
 #define FILE_ANALYSIS_MANAGER_H
 
@@ -24,12 +26,8 @@ namespace file_analysis {
  * Main entry point for interacting with file analysis.
  */
 class Manager {
-friend class FileTimer;
-
 public:
-
 	Manager();
-
 	~Manager();
 
 	/**
@@ -121,6 +119,7 @@ public:
 	bool IsIgnored(const string& unique);
 
 protected:
+	friend class FileTimer;
 
 	typedef map<string, File*> StrMap;
 	typedef set<string> StrSet;
@@ -167,6 +166,7 @@ protected:
 	 */
 	static bool IsDisabled(AnalyzerTag::Tag tag);
 
+private:
 	StrMap str_map;	/**< Map unique string to file_analysis::File. */
 	IDMap id_map;	/**< Map file ID to file_analysis::File records. */
 	StrSet ignored;	/**< Ignored files.  Will be finally removed on EOF. */

src/input.bif

@@ -60,3 +60,8 @@ const timedspread: double;
 
 module InputBinary;
 const chunk_size: count;
+
+module InputSQLite;
+const set_separator: string;
+const unset_field: string;
+const empty_field: string;

src/input/Manager.cc

@@ -9,6 +9,7 @@
 #include "readers/Raw.h"
 #include "readers/Benchmark.h"
 #include "readers/Binary.h"
+#include "readers/SQLite.h"
 
 #include "Event.h"
 #include "EventHandler.h"
@@ -36,6 +37,7 @@ ReaderDefinition input_readers[] = {
 	{ BifEnum::Input::READER_RAW, "Raw", 0, reader::Raw::Instantiate },
 	{ BifEnum::Input::READER_BENCHMARK, "Benchmark", 0, reader::Benchmark::Instantiate },
 	{ BifEnum::Input::READER_BINARY, "Binary", 0, reader::Binary::Instantiate },
+	{ BifEnum::Input::READER_SQLITE, "SQLite", 0, reader::SQLite::Instantiate },
 
 	// End marker
 	{ BifEnum::Input::READER_DEFAULT, "None", 0, (ReaderBackend* (*)(ReaderFrontend* frontend))0 }
@@ -745,6 +747,8 @@ bool Manager::RemoveStream(Stream *i)
 	DBG_LOG(DBG_INPUT, "Successfully queued removal of stream %s",
 		i->name.c_str());
 
+	i->reader->Stop();
+
 	return true;
 	}
 
@@ -2116,19 +2120,25 @@ Val* Manager::ValueToVal(const Value* val, BroType* request_type)
 		}
 
 	case TYPE_ENUM: {
-		// well, this is kind of stupid, because EnumType just mangles the module name and the var name together again...
+		// Convert to string first to not have to deal with missing
-		// but well
+		// \0's...
-		string module = extract_module_name(val->val.string_val.data);
+		string module_string(val->val.string_val.data, val->val.string_val.length);
-		string var = extract_var_name(val->val.string_val.data);
+		string var_string(val->val.string_val.data, val->val.string_val.length);
+
+		string module = extract_module_name(module_string.c_str());
+		string var = extract_var_name(var_string.c_str());
+
+		// Well, this is kind of stupid, because EnumType just
+		// mangles the module name and the var name together again...
+		// but well.
 		bro_int_t index = request_type->AsEnumType()->Lookup(module, var.c_str());
 		if ( index == -1 )
-			reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s",
+			reporter->InternalError("Value not found in enum mappimg. Module: %s, var: %s, var size: %zu",
-			                        module.c_str(), var.c_str());
+			                        module.c_str(), var.c_str(), var.size());
 
 		return new EnumVal(index, request_type->Ref()->AsEnumType());
 		}
 
-
 	default:
 		reporter->InternalError("unsupported type for input_read");
 	}

src/input/ReaderFrontend.cc

@@ -49,6 +49,15 @@ ReaderFrontend::ReaderFrontend(const ReaderBackend::ReaderInfo& arg_info, EnumVa
 	backend->Start();
 	}
 
+void ReaderFrontend::Stop()
+	{
+	if ( backend )
+		{
+		backend->SignalStop();
+		backend = 0; // Thread manager will clean it up once it finishes.
+		}
+	}
+
 ReaderFrontend::~ReaderFrontend()
 	{
 	delete [] name;

src/input/ReaderFrontend.h

@@ -75,7 +75,7 @@ public:
 	 * the corresponding message there. This method must only be called
 	 * from the main thread.
 	 */
-	void Close();
+	void Stop();
 
 	/**
 	 * Disables the reader frontend. From now on, all method calls that

src/input/readers/Binary.cc

@@ -19,6 +19,7 @@ Binary::Binary(ReaderFrontend *frontend)
 	if ( ! chunk_size )
 		{
 		chunk_size = BifConst::InputBinary::chunk_size;
+
 		if ( ! chunk_size )
 			chunk_size = 1024;
 		}
@@ -99,12 +100,14 @@ bool Binary::DoInit(const ReaderInfo& info, int num_fields,
 		return false;
 		}
 
-	// do Initialization
+	// do initialization
 	fname = info.source;
 
-	if ( ! OpenInput() ) return false;
+	if ( ! OpenInput() )
+		return false;
 
-	if ( UpdateModificationTime() == -1 ) return false;
+	if ( UpdateModificationTime() == -1 )
+		return false;
 
 #ifdef DEBUG
 	Debug(DBG_INPUT, "Binary reader created, will perform first update");
@@ -198,6 +201,7 @@ bool Binary::DoUpdate()
 				}
 
 			CloseInput();
+
 			if ( ! OpenInput() )
 				return false;