Merge remote-tracking branch 'origin/master' into topic/seth/unified2-analyzer

Conflicts:
	src/CMakeLists.txt

.gitmodules

@@ -16,3 +16,6 @@
 [submodule "cmake"]
 	path = cmake
 	url = git://git.bro-ids.org/cmake
+[submodule "magic"]
+	path = magic
+	url = git://git.bro.org/bromagic

CMakeLists.txt

@@ -1,5 +1,5 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
+cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
 include(cmake/CommonCMakeConfig.cmake)
 
 ########################################################################
@@ -17,12 +17,17 @@ set(BRO_SCRIPT_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/scripts)
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
     ABSOLUTE)
 
+set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic)
+set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database)
+
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
+     "export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
+     "setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -69,6 +74,12 @@ if (MISSING_PREREQS)
     message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()
 
+set(libmagic_req 5.04)
+if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
+    message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
+                        "(found ${LibMagic_VERSION})")
+endif ()
+
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${OpenSSL_INCLUDE_DIR}
@@ -88,16 +99,65 @@ if (LIBGEOIP_FOUND)
     list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
 endif ()
 
-set(USE_PERFTOOLS false)
+set(HAVE_PERFTOOLS false)
-if (ENABLE_PERFTOOLS)
+set(USE_PERFTOOLS_DEBUG false)
-    find_package(GooglePerftools)
+set(USE_PERFTOOLS_TCMALLOC false)
-    if (GOOGLEPERFTOOLS_FOUND)
+
-        set(USE_PERFTOOLS true)
+if (NOT DISABLE_PERFTOOLS)
-        include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+   find_package(GooglePerftools)
-        list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
+endif ()
+
+if (GOOGLEPERFTOOLS_FOUND)
+    set(HAVE_PERFTOOLS true)
+    # Non-Linux systems may not be well-supported by gperftools, so
+    # require explicit request from user to enable it in that case.
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS)
+        set(USE_PERFTOOLS_TCMALLOC true)
+
+        if (ENABLE_PERFTOOLS_DEBUG)
+            # Enable heap debugging with perftools.
+            set(USE_PERFTOOLS_DEBUG true)
+            include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+            list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
+        else ()
+            # Link in tcmalloc for better performance.
+            list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
+        endif ()
     endif ()
 endif ()
 
+set(USE_DATASERIES false)
+find_package(Lintel)
+find_package(DataSeries)
+find_package(LibXML2)
+
+if (NOT DISABLE_DATASERIES AND
+    LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
+    set(USE_DATASERIES true)
+	include_directories(BEFORE ${Lintel_INCLUDE_DIR})
+	include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
+	include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
+	list(APPEND OPTLIBS ${Lintel_LIBRARIES})
+	list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
+	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
+endif()
+
+set(USE_ELASTICSEARCH false)
+set(USE_CURL false)
+find_package(LibCURL)
+
+if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
+  set(USE_ELASTICSEARCH true)
+  set(USE_CURL true)
+  include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
+  list(APPEND OPTLIBS ${LibCURL_LIBRARIES})
+endif()
+
+if (ENABLE_PERFTOOLS_DEBUG)
+    # Just a no op to prevent CMake from complaining about manually-specified
+    # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
+endif ()
+
 set(brodeps
     ${BinPAC_LIBRARY}
     ${PCAP_LIBRARY}
@@ -141,12 +201,16 @@ CheckOptionalBuildSources(aux/broctl   Broctl   INSTALL_BROCTL)
 CheckOptionalBuildSources(aux/bro-aux  Bro-Aux  INSTALL_AUX_TOOLS)
 CheckOptionalBuildSources(aux/broccoli Broccoli INSTALL_BROCCOLI)
 
+install(DIRECTORY ./magic/database/
+        DESTINATION ${BRO_MAGIC_INSTALL_PATH}
+)
+
 ########################################################################
 ## Packaging Setup
 
 if (INSTALL_BROCTL)
     # CPack RPM Generator may not automatically detect this
-    set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.4.0")
+    set(CPACK_RPM_PACKAGE_REQUIRES "python >= 2.6.0")
 endif ()
 
 # If this CMake project is a sub-project of another, we will not
@@ -182,7 +246,13 @@ message(
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
-    "\nGoogle perftools:  ${USE_PERFTOOLS}"
+    "\ngperftools found:  ${HAVE_PERFTOOLS}"
+    "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
+    "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
+    "\ncURL:              ${USE_CURL}"
+    "\n"
+    "\nDataSeries:        ${USE_DATASERIES}"
+    "\nElasticSearch:     ${USE_ELASTICSEARCH}"
     "\n"
     "\n================================================================\n"
 )

COPYING

@@ -1,4 +1,4 @@
-Copyright (c) 1995-2011, The Regents of the University of California
+Copyright (c) 1995-2012, The Regents of the University of California
 through the Lawrence Berkeley National Laboratory and the
 International Computer Science Institute. All rights reserved.
 

INSTALL

@@ -1,80 +1,304 @@
+.. _CMake: http://www.cmake.org
+.. _SWIG: http://www.swig.org
+.. _Xcode: https://developer.apple.com/xcode/
+.. _MacPorts: http://www.macports.org
+.. _Fink: http://www.finkproject.org
+.. _Homebrew: http://mxcl.github.com/homebrew
+.. _bro downloads page: http://bro.org/download/index.html
+
 ==============
 Installing Bro
 ==============
 
+Bro can be downloaded in either pre-built binary package or
+source code forms.
+
 Prerequisites
 =============
 
-Bro relies on the following libraries and tools, which need to be installed
+Bro requires the following libraries and tools to be installed
 before you begin:
 
-    * CMake 2.6 or greater              http://www.cmake.org
+    * Libpcap                           http://www.tcpdump.org
 
-    * Libpcap (headers and libraries)   http://www.tcpdump.org
+    * OpenSSL libraries                 http://www.openssl.org
 
-    * OpenSSL (headers and libraries)   http://www.openssl.org
+    * BIND8 library
-
-    * SWIG                              http://www.swig.org
 
     * Libmagic
 
     * Libz
 
-Bro can make uses of some optional libraries if they are found at
+    * Bash  (for BroControl)
-installation time:
 
-    * LibGeoIP  For geo-locating IP addresses.
+To build Bro from source, the following additional dependencies are required:
 
-Bro also needs the following tools, but on most systems they will
+    * CMake 2.6.3 or greater            http://www.cmake.org
-already come preinstalled:
+
+    * SWIG                              http://www.swig.org
 
-    * BIND8 (headers and libraries)
     * Bison (GNU Parser Generator)
+
     * Flex  (Fast Lexical Analyzer)
-    * Perl  (Used only during the Bro build process)
+
+    * Libpcap headers                   http://www.tcpdump.org
+
+    * OpenSSL headers                   http://www.openssl.org
+
+    * libmagic headers
+
+    * zlib headers
+
+    * Perl
+
+Bro can make use of some optional libraries and tools if they are found at
+build time:
+
+    * LibGeoIP (for geo-locating IP addresses)
+
+    * gperftools (tcmalloc is used to improve memory and CPU usage)
+
+    * sendmail (for BroControl)
+
+    * ipsumdump (for trace-summary)     http://www.cs.ucla.edu/~kohler/ipsumdump
+
+    * Ruby executable, library, and headers (for Broccoli Ruby bindings)
 
 
-Installation
+Installing From Pre-Built Binary Release Packages
-============
+=================================================
 
-To build and install into ``/usr/local/bro``::
+See the `bro downloads page`_ for currently supported/targeted platforms.
+
+* RPM
+
+  .. console::
+
+      sudo yum localinstall Bro-*.rpm
+
+* DEB
+
+  .. console::
+
+      sudo gdebi Bro-*.deb
+
+* MacOS Disk Image with Installer
+
+  Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer.
+  Everything installed by the package will go into ``/opt/bro``.
+
+The primary install prefix for binary packages is ``/opt/bro``.
+Non-MacOS packages that include BroControl also put variable/runtime
+data (e.g. Bro logs) in ``/var/opt/bro``.
+
+
+Installing From Source
+======================
+
+Required Dependencies
+~~~~~~~~~~~~~~~~~~~~~
+
+The following dependencies are required to build Bro:
+
+* RPM/RedHat-based Linux:
+
+  .. console::
+
+     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+
+* DEB/Debian-based Linux:
+
+  .. console::
+
+     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+
+* FreeBSD
+
+  Most required dependencies should come with a minimal FreeBSD install
+  except for the following.
+
+  .. console::
+
+      sudo pkg_add -r bash cmake swig bison python
+
+  Note that ``bash`` needs to be in ``PATH``, which by default it is
+  not. The FreeBSD package installs the binary into
+  ``/usr/local/bin``.
+
+* Mac OS X
+
+  Compiling source code on Macs requires first downloading Xcode_,
+  then going through its "Preferences..." -> "Downloads" menus to
+  install the "Command Line Tools" component.
+
+  Lion (10.7) and Mountain Lion (10.8) come with all required
+  dependencies except for CMake_, SWIG_, and ``libmagic``.
+
+  Distributions of these dependencies can likely be obtained from your
+  preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
+  or Homebrew_).
+
+  Specifically for MacPorts, the ``swig``, ``swig-ruby``, ``swig-python``
+  and ``file`` packages provide the required dependencies.
+
+Optional Dependencies
+~~~~~~~~~~~~~~~~~~~~~
+
+Bro can use libGeoIP for geo-locating IP addresses, and sendmail for
+sending emails.
+
+* RedHat Enterprise Linux:
+
+  .. console::
+
+      sudo yum install geoip-devel sendmail
+
+* CentOS Linux:
+
+  .. console::
+  
+      sudo yum install GeoIP-devel sendmail
+
+* DEB/Debian-based Linux:
+
+  .. console::
+
+      sudo apt-get install libgeoip-dev sendmail
+
+* Ports-based FreeBSD
+
+  .. console::
+
+      sudo pkg_add -r GeoIP
+
+  sendmail is typically already available.
+
+* Mac OS X
+
+  Vanilla OS X installations don't ship with libGeoIP, but
+  if installed from your preferred package management system (e.g. MacPorts,
+  Fink, or Homebrew), they should be automatically detected and Bro will
+  compile against them.
+
+Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`.
+
+Compiling Bro Source Code
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Bro releases are bundled into source packages for convenience and
+available from the `bro downloads page`_.
+
+Alternatively, the latest Bro development version can be obtained through git
+repositories hosted at `git.bro.org <http://git.bro.org>`_.  See
+our `git development documentation
+<http://bro.org/development/process.html>`_ for comprehensive
+information on Bro's use of git revision control, but the short story
+for downloading the full source code experience for Bro via git is:
+
+.. console::
+
+    git clone --recursive git://git.bro.org/bro
+
+.. note:: If you choose to clone the ``bro`` repository non-recursively for
+   a "minimal Bro experience", be aware that compiling it depends on
+   BinPAC, which has its own ``binpac`` repository.  Either install it
+   first or initialize/update the cloned ``bro`` repository's
+   ``aux/binpac`` submodule.
+
+The typical way to build and install from source is (for more options,
+run ``./configure --help``):
+
+.. console::
 
     ./configure
     make
     make install
 
-This will first build Bro into a directory inside the distribution
+The default installation path is ``/usr/local/bro``, which would typically
-called ``build/``, using default build options. It then installs all
+require root privileges when doing the ``make install``.  A different 
-required files into ``/usr/local/bro``, including the Bro binary in
+installation path can be chosen by specifying the ``--prefix`` option.
-``/usr/local/bro/bin/bro``.
+Note that ``/usr`` and ``/opt/bro`` are the
-
+standard prefixes for binary Bro packages to be installed, so those are
-You can specify a different installation directory with::
+typically not good choices unless you are creating such a package.
-
-    ./configure --prefix=<dir>
-
-Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for
-binary Bro packages to be installed, so those are typically not good
-choices unless you are creating such a package.
-
-Run ``./configure --help`` for more options.
 
 Depending on the Bro package you downloaded, there may be auxiliary
-tools and libraries available in the ``aux/`` directory. All of them
+tools and libraries available in the ``aux/`` directory. Some of them
-except for ``aux/bro-aux`` will also be built and installed by doing
+will be automatically built and installed along with Bro. There are
-``make install``.  To install the programs that come in the
-``aux/bro-aux`` directory, use ``make install-aux``. There are
 ``--disable-*`` options that can be given to the configure script to
-turn off unwanted auxiliary projects.
+turn off unwanted auxiliary projects that would otherwise be installed
+automatically.  Finally, use ``make install-aux`` to install some of
+the other programs that are in the ``aux/bro-aux`` directory.
 
+OpenBSD users, please see our FAQ at
+http://www.bro.org/documentation/faq.html if you are having
+problems installing Bro.
+
+
+Upgrading From a Previous Version of Bro
+========================================
+
+If you're doing an upgrade install (rather than a fresh install),
+there's two suggested approaches: either install Bro using the same
+installation prefix directory as before, or pick a new prefix and copy
+local customizations over.
+
+Re-Use Previous Install Prefix
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you choose to configure and install Bro with the same prefix
+directory as before, local customization and configuration to files in
+``$prefix/share/bro/site`` and ``$prefix/etc`` won't be overwritten
+(``$prefix`` indicating the root of where Bro was installed). Also, logs
+generated at run-time won't be touched by the upgrade. (But making
+a backup of local changes before upgrading is still recommended.)
+
+After upgrading, remember to check ``$prefix/share/bro/site`` and
+``$prefix/etc`` for ``.example`` files, which indicate the
+distribution's version of the file differs from the local one, which may
+include local changes.  Review the differences, and make adjustments
+as necessary (for differences that aren't the result of a local change,
+use the new version's).
+
+Pick a New Install prefix
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to install the newer version in a different prefix
+directory than before, you can just copy local customization and
+configuration files from ``$prefix/share/bro/site`` and ``$prefix/etc``
+to the new location (``$prefix`` indicating the root of where Bro was
+originally installed).  Make sure to review the files for difference
+before copying and make adjustments as necessary (for differences that
+aren't the result of a local change, use the new version's).  Of
+particular note, the copied version of ``$prefix/etc/broctl.cfg`` is
+likely to need changes to the ``SpoolDir`` and ``LogDir`` settings.
+
+
+Configure the Run-Time Environment
+==================================
+
+Just remember that you may need to adjust your ``PATH`` environment variable
+according to the platform/shell/package you're using.  For example:
+
+Bourne-Shell Syntax:
+
+.. console::
+
+   export PATH=/usr/local/bro/bin:$PATH
+
+C-Shell Syntax:
+
+.. console::
+
+   setenv PATH /usr/local/bro/bin:$PATH
+
+Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
 
 Running Bro
 ===========
 
 Bro is a complex program and it takes a bit of time to get familiar
-with it.  A good place for newcomers to start is the Quickstart Guide at
+with it.  A good place for newcomers to start is the Quick Start Guide
-
+at http://www.bro.org/documentation/quickstart.html.
-    http://www.bro-ids.org/documentation/quickstart.bro.html
-
 
 For developers that wish to run Bro directly from the ``build/``
 directory (i.e., without performing ``make install``), they will have

Makefile

@@ -2,7 +2,7 @@
 # A simple static wrapper for a number of standard Makefile targets,
 # mostly just forwarding to build/Makefile. This is provided only for
 # convenience and supports only a subset of what CMake's Makefile
-# to offer. For more, execute that one directly. 
+# offers. For more, execute that one directly. 
 #
 
 BUILD=build
@@ -61,7 +61,10 @@ distclean:
 	rm -rf $(BUILD)
 
 test:
-	@(cd testing && make )
+	@( cd testing && make )
+
+test-all: test
+	test -d aux/broctl && ( cd aux/broctl && make test )
 
 configured:
 	@test -d $(BUILD) || ( echo "Error: No build/ directory found. Did you run configure?" && exit 1 )

NEWS

@@ -3,15 +3,418 @@ Release Notes
 =============
 
 This document summarizes the most important changes in the current Bro
-release. For a complete list of changes, see the ``CHANGES`` file.
+release. For a complete list of changes, see the ``CHANGES`` file
+(note that submodules, such as BroControl and Broccoli, come with
+their own CHANGES.)
+
+Bro 2.2
+-------
+
+New Functionality
+~~~~~~~~~~~~~~~~~
+
+- GPRS Tunnelling Protocol (GTPv1) decapsulation.
+
+- GridFTP support. TODO: Extend.
+
+- ssl.log now also records the subject client and issuer certificates.
+
+- Hooks: TODO: Briefly summarize the documention from
+  doc/scripts/builtins.rst here.
+
+- The ASCII writer can now output CSV files on a per filter basis.
+
+- Bro's language now has a working "switch" statement that generally
+  behaves like C-style switches except case labels can be comprised of
+  multiple literal constants delimited by commas.  Only atomic types
+  are allowed for now.  Case label bodies that don't execute a
+  "return" or "break" statement will fall through to subsequent cases.
+  A default case label is allowed.
+
+- Bro's language now has a new set of types "opaque of X". Opaque
+  values can be passed around like other values but they can only be
+  manipulated with BiF functions, not with other operators. Currently,
+  the following opaque types are supported:
+
+        - opaque of md5
+        - opaque of sha1
+        - opaque of sha256
+        - opaquey of entropy.
+
+  They go along with the corrsponding BiF functions md5_*, sha1_*,
+  sha256_*, and entropy_*, respectively. Note that these functions
+  have changed their signatures to work with opaques types rather
+  than global state as it was before.
+
+- The scripting language now supports a constructing sets, tables,
+  vectors, and records by name:
+
+        type MyRecordType: record {
+            c: count;
+            s: string &optional;
+        };
+
+        global r: MyRecordType = record($c = 7);
+
+        type MySet: set[MyRec];
+        global s = MySet([$c=1], [$c=2]);
+
+- Strings now support the subscript operator to extract individual
+  characters and substrings (e.g., s[4], s[1,5]). The index expression
+  can take up to two indices for the start and end index of the
+  substring to return (e.g. "mystring[1,3]").
+
+- Functions now support default parameters, e.g.:
+
+        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+
+- Scripts can now use two new "magic constants" @DIR and @FILENAME
+  that expand to the directory path of the current script and just the
+  script file name without path, respectively. (Jon Siwek)
+
+- The new file analysis framework moves most of the processing of file
+  content from script-land into the core, where it belongs. See
+  doc/file-analysis.rst for more information.
+
+  Much of this is an internal change, but the framework also comes
+  with the following user-visibible functionality (some of that was
+  already available before, but done differently):
+
+  [TODO: Update with changes from 984e9793db56.]
+
+      - A binary input reader interfaces the input framework with file
+        analysis, allowing to inject files on disk into Bro's
+        processing.
+
+      - Supports for analyzing data transfereed via HTTP range
+        requests.
+
+      - HTTP:
+        * Identify MIME type of message.
+        * Extract message to disk.
+        * Compute MD5 for messages.
+
+      - SMTP:
+        * Identify MIME type of message.
+        * Extract message to disk.
+        * Compute MD5 for messages.
+        * Provide access to start of entity data.
+
+      - FTP data transfers: Identify MIME type; record to disk.
+
+      - IRC DCC transfers: Record to disk.
+
+- New packet filter framework supports BPF-based load-balancing,
+  shunting, and sampling; plus plugin support to customize filters
+  dynamically.
+
+- Bro now provides Bloom filters of two kinds: basic Bloom filters
+  supporting membership tests, and counting Bloom filters that track
+  the frequency of elements. The corresponding functions are:
+
+    bloomfilter_basic_init(fp: double, capacity: count, name: string &default=""): opaque of bloomfilter
+    bloomfilter_basic_init2(k: count, cells: count, name: string &default=""): opaque of bloomfilter
+    bloomfilter_counting_init(k: count, cells: count, max: count, name: string &default=""): opaque of bloomfilter
+    bloomfilter_add(bf: opaque of bloomfilter, x: any)
+    bloomfilter_lookup(bf: opaque of bloomfilter, x: any): count
+    bloomfilter_merge(bf1: opaque of bloomfilter, bf2: opaque of bloomfilter): opaque of bloomfilter
+    bloomfilter_clear(bf: opaque of bloomfilter)
+
+  See <INSERT LINK> for full documentation.
+
+- Bro now provides a probabilistic data structure for computing
+  "top k" elements. The corresponding functions are:
+
+    topk_init(size: count): opaque of topk
+    topk_add(handle: opaque of topk, value: any)
+    topk_get_top(handle: opaque of topk, k: count)
+    topk_count(handle: opaque of topk, value: any): count
+    topk_epsilon(handle: opaque of topk, value: any): count
+    topk_size(handle: opaque of topk): count
+    topk_sum(handle: opaque of topk): count
+    topk_merge(handle1: opaque of topk, handle2: opaque of topk)
+    topk_merge_prune(handle1: opaque of topk, handle2: opaque of topk)
+
+  See <INSERT LINK> for full documentation.
+
+- base/utils/exec.bro provides a module to start external processes
+  asynchronously and retrieve their output on termination.
+  base/utils/dir.bro uses it to monitor a directory for changes, and
+  base/utils/active-http.bro for providing an interface for querying
+  remote web servers.
+
+Changed Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+- We removed the following, already deprecated, functionality:
+
+    * Scripting language:
+        - &disable_print_hook attribute.
+
+    * BiF functions:
+        - parse_dotted_addr(), dump_config(),
+          make_connection_persistent(), generate_idmef(),
+          split_complete()
+
+        - md5_*, sha1_*, sha256_*, and entropy_* have all changed
+        their signatures to work with opaque types (see above).
+
+
+- Removed a now unused argument from "do_split" helper function.
+
+- "this" is no longer a reserved keyword.
+
+- The Input Framework's update_finished event has been renamed to
+  end_of_data. It will now not only fire after table-reads have been
+  completed, but also after the last event of a whole-file-read (or
+  whole-db-read, etc.).
+
+- Renamed the option defining the frequency of alarm summary mails to
+  'Logging::default_alarm_mail_interval'. When using BroControl, the
+  value can now be set with the new broctl.cfg option
+  "MailAlarmsInterval".
+
+- We have completely reworded the "notice_policy" mechanism. It now no
+  linger uses a record of policy items but a "hook", a new language
+  element that's roughly equivalent to a function with multiple
+  bodies. The documentation [TODO: insert link] describes how to use
+  the new notice policy. For existing code, the two main changes are:
+
+    - What used to be a "redef" of "Notice::policy" now becomes a hook
+      implementation. Example:
+
+      Old:
+
+        redef Notice::policy += {
+            [$pred(n: Notice::Info) = {
+                return n$note == SSH::Login && n$id$resp_h == 10.0.0.1;
+                },
+            $action = Notice::ACTION_EMAIL]
+            };
+
+      New:
+
+        hook Notice::policy(n: Notice::Info)
+            {
+            if ( n$note == SSH::Login && n$id$resp_h == 10.0.0.1 )
+                add n$actions[Notice::ACTION_EMAIL];
+            }
+
+    - notice() is now likewise a hook, no longer an event. If you have
+      handlers for that event, you'll likely just need to change the
+      type accordingly. Example:
+
+      Old:
+
+        event notice(n: Notice::Info) { ... }
+
+      New:
+
+        hook notice(n: Notice::Info) { ... }
+
+- The notice_policy.log is gone. That's a result of the new notice
+  policy setup.
+
+- Removed the byte_len() and length() bif functions. Use the "|...|"
+  operator instead.
+
+- The SSH::Login notice has been superseded by an corresponding
+  intelligence framework observation (SSH::SUCCESSFUL_LOGIN).
+
+- PacketFilter::all_packets has been replaced with
+  PacketFilter::enable_auto_protocol_capture_filters.
+
+- We removed the BitTorrent DPD signatures pending further updates to
+  that analyzer.
+
+Bro 2.1
+-------
+
+New Functionality
+~~~~~~~~~~~~~~~~~
+
+- Bro now comes with extensive IPv6 support. Past versions offered
+  only basic IPv6 functionality that was rarely used in practice as it
+  had to be enabled explicitly. IPv6 support is now fully integrated
+  into all parts of Bro including protocol analysis and the scripting
+  language. It's on by default and no longer requires any special
+  configuration.
+
+  Some of the most significant enhancements include support for IPv6
+  fragment reassembly, support for following IPv6 extension header
+  chains, and support for tunnel decapsulation (6to4 and Teredo). The
+  DNS analyzer now handles AAAA records properly, and DNS lookups that
+  Bro itself performs now include AAAA queries, so that, for example,
+  the result returned by script-level lookups is a set that can
+  contain both IPv4 and IPv6 addresses. Support for the most common
+  ICMPv6 message types has been added. Also, the FTP EPSV and EPRT
+  commands are now handled properly. Internally, the way IP addresses
+  are stored has been improved, so Bro can handle both IPv4
+  and IPv6 by default without any special configuration.
+
+  In addition to Bro itself, the other Bro components have also been
+  made IPv6-aware by default. In particular, significant changes were
+  made to trace-summary, PySubnetTree, and Broccoli to support IPv6.
+
+- Bro now decapsulates tunnels via its new tunnel framework located in
+  scripts/base/frameworks/tunnels. It currently supports Teredo,
+  AYIYA, IP-in-IP (both IPv4 and IPv6), and SOCKS. For all these, it
+  logs the outer tunnel connections in both conn.log and tunnel.log,
+  and then proceeds to analyze the inner payload as if it were not
+  tunneled, including also logging that session in conn.log. For
+  SOCKS, it generates a new socks.log in addition with more
+  information.
+
+- Bro now features a flexible input framework that allows users to
+  integrate external information in real-time into Bro while it's
+  processing network traffic. The most direct use-case at the moment
+  is reading data from ASCII files into Bro tables, with updates
+  picked up automatically when the file changes during runtime. See
+  doc/input.rst for more information.
+
+  Internally, the input framework is structured around the notion of
+  "reader plugins" that make it easy to interface to different data
+  sources. We will add more in the future.
+
+- BroControl now has built-in support for host-based load-balancing
+  when using either PF_RING, Myricom cards, or individual interfaces.
+  Instead of adding a separate worker entry in node.cfg for each Bro
+  worker process on each worker host, it is now possible to just
+  specify the number of worker processes on each host and BroControl
+  configures everything correctly (including any neccessary enviroment
+  variables for the balancers).
+
+  This change adds three new keywords to the node.cfg file (to be used
+  with worker entries): lb_procs (specifies number of workers on a
+  host), lb_method (specifies what type of load balancing to use:
+  pf_ring, myricom, or interfaces), and lb_interfaces (used only with
+  "lb_method=interfaces" to specify which interfaces to load-balance
+  on).
+
+- Bro's default ASCII log format is not exactly the most efficient way
+  for storing and searching large volumes of data. An alternatives,
+  Bro now comes with experimental support for two alternative output
+  formats:
+
+    * DataSeries: an efficient binary format for recording structured
+      bulk data. DataSeries is developed and maintained at HP Labs.
+      See doc/logging-dataseries for more information.
+
+    * ElasticSearch: a distributed RESTful, storage engine and search
+      engine built on top of Apache Lucene. It scales very well, both
+      for distributed indexing and distributed searching. See
+      doc/logging-elasticsearch.rst for more information.
+
+  Note that at this point, we consider Bro's support for these two
+  formats as prototypes for collecting experience with alternative
+  outputs. We do not yet recommend them for production (but welcome
+  feedback!)
+
+- Summary statistics framework. [Extend]
+
+- A number of new applications build on top of the summary statistics
+  framework:
+
+    * Scan detection: Detectors for port and address scans return. See
+      policy/misc/scan.bro.
+
+    * Tracerouter detector: policy/misc/detect-traceroute
+
+    * Web application detection/measurement: policy/misc/app-metrics.bro
+
+    * FTP brute-forcing detector: policy/protocols/ftp/detect-bruteforcing.bro
+
+    * HTTP-based SQL injection detector: policy/protocols/http/detect-sqli.bro
+      (existed before, but now ported to the new framework)
+
+    * SSH brute-forcing detector feeding the intelligence framework:
+      policy/protocols/ssh/detect-bruteforcing.bro
+
+
+
+Changed Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+The following summarizes the most important differences in existing
+functionality. Note that this list is not complete, see CHANGES for
+the full set.
+
+- Changes in dependencies:
+
+    * Bro now requires CMake >= 2.6.3.
+
+    * On Linux, Bro now links in tcmalloc (part of Google perftools)
+      if found at configure time. Doing so can significantly improve
+      memory and CPU use.
+
+      On the other platforms, the new configure option
+      --enable-perftools can be used to enable linking to tcmalloc.
+      (Note that perftools's support for non-Linux platforms may be
+      less reliable).
+
+- The configure switch --enable-brov6 is gone.
+
+- DNS name lookups performed by Bro now also query AAAA records. The
+  results of the A and AAAA queries for a given hostname are combined
+  such that at the scripting layer, the name resolution can yield a
+  set with both IPv4 and IPv6 addresses.
+
+- The connection compressor was already deprecated in 2.0 and has now
+  been removed from the code base.
+
+- We removed the "match" statement, which was no longer used by any of
+  the default scripts, nor was it likely to be used by anybody anytime
+  soon. With that, "match" and "using" are no longer reserved keywords.
+
+- The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012"
+  to "[2607:f8b0:4009:802::1012]".
+
+- Bro now spawns threads for doing its logging. From a user's
+  perspective not much should change, except that the OS may now show
+  a bunch of Bro threads.
+
+- We renamed the configure option --enable-perftools to
+  --enable-perftools-debug to indicate that the switch is only relevant
+  for debugging the heap.
+
+- Bro's ICMP analyzer now handles both IPv4 and IPv6 messages with a
+  joint set of events.  The `icmp_conn` record got a new boolean field
+  'v6' that indicates whether the ICMP message is v4 or v6.
+
+- Log postprocessor scripts get an additional argument indicating the
+  type of the log writer in use (e.g., "ascii").
+
+- BroControl's make-archive-name script also receives the writer
+  type, but as its 2nd(!) argument. If you're using a custom version
+  of that script, you need to adapt it. See the shipped version for
+  details.
+
+- Signature files can now be loaded via the new "@load-sigs"
+  directive. In contrast to the existing (and still supported)
+  signature_files constant, this can be used to load signatures
+  relative to the current script (e.g., "@load-sigs ./foo.sig").
+
+- The options "tunnel_port" and "parse_udp_tunnels" have been removed.
+  Bro now supports decapsulating tunnels directly for protocols it
+  understands.
+
+- ASCII logs now record the time when they were opened/closed at the
+  beginning and end of the file, respectively (wall clock). The
+  options LogAscii::header_prefix and LogAscii::include_header have
+  been renamed to LogAscii::meta_prefix and LogAscii::include_meta,
+  respectively.
+
+- The ASCII writers "header_*" options have been renamed to "meta_*"
+  (because there's now also a footer).
+
 
 Bro 2.0
 -------
 
 As the version number jump suggests, Bro 2.0 is a major upgrade and
-lots of things have changed. We have assembled a separate upprade
+lots of things have changed. We have assembled a separate upgrade
 guide with the most important changes compared to Bro 1.5 at
-http://www.bro-ids.org/documentation/upgrade.bro.html. You can find
+http://www.bro.org/documentation/upgrade.html. You can find
 the offline version of that document in ``doc/upgrade.rst.``.
 
 Compared to the earlier 2.0 Beta version, the major changes in the
@@ -19,7 +422,7 @@ final release are:
 
     * The default scripts now come with complete reference
       documentation. See
-      http://www.bro-ids.org/documentation/index.html.
+      http://www.bro.org/documentation/index.html.
 
     * libz and libmagic are now required dependencies.
 
@@ -32,8 +435,9 @@ final release are:
       be redefined on the command line, e.g. ``bro -i eth0
       snaplen=65535``.
 
-    * Reintroduced the BRO_LOG_SUFFIX environment that the ASCII
+    * Reintroduced the BRO_LOG_SUFFIX environment variable that the
-      logger now respects to add a suffix to the log files it creates.
+      ASCII logger now respects to add a suffix to the log files it
+      creates.
 
     * The ASCII logs now include further header information, and
       fields set to an empty value are now logged as ``(empty)`` by
@@ -43,9 +447,20 @@ final release are:
     * Some NOTICES were renamed, and the signatures of some SSL events
       have changed.
 
+    * bro-cut got some new capabilities:
+
+        - If no field names are given on the command line, we now pass
+          through all fields.
+
+        - New options -u/-U for time output in UTC.
+
+        - New option -F to give output field separator.
+
+    * Broccoli supports more types internally, allowing to send
+      complex records.
+
     * Many smaller bug fixes, portability improvements, and general
-      polishing.
+      polishing across all modules.
-
 
 
 

README

@@ -4,14 +4,14 @@ Bro Network Security Monitor
 
 Bro is a powerful framework for network analysis and security
 monitoring. Please see the INSTALL file for installation instructions
-and pointers for getting started. NEWS contains releases notes for the
+and pointers for getting started. NEWS contains release notes for the
 current version, and CHANGES has the complete history of changes.
 Please see COPYING for licensing information.
 
 For more documentation, research publications, and community contact
 information, please see Bro's home page:
 
-    http://www.bro-ids.org
+    http://www.bro.org
 
 On behalf of the Bro Development Team,
 

VERSION

@@ -1 +1 @@
-2.0-beta-177
+2.1-1036

aux/binpac

@@ -1 +1 @@
-Subproject commit e94d92b01f327655fd2061157942b95ae75b5f0f
+Subproject commit 314fa8f65fc240e960c23c3bba98623436a72b98

aux/bro-aux

@@ -1 +1 @@
-Subproject commit f6b92bf5732c26e54eb4387efadc612663980389
+Subproject commit d9963983c0b4d426b24836f8d154d014d5aecbba

aux/broccoli

@@ -1 +1 @@
-Subproject commit c5cee3d5746ed3d5c14348c1f264d19404caa761
+Subproject commit d59c73b6e0966ad63bbc63a35741b5f68263e7b1

aux/broctl

@@ -1 +1 @@
-Subproject commit d3d5934310a94452b1dddabb2e75f6c5c86b4860
+Subproject commit 090d4553ace0f9acf2d86eafab07ecfdcc534878

aux/btest

@@ -1 +1 @@
-Subproject commit 38890e851416fa9fc827a1d36f06c4cb9f7d4e69
+Subproject commit 69606f8f3cc84d694ca1da14868a5fecd4abbc96

bro-path-dev.in

@@ -12,7 +12,7 @@
 
 broPolicies=${BRO_SCRIPT_SOURCE_PATH}:${BRO_SCRIPT_SOURCE_PATH}/policy:${BRO_SCRIPT_SOURCE_PATH}/site
 
-broGenPolicies=${CMAKE_BINARY_DIR}/src
+broGenPolicies=${CMAKE_BINARY_DIR}/scripts
 
 installedPolicies=${BRO_SCRIPT_INSTALL_PATH}:${BRO_SCRIPT_INSTALL_PATH}/site
 

cmake

@@ -1 +1 @@
-Subproject commit 0c0a4697687df7f17c09391a1d0d95b25297a662
+Subproject commit 026639f8368e56742c0cb5d9fb390ea64e60ec50

config.h.in

@@ -1,6 +1,3 @@
-/* enable IPV6 processing */
-#cmakedefine BROv6
-
 /* Old libpcap versions (< 0.6.1) need defining pcap_freecode and
    pcap_compile_nopcap */
 #cmakedefine DONT_HAVE_LIBPCAP_PCAP_FREECODE
@@ -32,8 +29,8 @@
 /* Define if you have the <net/ethernet.h> header file. */
 #cmakedefine HAVE_NET_ETHERNET_H
 
-/* We are on a OpenBSD system */
+/* Define if you have the <net/ethertypes.h> header file. */
-#cmakedefine HAVE_OPENBSD
+#cmakedefine HAVE_NET_ETHERTYPES_H
 
 /* have os-proto.h */
 #cmakedefine HAVE_OS_PROTO_H
@@ -112,7 +109,19 @@
 #cmakedefine HAVE_GEOIP_CITY_EDITION_REV0_V6
 
 /* Use Google's perftools */
-#cmakedefine USE_PERFTOOLS
+#cmakedefine USE_PERFTOOLS_DEBUG
+
+/* Analyze Mobile IPv6 traffic */
+#cmakedefine ENABLE_MOBILE_IPV6
+
+/* Use libCurl. */
+#cmakedefine USE_CURL
+
+/* Use the DataSeries writer. */
+#cmakedefine USE_DATASERIES
+
+/* Use the ElasticSearch writer. */
+#cmakedefine USE_ELASTICSEARCH
 
 /* Version number of package */
 #define VERSION "@VERSION@"
@@ -145,3 +154,58 @@
 
 /* Define u_int8_t */
 #cmakedefine u_int8_t @u_int8_t@
+
+/* OpenBSD's bpf.h may not declare this data link type, but it's supposed to be
+   used consistently for the same purpose on all platforms. */
+#cmakedefine HAVE_DLT_PPP_SERIAL
+#ifndef HAVE_DLT_PPP_SERIAL
+#define DLT_PPP_SERIAL @DLT_PPP_SERIAL@
+#endif
+
+/* IPv6 Next Header values defined by RFC 3542 */
+#cmakedefine HAVE_IPPROTO_HOPOPTS
+#ifndef HAVE_IPPROTO_HOPOPTS
+#define IPPROTO_HOPOPTS 0
+#endif
+#cmakedefine HAVE_IPPROTO_IPV6
+#ifndef HAVE_IPPROTO_IPV6
+#define IPPROTO_IPV6 41
+#endif
+#cmakedefine HAVE_IPPROTO_IPV4
+#ifndef HAVE_IPPROTO_IPV4
+#define IPPROTO_IPV4 4
+#endif
+#cmakedefine HAVE_IPPROTO_ROUTING
+#ifndef HAVE_IPPROTO_ROUTING
+#define IPPROTO_ROUTING 43
+#endif
+#cmakedefine HAVE_IPPROTO_FRAGMENT
+#ifndef HAVE_IPPROTO_FRAGMENT
+#define IPPROTO_FRAGMENT 44
+#endif
+#cmakedefine HAVE_IPPROTO_ESP
+#ifndef HAVE_IPPROTO_ESP
+#define IPPROTO_ESP 50
+#endif
+#cmakedefine HAVE_IPPROTO_AH
+#ifndef HAVE_IPPROTO_AH
+#define IPPROTO_AH 51
+#endif
+#cmakedefine HAVE_IPPROTO_ICMPV6
+#ifndef HAVE_IPPROTO_ICMPV6
+#define IPPROTO_ICMPV6 58
+#endif
+#cmakedefine HAVE_IPPROTO_NONE
+#ifndef HAVE_IPPROTO_NONE
+#define IPPROTO_NONE 59
+#endif
+#cmakedefine HAVE_IPPROTO_DSTOPTS
+#ifndef HAVE_IPPROTO_DSTOPTS
+#define IPPROTO_DSTOPTS 60
+#endif
+
+/* IPv6 options structure defined by RFC 3542 */
+#cmakedefine HAVE_IP6_OPT
+
+/* Common IPv6 extension structure */
+#cmakedefine HAVE_IP6_EXT

configure

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Convenience wrapper for easily viewing/setting options that
 # the project's CMake scripts will recognize
-
+set -e
 command="$0 $*"
 
 # check for `cmake` command
@@ -24,16 +24,22 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --prefix=PREFIX        installation directory [/usr/local/bro]
     --scriptdir=PATH       root installation directory for Bro scripts
                            [PREFIX/share/bro]
+    --conf-files-dir=PATH  config files installation directory [PREFIX/etc]
 
   Optional Features:
     --enable-debug         compile in debugging mode
-    --enable-brov6         enable IPv6 processing
+    --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
-    --enable-perftools     use Google's perftools
+    --enable-perftools     force use of Google perftools on non-Linux systems
+                           (automatically on when perftools is present on Linux)
+    --enable-perftools-debug use Google's perftools for debugging
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
-    --disable-auxtools     don't build or install auxilliary tools
+    --disable-auxtools     don't build or install auxiliary tools
+    --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
+    --disable-dataseries   don't use the optional DataSeries log writer
+    --disable-elasticsearch don't use the optional ElasticSearch log writer
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -55,6 +61,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
+    --with-dataseries=PATH path to DataSeries and Lintel libraries
+    --with-xml2=PATH       path to libxml2 installation (for DataSeries)
+    --with-curl=PATH       path to libcurl install root (for ElasticSearch)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -86,20 +95,24 @@ append_cache_entry () {
 
 # set defaults
 builddir=build
+prefix=/usr/local/bro
 CMakeCacheEntries=""
-append_cache_entry CMAKE_INSTALL_PREFIX PATH   /usr/local/bro
+append_cache_entry CMAKE_INSTALL_PREFIX PATH   $prefix
-append_cache_entry BRO_ROOT_DIR         PATH   /usr/local/bro
+append_cache_entry BRO_ROOT_DIR         PATH   $prefix
-append_cache_entry PY_MOD_INSTALL_DIR   PATH   /usr/local/bro/lib/broctl
+append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/broctl
-append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING /usr/local/bro/share/bro
+append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
+append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
-append_cache_entry BROv6                BOOL   false
 append_cache_entry ENABLE_PERFTOOLS     BOOL   false
+append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
 append_cache_entry INSTALL_AUX_TOOLS    BOOL   true
 append_cache_entry INSTALL_BROCCOLI     BOOL   true
 append_cache_entry INSTALL_BROCTL       BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
+append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
+append_cache_entry DISABLE_PERFTOOLS    BOOL   false
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -120,26 +133,32 @@ while [ $# -ne 0 ]; do
             CMakeGenerator="$optarg"
             ;;
         --prefix=*)
+            prefix=$optarg
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
             append_cache_entry BRO_ROOT_DIR         PATH   $optarg
             append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
-            if [ "$user_set_scriptdir" != "true" ]; then
-                append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg/share/bro
-            fi
             ;;
         --scriptdir=*)
             append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
             user_set_scriptdir="true"
             ;;
+        --conf-files-dir=*)
+            append_cache_entry BRO_ETC_INSTALL_DIR PATH $optarg
+            user_set_conffilesdir="true"
+            ;;
         --enable-debug)
             append_cache_entry ENABLE_DEBUG         BOOL   true
             ;;
-        --enable-brov6)
+        --enable-mobile-ipv6)
-            append_cache_entry BROv6                BOOL   true
+            append_cache_entry ENABLE_MOBILE_IPV6         BOOL   true
             ;;
         --enable-perftools)
             append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             ;;
+        --enable-perftools-debug)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
+            append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
+            ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
             ;;
@@ -149,12 +168,21 @@ while [ $# -ne 0 ]; do
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS    BOOL   false
             ;;
+        --disable-perftools)
+            append_cache_entry DISABLE_PERFTOOLS    BOOL   true
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;
         --disable-ruby)
             append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   true
             ;;
+        --disable-dataseries)
+            append_cache_entry DISABLE_DATASERIES     BOOL   true
+            ;;
+        --disable-elasticsearch)
+            append_cache_entry DISABLE_ELASTICSEARCH     BOOL   true
+            ;;
         --with-openssl=*)
             append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
             ;;
@@ -183,7 +211,6 @@ while [ $# -ne 0 ]; do
             append_cache_entry LibGeoIP_ROOT_DIR PATH $optarg
             ;;
         --with-perftools=*)
-            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry GooglePerftools_ROOT_DIR PATH $optarg
             ;;
         --with-python=*)
@@ -209,6 +236,16 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-dataseries=*)
+            append_cache_entry DataSeries_ROOT_DIR PATH $optarg
+            append_cache_entry Lintel_ROOT_DIR PATH $optarg
+            ;;
+        --with-xml2=*)
+            append_cache_entry LibXML2_ROOT_DIR PATH $optarg
+            ;;
+        --with-curl=*)
+            append_cache_entry LibCURL_ROOT_DIR PATH $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;
@@ -232,6 +269,14 @@ while [ $# -ne 0 ]; do
     shift
 done
 
+if [ "$user_set_scriptdir" != "true" ]; then
+    append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
+fi
+
+if [ "$user_set_conffilesdir" != "true" ]; then
+    append_cache_entry BRO_ETC_INSTALL_DIR PATH $prefix/etc
+fi
+
 if [ -d $builddir ]; then
     # If build directory exists, check if it has a CMake cache
     if [ -f $builddir/CMakeCache.txt ]; then

doc/.gitignore

@@ -1 +1,2 @@
 html
+*.pyc

doc/README

@@ -2,7 +2,7 @@
 Documentation
 =============
 
-This directory contains Bro documentation in reStructured text format
+This directory contains Bro documentation in reStructuredText format
 (see http://docutils.sourceforge.net/rst.html).
 
 It is the root of a Sphinx source tree and can be modified to add more

doc/_static/basic.css

@@ -0,0 +1,513 @@
+/*
+ * basic.css
+ * ~~~~~~~~~
+ *
+ * Sphinx stylesheet -- basic theme.
+ *
+ * :copyright: Copyright 2007-2011 by the Sphinx team, see AUTHORS.
+ * :license: BSD, see LICENSE for details.
+ *
+ */
+
+/* -- main layout ----------------------------------------------------------- */
+
+div.clearer {
+    clear: both;
+}
+
+/* -- relbar ---------------------------------------------------------------- */
+
+div.related {
+    width: 100%;
+    font-size: 90%;
+}
+
+div.related h3 {
+    display: none;
+}
+
+div.related ul {
+    margin: 0;
+    padding: 0 0 0 10px;
+    list-style: none;
+}
+
+div.related li {
+    display: inline;
+}
+
+div.related li.right {
+    float: right;
+    margin-right: 5px;
+}
+
+/* -- sidebar --------------------------------------------------------------- */
+
+div.sphinxsidebarwrapper {
+    padding: 10px 5px 0 10px;
+}
+
+div.sphinxsidebar {
+    float: left;
+    width: 230px;
+    margin-left: -100%;
+    font-size: 90%;
+}
+
+div.sphinxsidebar ul {
+    list-style: none;
+}
+
+div.sphinxsidebar ul ul,
+div.sphinxsidebar ul.want-points {
+    margin-left: 20px;
+    list-style: square;
+}
+
+div.sphinxsidebar ul ul {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+div.sphinxsidebar form {
+    margin-top: 10px;
+}
+
+div.sphinxsidebar input {
+    border: 1px solid #98dbcc;
+    font-family: sans-serif;
+    font-size: 1em;
+}
+
+div.sphinxsidebar input[type="text"] {
+    width: 170px;
+}
+
+div.sphinxsidebar input[type="submit"] {
+    width: 30px;
+}
+
+img {
+    border: 0;
+}
+
+/* -- search page ----------------------------------------------------------- */
+
+ul.search {
+    margin: 10px 0 0 20px;
+    padding: 0;
+}
+
+ul.search li {
+    padding: 5px 0 5px 20px;
+    background-image: url(file.png);
+    background-repeat: no-repeat;
+    background-position: 0 7px;
+}
+
+ul.search li a {
+    font-weight: bold;
+}
+
+ul.search li div.context {
+    color: #888;
+    margin: 2px 0 0 30px;
+    text-align: left;
+}
+
+ul.keywordmatches li.goodmatch a {
+    font-weight: bold;
+}
+
+/* -- index page ------------------------------------------------------------ */
+
+table.contentstable {
+    width: 90%;
+}
+
+table.contentstable p.biglink {
+    line-height: 150%;
+}
+
+a.biglink {
+    font-size: 1.3em;
+}
+
+span.linkdescr {
+    font-style: italic;
+    padding-top: 5px;
+    font-size: 90%;
+}
+
+/* -- general index --------------------------------------------------------- */
+
+table.indextable {
+    width: 100%;
+}
+
+table.indextable td {
+    text-align: left;
+    vertical-align: top;
+}
+
+table.indextable dl, table.indextable dd {
+    margin-top: 0;
+    margin-bottom: 0;
+}
+
+table.indextable tr.pcap {
+    height: 10px;
+}
+
+table.indextable tr.cap {
+    margin-top: 10px;
+    background-color: #f2f2f2;
+}
+
+img.toggler {
+    margin-right: 3px;
+    margin-top: 3px;
+    cursor: pointer;
+}
+
+div.modindex-jumpbox {
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 1em 0 1em 0;
+    padding: 0.4em;
+}
+
+div.genindex-jumpbox {
+    border-top: 1px solid #ddd;
+    border-bottom: 1px solid #ddd;
+    margin: 1em 0 1em 0;
+    padding: 0.4em;
+}
+
+/* -- general body styles --------------------------------------------------- */
+
+a.headerlink {
+    visibility: hidden;
+}
+
+div.body p.caption {
+    text-align: inherit;
+}
+
+div.body td {
+    text-align: left;
+}
+
+.field-list ul {
+    padding-left: 1em;
+}
+
+.first {
+    margin-top: 0 !important;
+}
+
+p.rubric {
+    margin-top: 30px;
+    font-weight: bold;
+}
+
+img.align-left, .figure.align-left, object.align-left {
+    clear: left;
+    float: left;
+    margin-right: 1em;
+}
+
+img.align-right, .figure.align-right, object.align-right {
+    clear: right;
+    float: right;
+    margin-left: 1em;
+}
+
+img.align-center, .figure.align-center, object.align-center {
+  display: block;
+  margin-left: auto;
+  margin-right: auto;
+}
+
+.align-left {
+    text-align: left;
+}
+
+.align-center {
+    text-align: center;
+}
+
+.align-right {
+    text-align: right;
+}
+
+/* -- sidebars -------------------------------------------------------------- */
+
+div.sidebar {
+    margin: 0 0 0.5em 1em;
+    border: 1px solid #ddb;
+    padding: 7px 7px 0 7px;
+    background-color: #ffe;
+    width: 40%;
+    float: right;
+}
+
+p.sidebar-title {
+    font-weight: bold;
+}
+
+/* -- topics ---------------------------------------------------------------- */
+
+div.topic {
+    border: 1px solid #ccc;
+    padding: 7px 7px 0 7px;
+    margin: 10px 0 10px 0;
+}
+
+p.topic-title {
+    font-size: 1.1em;
+    font-weight: bold;
+    margin-top: 10px;
+}
+
+/* -- admonitions ----------------------------------------------------------- */
+
+div.admonition {
+    margin-top: 10px;
+    margin-bottom: 10px;
+    padding: 7px;
+}
+
+div.admonition dt {
+    font-weight: bold;
+}
+
+div.admonition dl {
+    margin-bottom: 0;
+}
+
+p.admonition-title {
+    margin: 0px 10px 5px 0px;
+    font-weight: bold;
+}
+
+div.body p.centered {
+    text-align: center;
+    margin-top: 25px;
+}
+
+/* -- tables ---------------------------------------------------------------- */
+
+table.field-list td, table.field-list th {
+    border: 0 !important;
+}
+
+table.footnote td, table.footnote th {
+    border: 0 !important;
+}
+
+th {
+    text-align: left;
+    padding-right: 5px;
+}
+
+table.citation {
+    border-left: solid 1px gray;
+    margin-left: 1px;
+}
+
+table.citation td {
+    border-bottom: none;
+}
+
+/* -- other body styles ----------------------------------------------------- */
+
+ol.arabic {
+    list-style: decimal;
+}
+
+ol.loweralpha {
+    list-style: lower-alpha;
+}
+
+ol.upperalpha {
+    list-style: upper-alpha;
+}
+
+ol.lowerroman {
+    list-style: lower-roman;
+}
+
+ol.upperroman {
+    list-style: upper-roman;
+}
+
+dd p {
+    margin-top: 0px;
+}
+
+dd ul, dd table {
+    margin-bottom: 10px;
+}
+
+dd {
+    margin-top: 3px;
+    margin-bottom: 10px;
+    margin-left: 30px;
+}
+
+dt:target, .highlighted {
+    background-color: #fbe54e;
+}
+
+dl.glossary dt {
+    font-weight: bold;
+    font-size: 1.1em;
+}
+
+.field-list ul {
+    margin: 0;
+    padding-left: 1em;
+}
+
+.field-list p {
+    margin: 0;
+}
+
+.refcount {
+    color: #060;
+}
+
+.optional {
+    font-size: 1.3em;
+}
+
+.versionmodified {
+    font-style: italic;
+}
+
+.system-message {
+    background-color: #fda;
+    padding: 5px;
+    border: 3px solid red;
+}
+
+.footnote:target  {
+    background-color: #ffa;
+}
+
+.line-block {
+    display: block;
+    margin-top: 1em;
+    margin-bottom: 1em;
+}
+
+.line-block .line-block {
+    margin-top: 0;
+    margin-bottom: 0;
+    margin-left: 1.5em;
+}
+
+.guilabel, .menuselection {
+    font-family: sans-serif;
+}
+
+.accelerator {
+    text-decoration: underline;
+}
+
+.classifier {
+    font-style: oblique;
+}
+
+abbr, acronym {
+    border-bottom: dotted 1px;
+    cursor: help;
+}
+
+/* -- code displays --------------------------------------------------------- */
+
+pre {
+    overflow: auto;
+    overflow-y: hidden;  /* fixes display issues on Chrome browsers */
+}
+
+td.linenos pre {
+    padding: 5px 0px;
+    border: 0;
+    background-color: transparent;
+    color: #aaa;
+}
+
+table.highlighttable {
+    margin-left: 0.5em;
+}
+
+table.highlighttable td {
+    padding: 0 0.5em 0 0.5em;
+}
+
+tt.descname {
+    background-color: transparent;
+    font-weight: bold;
+#   font-size: 1.2em;
+}
+
+tt.descclassname {
+    background-color: transparent;
+}
+
+tt.xref, a tt {
+    background-color: transparent;
+#    font-weight: bold;
+}
+
+h1 tt, h2 tt, h3 tt, h4 tt, h5 tt, h6 tt {
+    background-color: transparent;
+}
+
+.viewcode-link {
+    float: right;
+}
+
+.viewcode-back {
+    float: right;
+    font-family: sans-serif;
+}
+
+div.viewcode-block:target {
+    margin: -1px -10px;
+    padding: 0 10px;
+}
+
+/* -- math display ---------------------------------------------------------- */
+
+img.math {
+    vertical-align: middle;
+}
+
+div.body div.math p {
+    text-align: center;
+}
+
+span.eqno {
+    float: right;
+}
+
+/* -- printout stylesheet --------------------------------------------------- */
+
+@media print {
+    div.document,
+    div.documentwrapper,
+    div.bodywrapper {
+        margin: 0 !important;
+        width: 100%;
+    }
+
+    div.sphinxsidebar,
+    div.related,
+    div.footer,
+    #top-link {
+        display: none;
+    }
+}

doc/_static/broxygen-extra.css

@@ -1,3 +1,17 @@
+
+a.toc-backref {
+    color: #333;
+}
+
+h1, h2, h3, h4, h5, h6,
+h1 a, h2 a, h3 a, h4 a, h5 a, h6 a {
+	padding:0 0 0px 0;
+}
+
+ul {
+    padding-bottom: 0px;
+}    
+
 h1 {
     font-weight: bold;
     font-size: 32px;
@@ -14,3 +28,133 @@ th.field-name
 {
     white-space:nowrap;
 }
+
+h2 {
+    margin-top: 50px;
+    padding-bottom: 5px;
+    margin-bottom: 30px;
+    border-bottom: 1px solid;
+    border-color: #aaa;
+    font-style: normal;
+}
+
+div.section h3 {
+    font-style: normal;
+    }
+
+h3 {
+    font-size: 20px;
+    margin-top: 40px;
+    margin-bottom: 0¡px;
+    font-weight: bold;
+    font-style: normal;
+}
+
+h3.widgettitle {
+    font-style: normal;
+}    
+
+h4 {
+    font-size:18px;
+    font-style: normal;
+    margin-bottom: 0em;
+    margin-top: 40px;
+    font-style: italic;
+}
+
+h5 {
+    font-size:16px;
+}
+
+h6 {
+    font-size:15px;
+}
+
+.toc-backref {
+    color: #333;
+}
+
+.contents ul {
+   padding-bottom: 1em;
+}
+
+dl.namespace {
+    display: none;
+}
+
+dl dt {
+    font-weight: normal;
+}
+
+table.docutils tbody {
+    margin: 1em 1em 1em 1em;
+}
+
+table.docutils td {
+    padding: 5pt 5pt 5pt 5pt;
+    font-size: 14px;
+    border-left: 0;
+    border-right: 0;
+}
+
+dl pre {
+    font-size: 14px;
+}
+
+table.docutils th {
+    padding: 5pt 5pt 5pt 5pt;
+    font-size: 14px;
+    font-style: normal;
+    border-left: 0;
+    border-right: 0;
+}
+
+table.docutils tr:first-child td {
+    #border-top: 1px solid #aaa;
+}
+
+.download {
+    font-family:"Courier New", Courier, mono;
+    font-weight: normal;
+}
+
+dt:target, .highlighted {
+    background-color: #ccc;
+}
+
+p {
+    padding-bottom: 0px;
+}
+
+p.last {
+    margin-bottom: 0px;
+}
+
+dl {
+    padding: 1em 1em 1em 1em;
+    background: #fffff0;
+    border: 1px solid #aaa;
+
+}
+
+dl {
+   margin-bottom: 10px;
+}
+
+
+table.docutils {
+    background: #fffff0;
+    border-collapse: collapse;
+    border: 1px solid #ddd;
+}
+
+dl table.docutils {
+    border: 0;
+}
+
+table.docutils dl {
+    border: 1px dashed #666;
+}
+
+
+

doc/_static/broxygen.css

@@ -0,0 +1,437 @@
+/* Automatically generated. Do not edit. */
+
+
+
+
+
+#bro-main, #bro-standalone-main {
+	padding: 0 0 0 0;
+	position:relative;
+	z-index:1;
+}
+
+#bro-main {
+    margin-bottom: 2em;
+    }
+
+#bro-standalone-main {
+    margin-bottom: 0em;
+    padding-left: 50px;
+    padding-right: 50px;
+    }
+
+#bro-outer {
+    color: #333;
+    background: #ffffff;
+}
+
+#bro-title {
+    font-weight: bold;
+    font-size: 32px;
+	line-height:32px;
+    text-align: center;
+    padding-top: 3px;
+    margin-bottom: 30px;
+    font-family: Palatino,'Palatino Linotype',Georgia,serif;;
+    color: #000;
+    }
+
+.opening:first-letter {
+    font-size: 24px;
+    font-weight: bold;
+    letter-spacing: 0.05em;
+    }
+
+.opening {
+    font-size: 17px;
+}
+
+.version {
+    text-align: right;
+    font-size: 12px;
+    color: #aaa;
+   	line-height: 0;
+	height: 0;
+}
+
+.git-info-version {
+    position: relative;
+    height: 2em;
+    top: -1em;
+    color: #ccc;
+    float: left;
+    font-size: 12px;
+}
+
+.git-info-date {
+    position: relative;
+    height: 2em;
+    top: -1em;
+    color: #ccc;
+    float: right;
+    font-size: 12px;
+}
+
+body {
+	font-family:Arial, Helvetica, sans-serif;
+	font-size:15px;
+	line-height:22px;
+    color: #333;
+    margin: 0px;
+}
+
+h1, h2, h3, h4, h5, h6,
+h1 a, h2 a, h3 a, h4 a, h5 a, h6 a {
+	padding:0 0 20px 0;
+	font-weight:bold;
+	text-decoration:none;
+}
+
+div.section h3, div.section h4, div.section h5, div.section h6 {
+    font-style: italic;
+}
+
+h1, h2 {
+	font-size:27px;
+	letter-spacing:-1px;
+}
+
+h3 {
+    margin-top: 1em;
+	font-size:18px;
+}
+
+h4 {
+	font-size:16px;
+}
+
+h5 {
+	font-size:15px;
+}
+
+h6 {
+	font-size:12px;
+}
+
+p {
+	padding:0 0 20px 0;
+}
+
+hr {
+	background:none;
+	height:1px;
+	line-height:1px;
+	border:0;
+	margin:0 0 20px 0;
+}
+
+ul, ol {
+	margin:0 20px 20px 0;
+	padding-left:40px;
+}
+
+ul.simple, ol.simple {
+    margin:0 0px 0px 0;
+}
+
+blockquote {
+	margin:0 0 0 40px;
+}
+
+strong, dfn {
+	font-weight:bold;
+}
+
+em, dfn {
+	font-style:italic;
+}
+
+sup, sub {
+	line-height:0;
+}
+
+pre  {
+	white-space:pre;
+}
+
+pre, code, tt {
+	font-family:"Courier New", Courier, mono;
+}
+
+dl {
+	margin: 0 0 20px 0;
+}
+
+dl dt {
+	font-weight: bold;
+}
+
+dd {
+	margin:0 0 20px 20px;
+}
+
+small {
+	font-size:75%;
+}
+
+a:link,
+a:visited,
+a:active
+{
+    color: #2a85a7;
+}
+
+a:hover
+{
+	color:#c24444;
+}
+
+h1, h2, h3, h4, h5, h6,
+h1 a, h2 a, h3 a, h4 a, h5 a, h6 a
+{
+	color: #333;
+}
+
+hr {
+	border-bottom:1px solid #ddd;
+}
+
+pre {
+    color: #333;
+	background: #FFFAE2;
+	padding: 7px 5px 3px 5px;
+    margin-bottom: 25px;
+    margin-top: 0px;
+}
+
+ul {
+    padding-bottom: 5px;
+    }
+
+h1, h2 {
+	margin-top: 30px;
+    }
+
+h1 {
+	margin-bottom: 50px;
+	margin-bottom: 20px;
+    padding-bottom: 5px;
+    border-bottom: 1px solid;
+    border-color: #aaa;
+    }
+
+h2 {
+	font-size: 24px;
+    }
+
+pre {
+	-moz-box-shadow:0 0 6px #ddd;
+	-webkit-box-shadow:0 0 6px #ddd;
+	box-shadow:0 0 6px #ddd;
+}
+
+a {
+	text-decoration:none;
+    }
+
+p {
+    padding-bottom: 15px;
+    }
+
+p, dd, li {
+    text-align: justify;
+    }
+
+li {
+    margin-bottom: 5px;
+    }
+
+
+
+#footer .widget_links ul a,
+#footer .widget_links ol a
+{
+	color: #ddd;
+}
+
+#footer .widget_links ul a:hover,
+#footer .widget_links ol a:hover
+{
+	color:#c24444;
+}
+
+
+#footer .widget li {
+	padding-bottom:10px;
+}
+
+#footer .widget_links li {
+	padding-bottom:1px;
+}
+
+#footer .widget li:last-child {
+	padding-bottom:0;
+}
+
+#footer .widgettitle {
+	color: #ddd;
+}
+
+
+.widget {
+	margin:0 0 40px 0;
+}
+
+.widget, .widgettitle {
+	font-size:12px;
+	line-height:18px;
+}
+
+.widgettitle {
+	font-weight:bold;
+	text-transform:uppercase;
+	padding:0 0 10px 0;
+	margin:0 0 20px 0;
+	line-height:100%;
+}
+
+.widget UL, .widget OL {
+	list-style-type:none;
+	margin:0;
+	padding:0;
+}
+
+.widget p {
+	padding:0;
+}
+
+.widget li {
+	padding-bottom:10px;
+}
+
+.widget a {
+	text-decoration:none;
+}
+
+#bro-main .widgettitle,
+{
+	color: #333;
+}
+
+
+.widget img.left {
+	padding:5px 10px 10px 0;
+}
+
+.widget img.right {
+	padding:5px 0 10px 10px;
+}
+
+.ads .widgettitle {
+	margin-right:16px;
+}
+
+.widget {
+   margin-left: 1em;
+}
+
+.widgettitle {
+	color: #333;
+}
+
+.widgettitle {
+	border-bottom:1px solid #ddd;
+}
+
+
+.sidebar-toc ul li {
+    padding-bottom: 0px;
+    text-align: left;
+    list-style-type: square;
+    list-style-position: inside;
+    padding-left: 1em;
+    text-indent: -1em;
+    }
+
+.sidebar-toc ul li li {
+    margin-left: 1em;
+    margin-bottom: 0px;
+    list-style-type: square;
+    }
+
+.sidebar-toc ul li li a {
+    font-size: 8pt;
+}
+
+.contents {
+   padding: 10px;
+   background: #FFFAE2;
+   margin: 20px;
+   }
+
+.topic-title {
+   font-size: 20px;
+   font-weight: bold;
+   padding: 0px 0px 5px 0px;
+   text-align: center;
+   padding-top: .5em;
+}
+
+.contents li {
+   margin-bottom: 0px;
+   list-style-type: square;
+}
+
+.contents ul ul li {
+   margin-left: 0px;
+   padding-left: 0px;
+   padding-top: 0em;
+   font-size: 90%;
+   list-style-type: square;
+   font-weight: normal;
+}
+
+.contents ul ul ul li {
+   list-style-type: none;
+}
+
+.contents ul ul ul ul li {
+   display:none;
+}
+
+.contents ul li {
+   padding-top: 1em;
+   list-style-type: none;
+    font-weight: bold;
+}
+
+.contents ul {
+   margin-left: 0px;
+   padding-left: 2em;
+   margin: 0px 0px 0px 0px;
+}
+
+.note, .warning, .error {
+    margin-left: 2em;
+    margin-right: 2em;
+    margin-top: 1.5em;
+    margin-bottom: 1.5em;
+    padding: 0.5em 1em 0.5em 1em;
+    overflow: auto;
+    border-left: solid 3px #aaa;
+    font-size: 15px;
+    color: #333;
+}
+
+.admonition p {
+    margin-left: 1em;
+    }
+
+.admonition-title {
+    font-size: 16px;
+	font-weight: bold;
+    color: #000;
+    padding-bottom: 0em;
+    margin-bottom: .5em;
+    margin-top: 0em;
+}

doc/_static/download.js

@@ -1,3 +0,0 @@
-$(document).ready(function() {
-    $('.docutils.download').removeClass('download');
-});

doc/_static/pygments.css

@@ -0,0 +1,58 @@
+.hll { background-color: #ffffcc }
+.c { color: #aaaaaa; font-style: italic } /* Comment */
+.err { color: #F00000; background-color: #F0A0A0 } /* Error */
+.k { color: #0000aa } /* Keyword */
+.cm { color: #aaaaaa; font-style: italic } /* Comment.Multiline */
+.cp { color: #4c8317 } /* Comment.Preproc */
+.c1 { color: #aaaaaa; font-style: italic } /* Comment.Single */
+.cs { color: #0000aa; font-style: italic } /* Comment.Special */
+.gd { color: #aa0000 } /* Generic.Deleted */
+.ge { font-style: italic } /* Generic.Emph */
+.gr { color: #aa0000 } /* Generic.Error */
+.gh { color: #000080; font-weight: bold } /* Generic.Heading */
+.gi { color: #00aa00 } /* Generic.Inserted */
+.go { color: #888888 } /* Generic.Output */
+.gp { color: #555555 } /* Generic.Prompt */
+.gs { font-weight: bold } /* Generic.Strong */
+.gu { color: #800080; font-weight: bold } /* Generic.Subheading */
+.gt { color: #aa0000 } /* Generic.Traceback */
+.kc { color: #0000aa } /* Keyword.Constant */
+.kd { color: #0000aa } /* Keyword.Declaration */
+.kn { color: #0000aa } /* Keyword.Namespace */
+.kp { color: #0000aa } /* Keyword.Pseudo */
+.kr { color: #0000aa } /* Keyword.Reserved */
+.kt { color: #00aaaa } /* Keyword.Type */
+.m { color: #009999 } /* Literal.Number */
+.s { color: #aa5500 } /* Literal.String */
+.na { color: #1e90ff } /* Name.Attribute */
+.nb { color: #00aaaa } /* Name.Builtin */
+.nc { color: #00aa00; text-decoration: underline } /* Name.Class */
+.no { color: #aa0000 } /* Name.Constant */
+.nd { color: #888888 } /* Name.Decorator */
+.ni { color: #800000; font-weight: bold } /* Name.Entity */
+.nf { color: #00aa00 } /* Name.Function */
+.nn { color: #00aaaa; text-decoration: underline } /* Name.Namespace */
+.nt { color: #1e90ff; font-weight: bold } /* Name.Tag */
+.nv { color: #aa0000 } /* Name.Variable */
+.ow { color: #0000aa } /* Operator.Word */
+.w { color: #bbbbbb } /* Text.Whitespace */
+.mf { color: #009999 } /* Literal.Number.Float */
+.mh { color: #009999 } /* Literal.Number.Hex */
+.mi { color: #009999 } /* Literal.Number.Integer */
+.mo { color: #009999 } /* Literal.Number.Oct */
+.sb { color: #aa5500 } /* Literal.String.Backtick */
+.sc { color: #aa5500 } /* Literal.String.Char */
+.sd { color: #aa5500 } /* Literal.String.Doc */
+.s2 { color: #aa5500 } /* Literal.String.Double */
+.se { color: #aa5500 } /* Literal.String.Escape */
+.sh { color: #aa5500 } /* Literal.String.Heredoc */
+.si { color: #aa5500 } /* Literal.String.Interpol */
+.sx { color: #aa5500 } /* Literal.String.Other */
+.sr { color: #009999 } /* Literal.String.Regex */
+.s1 { color: #aa5500 } /* Literal.String.Single */
+.ss { color: #0000aa } /* Literal.String.Symbol */
+.bp { color: #00aaaa } /* Name.Builtin.Pseudo */
+.vc { color: #aa0000 } /* Name.Variable.Class */
+.vg { color: #aa0000 } /* Name.Variable.Global */
+.vi { color: #aa0000 } /* Name.Variable.Instance */
+.il { color: #009999 } /* Literal.Number.Integer.Long */

doc/_templates/layout.html

@@ -1,15 +1,16 @@
 {% extends "!layout.html" %}
 
 {% block extrahead %}
-<link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/bro-ids.css" />
+<link rel="stylesheet" type="text/css" href="{{ pathto('_static/broxygen.css', 1) }}"></script>
-<link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/960.css" />
+<link rel="stylesheet" type="text/css" href="{{ pathto('_static/960.css', 1) }}"></script>
-<link rel="stylesheet" type="text/css" href="http://www.bro-ids.org/css/pygments.css" />
+<link rel="stylesheet" type="text/css" href="{{ pathto('_static/pygments.css', 1) }}"></script>
 <link rel="stylesheet" type="text/css" href="{{ pathto('_static/broxygen-extra.css', 1) }}"></script>
-<script type="text/javascript" src="{{ pathto('_static/download.js', 1) }}"></script>
+
+<script type="text/javascript" src="{{ pathto('_static/broxygen-extra.js', 1) }}"></script>
 {% endblock %}
 
 {% block header %}
-<iframe src="http://www.bro-ids.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+<iframe src="http://www.bro.org/frames/header-no-logo.html" width="100%" height="100px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
 </iframe>
 {% endblock %}
 
@@ -47,6 +48,7 @@
                             Table of Contents
                         </h3>
                         <p>
+                            <!-- <ul id="sidebar-toc"></ul> -->
                             <ul>{{toc}}</ul>
                         </p>
                     </div>
@@ -106,6 +108,6 @@
 {% endblock %}
 
 {% block footer %}
-<iframe src="http://www.bro-ids.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
+<iframe src="http://www.bro.org/frames/footer.html" width="100%" height="420px" frameborder="0" marginheight="0" scrolling="no" marginwidth="0">
 </iframe>
 {% endblock %}

doc/cluster.rst

@@ -45,7 +45,7 @@ This is the Bro process that sniffs network traffic and does protocol analysis o
 
 The rule of thumb we have followed recently is to allocate approximately 1 core for every 80Mbps of traffic that is being analyzed, however this estimate could be extremely traffic mix specific.  It has generally worked for mixed traffic with many users and servers.  For example, if your traffic peaks around 2Gbps (combined) and you want to handle traffic at peak load, you may want to have 26 cores available (2048 / 80 == 25.6).  If the 80Mbps estimate works for your traffic, this could be handled by 3 physical hosts dedicated to being workers with each one containing dual 6-core processors.  
 
-Once a flow based load balancer is put into place this model is extremely easy to scale as well so it’s recommended that you guess at the amount of hardware you will need to fully analyze your traffic.  If it turns out that you need more, it’s relatively easy to easy increase the size of the cluster in most cases.
+Once a flow based load balancer is put into place this model is extremely easy to scale as well so it’s recommended that you guess at the amount of hardware you will need to fully analyze your traffic.  If it turns out that you need more, it’s relatively easy to increase the size of the cluster in most cases.
 
 Frontend Options
 ----------------
@@ -58,7 +58,7 @@ Discrete hardware flow balancers
 cPacket
 ^^^^^^^
 
-If you are monitoring one or more 10G physical interfaces, the recommended solution is to use either a cFlow or cVu device from cPacket because they are currently being used very successfully at a number of sites.  These devices will perform layer-2 load balancing by rewriting the destination ethernet MAC address to cause each packet associated with a particular flow to have the same destination MAC.  The packets can then be passed directly to a monitoring host where each worker has a BPF filter to limit it's visibility to only that stream of flows or onward to a commodity switch to split the traffic out to multiple 1G interfaces for the workers.  This can ultimately greatly reduce costs since workers can use relatively inexpensive 1G interfaces.
+If you are monitoring one or more 10G physical interfaces, the recommended solution is to use either a cFlow or cVu device from cPacket because they are currently being used very successfully at a number of sites.  These devices will perform layer-2 load balancing by rewriting the destination ethernet MAC address to cause each packet associated with a particular flow to have the same destination MAC.  The packets can then be passed directly to a monitoring host where each worker has a BPF filter to limit its visibility to only that stream of flows or onward to a commodity switch to split the traffic out to multiple 1G interfaces for the workers.  This can ultimately greatly reduce costs since workers can use relatively inexpensive 1G interfaces.
 
 OpenFlow Switches
 ^^^^^^^^^^^^^^^^^
@@ -76,7 +76,7 @@ The PF_RING software for Linux has a “clustering” feature which will do flow
 Netmap
 ^^^^^^
 
-FreeBSD has an in-progress project named Netmap which will enabled flow based load balancing as well.  When it becomes viable for real world use, this document will be updated.
+FreeBSD has an in-progress project named Netmap which will enable flow based load balancing as well.  When it becomes viable for real world use, this document will be updated.
 
 Click! Software Router
 ^^^^^^^^^^^^^^^^^^^^^^

doc/conf.py.in

@@ -24,7 +24,7 @@ sys.path.insert(0, os.path.abspath('sphinx-sources/ext'))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = ['bro', 'rst_directive']
+extensions = ['bro', 'rst_directive', 'sphinx.ext.todo', 'adapt-toc']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['sphinx-sources/_templates', 'sphinx-sources/_static']
@@ -40,7 +40,7 @@ master_doc = 'index'
 
 # General information about the project.
 project = u'Bro'
-copyright = u'2011, The Bro Project'
+copyright = u'2012, The Bro Project'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -169,6 +169,7 @@ html_sidebars = {
 # Output file base name for HTML help builder.
 htmlhelp_basename = 'Broxygen'
 
+html_add_permalinks = None
 
 # -- Options for LaTeX output --------------------------------------------------
 
@@ -208,7 +209,6 @@ latex_documents = [
 # If false, no module index is generated.
 #latex_domain_indices = True
 
-
 # -- Options for manual page output --------------------------------------------
 
 # One entry per manual page. List of tuples
@@ -217,3 +217,6 @@ man_pages = [
     ('index', 'bro', u'Bro Documentation',
      [u'The Bro Project'], 1)
 ]
+
+# -- Options for todo plugin --------------------------------------------
+todo_include_todos=True

doc/ext/adapt-toc.py

@@ -0,0 +1,29 @@
+
+import sys
+import re
+
+# Removes the first TOC level, which is just the page title.
+def process_html_toc(app, pagename, templatename, context, doctree):
+
+    if not "toc" in context:
+        return
+
+    toc = context["toc"]
+
+    lines = toc.strip().split("\n")
+    lines = lines[2:-2]
+
+    toc = "\n".join(lines)
+    toc = "<ul>" + toc
+
+    context["toc"] = toc
+
+    # print >>sys.stderr, pagename
+    # print >>sys.stderr, context["toc"]
+    # print >>sys.stderr, "-----"
+    # print >>sys.stderr, toc
+    # print >>sys.stderr, "===="
+
+def setup(app):
+    app.connect('html-page-context', process_html_toc)
+

doc/ext/bro.py

@@ -82,7 +82,8 @@ class BroGeneric(ObjectDescription):
 
             objects = self.env.domaindata['bro']['objects']
             key = (self.objtype, name)
-            if key in objects:
+            if ( key in objects and self.objtype != "id" and
+                 self.objtype != "type" ):
                 self.env.warn(self.env.docname,
                               'duplicate description of %s %s, ' %
                               (self.objtype, name) +
@@ -150,6 +151,12 @@ class BroEnum(BroGeneric):
         #self.indexnode['entries'].append(('single', indextext,
         #                                  targetname, targetname))
         m = sig.split()
+
+        if len(m) < 2:
+            self.env.warn(self.env.docname,
+                          "bro:enum directive missing argument(s)")
+            return
+
         if m[1] == "Notice::Type":
             if 'notices' not in self.env.domaindata['bro']:
                 self.env.domaindata['bro']['notices'] = []

doc/ext/bro_lexer/bro.py

@@ -29,7 +29,7 @@ class BroLexer(RegexLexer):
              r'|vector)\b', Keyword.Type),
             (r'(T|F)\b', Keyword.Constant),
             (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
-             r'|default|disable_print_hook|raw_output|encrypt|group|log'
+             r'|default|raw_output|encrypt|group|log'
              r'|mergeable|optional|persistent|priority|redef'
              r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
                  Keyword)),

doc/faq.rst

@@ -46,12 +46,34 @@ directions:
 http://securityonion.blogspot.com/2011/10/when-is-full-packet-capture-not-full.html
 
 What does an error message like ``internal error: NB-DNS error`` mean?
----------------------------------------------------------------------------------------------------------------------------------
+----------------------------------------------------------------------
 
 That often means that DNS is not set up correctly on the system
 running Bro. Try verifying from the command line that DNS lookups
 work, e.g., ``host www.google.com``.
 
+I am using OpenBSD and having problems installing Bro?
+------------------------------------------------------
+
+One potential issue is that the top-level Makefile may not work with
+OpenBSD's default make program, in which case you can either install
+the ``gmake`` package and use it instead or first change into the
+``build/`` directory before doing either ``make`` or ``make install``
+such that the CMake-generated Makefile's are used directly.
+
+Generally, please note that we do not regularly test OpenBSD builds.
+We appreciate any patches that improve Bro's support for this
+platform.
+
+How do BroControl options affect Bro script variables?
+------------------------------------------------------
+
+Some (but not all) BroControl options override a corresponding Bro script variable.
+For example, setting the BroControl option "LogRotationInterval" will override
+the value of the Bro script variable "Log::default_rotation_interval".
+See the :doc:`BroControl Documentation <components/broctl/README>` to find out
+which BroControl options override Bro script variables, and for more discussion
+on site-specific customization.
 
 Usage
 =====
@@ -59,34 +81,30 @@ Usage
 How can I identify backscatter?
 -------------------------------
 
-Identifying backscatter via connections labeled as ``OTH`` is not
+Identifying backscatter via connections labeled as ``OTH`` is not a reliable
-a reliable means to detect backscatter. Use rather the following
+means to detect backscatter. Backscatter is however visible by interpreting
-procedure:
+the contents of the ``history`` field in the ``conn.log`` file. The basic idea
-
+is to watch for connections that never had an initial ``SYN`` but started
-* Enable connection history via ``redef record_state_history=T`` to
+instead with a ``SYN-ACK`` or ``RST`` (though this latter generally is just
-  track all control/data packet types in connection logs.
+discarded). Here are some history fields which provide backscatter examples:
-
+``hAFf``, ``r``. Refer to the conn protocol analysis scripts to interpret the
-* Backscatter is now visible in terms of connections that never had an
+individual character meanings in the history field.
-  initial ``SYN`` but started instead with a ``SYN-ACK`` or ``RST``
-  (though this latter generally is just discarded).
 
 Is there help for understanding Bro's resource consumption?
 -----------------------------------------------------------
 
 There are two scripts that collect statistics on resource usage:
-``stats.bro`` and ``profiling.bro``. The former is quite lightweight,
+``misc/stats.bro`` and ``misc/profiling.bro``. The former is quite
-while the latter should only be used for debugging. Furthermore,
+lightweight, while the latter should only be used for debugging.
-there's also ``print-globals.bro``, which prints the size of all
-global script variable at termination.
 
 How can I capture packets as an unprivileged user?
 --------------------------------------------------
 
-Normally, unprivileged users cannot capture packets from a network
+Normally, unprivileged users cannot capture packets from a network interface,
-interface, which means they would not be able to use Bro to read/analyze
+which means they would not be able to use Bro to read/analyze live traffic.
-live traffic.  However, there are ways to enable packet capture
+However, there are operating system specific ways to enable packet capture
-permission for non-root users, which is worth doing in the context of
+permission for non-root users, which is worth doing in the context of using
-using Bro to monitor live traffic
+Bro to monitor live traffic.
 
 With Linux Capabilities
 ^^^^^^^^^^^^^^^^^^^^^^^
@@ -147,8 +165,8 @@ alteration tools.
 Bro has two options to workaround such situations and ignore bad checksums:
 
 1) The ``-C`` command line option to ``bro``.
-2) An option called ``ignore_checksums`` that can be redefined at the policy
+2) An option called ``ignore_checksums`` that can be redefined at the
-   policy script layer (e.g. in your ``$PREFIX/share/bro/site/local/bro``):
+   policy script layer (e.g. in your ``$PREFIX/share/bro/site/local.bro``):
 
     .. code:: bro
 

doc/file-analysis.rst

@@ -0,0 +1,185 @@
+=============
+File Analysis
+=============
+
+.. rst-class:: opening
+
+    In the past, writing Bro scripts with the intent of analyzing file
+    content could be cumbersome because of the fact that the content
+    would be presented in different ways, via events, at the
+    script-layer depending on which network protocol was involved in the
+    file transfer.  Scripts written to analyze files over one protocol
+    would have to be copied and modified to fit other protocols.  The
+    file analysis framework (FAF) instead provides a generalized
+    presentation of file-related information.  The information regarding
+    the protocol involved in transporting a file over the network is
+    still available, but it no longer has to dictate how one organizes
+    their scripting logic to handle it.  A goal of the FAF is to
+    provide analysis specifically for files that is analogous to the
+    analysis Bro provides for network connections.
+
+.. contents::
+
+File Lifecycle Events
+=====================
+
+The key events that may occur during the lifetime of a file are:
+:bro:see:`file_new`, :bro:see:`file_over_new_connection`,
+:bro:see:`file_timeout`, :bro:see:`file_gap`, and
+:bro:see:`file_state_remove`.  Handling any of these events provides
+some information about the file such as which network
+:bro:see:`connection` and protocol are transporting the file, how many
+bytes have been transferred so far, and its MIME type.
+
+.. code:: bro
+
+    event connection_state_remove(c: connection)
+        {
+        print "connection_state_remove";
+        print c$uid;
+        print c$id;
+        for ( s in c$service )
+            print s;
+        }
+
+    event file_state_remove(f: fa_file)
+        {
+        print "file_state_remove";
+        print f$id;
+        for ( cid in f$conns )
+            {
+            print f$conns[cid]$uid;
+            print cid;
+            }
+        print f$source;
+        }
+
+might give output like::
+
+    file_state_remove
+    Cx92a0ym5R8
+    REs2LQfVW2j
+    [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+    HTTP
+    connection_state_remove
+    REs2LQfVW2j
+    [orig_h=10.0.0.7, orig_p=59856/tcp, resp_h=192.150.187.43, resp_p=80/tcp]
+    HTTP
+
+This doesn't perform any interesting analysis yet, but does highlight
+the similarity between analysis of connections and files.  Connections
+are identified by the usual 5-tuple or a convenient UID string while
+files are identified just by a string of the same format as the
+connection UID.  So there's unique ways to identify both files and
+connections and files hold references to a connection (or connections)
+that transported it.
+
+Adding Analysis
+===============
+
+There are builtin file analyzers which can be attached to files.  Once
+attached, they start receiving the contents of the file as Bro extracts
+it from an ongoing network connection.  What they do with the file
+contents is up to the particular file analyzer implementation, but
+they'll typically either report further information about the file via
+events (e.g. :bro:see:`Files::ANALYZER_MD5` will report the
+file's MD5 checksum via :bro:see:`file_hash` once calculated) or they'll
+have some side effect (e.g. :bro:see:`Files::ANALYZER_EXTRACT`
+will write the contents of the file out to the local file system).
+
+In the future there may be file analyzers that automatically attach to
+files based on heuristics, similar to the Dynamic Protocol Detection
+(DPD) framework for connections, but many will always require an
+explicit attachment decision:
+
+.. code:: bro
+
+    event file_new(f: fa_file)
+        {
+        print "new file", f$id;
+        if ( f?$mime_type && f$mime_type == "text/plain" )
+            Files::add_analyzer(f, Files::ANALYZER_MD5);
+        }
+
+    event file_hash(f: fa_file, kind: string, hash: string)
+        {
+        print "file_hash", f$id, kind, hash;
+        }
+
+this script calculates MD5s for all plain text files and might give
+output::
+
+    new file, Cx92a0ym5R8
+    file_hash, Cx92a0ym5R8, md5, 397168fd09991a0e712254df7bc639ac
+
+Some file analyzers might have tunable parameters that need to be
+specified in the call to :bro:see:`Files::add_analyzer`:
+
+.. code:: bro
+
+    event file_new(f: fa_file)
+        {
+        Files::add_analyzer(f, Files::ANALYZER_EXTRACT,
+                            [$extract_filename="myfile"]);
+        }
+
+In this case, the file extraction analyzer doesn't generate any further
+events, but does have the effect of writing out the file contents to the
+local file system at the location resulting from the concatenation of
+the path specified by :bro:see:`FileExtract::prefix` and the string,
+``myfile``.  Of course, for a network with more than a single file being
+transferred, it's probably preferable to specify a different extraction
+path for each file, unlike this example.
+
+Regardless of which file analyzers end up acting on a file, general
+information about the file (e.g. size, time of last data transferred,
+MIME type, etc.) are logged in ``files.log``.
+
+Input Framework Integration
+===========================
+
+The FAF comes with a simple way to integrate with the :doc:`Input
+Framework <input>`, so that Bro can analyze files from external sources
+in the same way it analyzes files that it sees coming over traffic from
+a network interface it's monitoring.  It only requires a call to
+:bro:see:`Input::add_analysis`:
+
+.. code:: bro
+
+    redef exit_only_after_terminate = T;
+
+    event file_new(f: fa_file)
+        {
+        print "new file", f$id;
+        Files::add_analyzer(f, Files::ANALYZER_MD5);
+        }
+
+    event file_state_remove(f: fa_file)
+        {
+        Input::remove(f$source);
+        terminate();
+        }
+
+    event file_hash(f: fa_file, kind: string, hash: string)
+        {
+        print "file_hash", f$id, kind, hash;
+        }
+
+    event bro_init()
+        {
+        local source: string = "./myfile";
+        Input::add_analysis([$source=source, $name=source]);
+        }
+
+Note that the "source" field of :bro:see:`fa_file` corresponds to the
+"name" field of :bro:see:`Input::AnalysisDescription` since that is what
+the input framework uses to uniquely identify an input stream.
+
+The output of the above script may be::
+
+    new file, G1fS2xthS4l
+    file_hash, G1fS2xthS4l, md5, 54098b367d2e87b078671fad4afb9dbb
+
+Nothing that special, but it at least verifies the MD5 file analyzer
+saw all the bytes of the input file and calculated the checksum
+correctly!

doc/index.rst

@@ -11,8 +11,8 @@ Guides
    :maxdepth: 1
 
    INSTALL
-   quickstart
    upgrade
+   quickstart
    faq
    reporting-problems
 
@@ -24,6 +24,8 @@ Frameworks
 
    notice
    logging
+   input
+   file-analysis
    cluster
    signatures
 
@@ -41,10 +43,11 @@ Script Reference
 .. toctree::
    :maxdepth: 1
 
-   scripts/builtins
-   scripts/bifs
    scripts/packages
    scripts/index
+   scripts/builtins
+   scripts/proto-analyzers
+   scripts/file-analyzers
 
 Other Bro Components
 --------------------
@@ -52,7 +55,7 @@ Other Bro Components
 The following are snapshots of documentation for components that come
 with this version of Bro (|version|). Since they can also be used
 independently, see the `download page
-<http://bro-ids.org/download/index.html>`_ for documentation of any
+<http://bro.org/download/index.html>`_ for documentation of any
 current, independent component releases.
 
 .. toctree::

doc/input.rst

@@ -0,0 +1,407 @@
+==============================================
+Loading Data into Bro with the Input Framework
+==============================================
+
+.. rst-class:: opening
+
+   Bro now features a flexible input framework that allows users
+   to import data into Bro. Data is either read into Bro tables or
+   converted to events which can then be handled by scripts.
+   This document gives an overview of how to use the input framework
+   with some examples. For more complex scenarios it is
+   worthwhile to take a look at the unit tests in
+   ``testing/btest/scripts/base/frameworks/input/``.
+
+.. contents::
+
+Reading Data into Tables
+========================
+
+Probably the most interesting use-case of the input framework is to
+read data into a Bro table.
+
+By default, the input framework reads the data in the same format
+as it is written by the logging framework in Bro - a tab-separated
+ASCII file.
+
+We will show the ways to read files into Bro with a simple example.
+For this example we assume that we want to import data from a blacklist
+that contains server IP addresses as well as the timestamp and the reason
+for the block.
+
+An example input file could look like this:
+
+::
+
+        #fields ip timestamp reason
+        192.168.17.1 1333252748 Malware host
+        192.168.27.2 1330235733 Botnet server
+        192.168.250.3 1333145108 Virus detected
+
+To read a file into a Bro table, two record types have to be defined.
+One contains the types and names of the columns that should constitute the
+table keys and the second contains the types and names of the columns that
+should constitute the table values.
+
+In our case, we want to be able to lookup IPs. Hence, our key record
+only contains the server IP. All other elements should be stored as
+the table content.
+
+The two records are defined as:
+
+.. code:: bro
+
+        type Idx: record {
+                ip: addr;
+        };
+
+        type Val: record {
+                timestamp: time;
+                reason: string;
+        };
+
+Note that the names of the fields in the record definitions have to correspond
+to the column names listed in the '#fields' line of the log file, in this
+case 'ip', 'timestamp', and 'reason'.
+
+The log file is read into the table with a simple call of the ``add_table``
+function:
+
+.. code:: bro
+
+        global blacklist: table[addr] of Val = table();
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist]);
+        Input::remove("blacklist");
+
+With these three lines we first create an empty table that should contain the
+blacklist data and then instruct the input framework to open an input stream
+named ``blacklist`` to read the data into the table. The third line removes the
+input stream again, because we do not need it any more after the data has been
+read.
+
+Because some data files can - potentially - be rather big, the input framework
+works asynchronously. A new thread is created for each new input stream.
+This thread opens the input data file, converts the data into a Bro format and
+sends it back to the main Bro thread.
+
+Because of this, the data is not immediately accessible. Depending on the
+size of the data source it might take from a few milliseconds up to a few
+seconds until all data is present in the table. Please note that this means
+that when Bro is running without an input source or on very short captured
+files, it might terminate before the data is present in the system (because
+Bro already handled all packets before the import thread finished).
+
+Subsequent calls to an input source are queued until the previous action has
+been completed. Because of this, it is, for example, possible to call
+``add_table`` and ``remove`` in two subsequent lines: the ``remove`` action
+will remain queued until the first read has been completed.
+
+Once the input framework finishes reading from a data source, it fires
+the ``end_of_data`` event. Once this event has been received all data
+from the input file is available in the table.
+
+.. code:: bro
+
+        event Input::end_of_data(name: string, source: string) {
+                # now all data is in the table
+                print blacklist;
+        }
+
+The table can also already be used while the data is still being read - it
+just might not contain all lines in the input file when the event has not
+yet fired. After it has been populated it can be used like any other Bro
+table and blacklist entries can easily be tested:
+
+.. code:: bro
+
+        if ( 192.168.18.12 in blacklist )
+                # take action
+
+
+Re-reading and streaming data
+-----------------------------
+
+For many data sources, like for many blacklists, the source data is continually
+changing. For these cases, the Bro input framework supports several ways to
+deal with changing data files.
+
+The first, very basic method is an explicit refresh of an input stream. When
+an input stream is open, the function ``force_update`` can be called. This
+will trigger a complete refresh of the table; any changed elements from the
+file will be updated.  After the update is finished the ``end_of_data``
+event will be raised.
+
+In our example the call would look like:
+
+.. code:: bro
+
+        Input::force_update("blacklist");
+
+The input framework also supports two automatic refresh modes. The first mode
+continually checks if a file has been changed. If the file has been changed, it
+is re-read and the data in the Bro table is updated to reflect the current
+state.  Each time a change has been detected and all the new data has been
+read into the table, the ``end_of_data`` event is raised.
+
+The second mode is a streaming mode. This mode assumes that the source data
+file is an append-only file to which new data is continually appended. Bro
+continually checks for new data at the end of the file and will add the new
+data to the table.  If newer lines in the file have the same index as previous
+lines, they will overwrite the values in the output table.  Because of the
+nature of streaming reads (data is continually added to the table),
+the ``end_of_data`` event is never raised when using streaming reads.
+
+The reading mode can be selected by setting the ``mode`` option of the
+add_table call.  Valid values are ``MANUAL`` (the default), ``REREAD``
+and ``STREAM``.
+
+Hence, when adding ``$mode=Input::REREAD`` to the previous example, the
+blacklist table will always reflect the state of the blacklist input file.
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD]);
+
+Receiving change events
+-----------------------
+
+When re-reading files, it might be interesting to know exactly which lines in
+the source files have changed.
+
+For this reason, the input framework can raise an event each time when a data
+item is added to, removed from or changed in a table.
+
+The event definition looks like this:
+
+.. code:: bro
+
+        event entry(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+                # act on values
+        }
+
+The event has to be specified in ``$ev`` in the ``add_table`` call:
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD, $ev=entry]);
+
+The ``description`` field of the event contains the arguments that were
+originally supplied to the add_table call.  Hence, the name of the stream can,
+for example, be accessed with ``description$name``. ``tpe`` is an enum
+containing the type of the change that occurred.
+
+If a line that was not previously present in the table has been added,
+then ``tpe`` will contain ``Input::EVENT_NEW``. In this case ``left`` contains
+the index of the added table entry and ``right`` contains the values of the
+added entry.
+
+If a table entry that already was present is altered during the re-reading or
+streaming read of a file, ``tpe`` will contain ``Input::EVENT_CHANGED``. In
+this case ``left`` contains the index of the changed table entry and ``right``
+contains the values of the entry before the change. The reason for this is
+that the table already has been updated when the event is raised. The current
+value in the table can be ascertained by looking up the current table value.
+Hence it is possible to compare the new and the old values of the table.
+
+If a table element is removed because it was no longer present during a
+re-read, then ``tpe`` will contain ``Input::REMOVED``.  In this case ``left``
+contains the index and ``right`` the values of the removed element.
+
+
+Filtering data during import
+----------------------------
+
+The input framework also allows a user to filter the data during the import.
+To this end, predicate functions are used. A predicate function is called
+before a new element is added/changed/removed from a table. The predicate
+can either accept or veto the change by returning true for an accepted
+change and false for a rejected change. Furthermore, it can alter the data
+before it is written to the table.
+
+The following example filter will reject to add entries to the table when
+they were generated over a month ago. It will accept all changes and all
+removals of values that are already present in the table.
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD,
+                        $pred(typ: Input::Event, left: Idx, right: Val) = {
+                                if ( typ != Input::EVENT_NEW ) {
+                                        return T;
+                                }
+                                return ( ( current_time() - right$timestamp ) < (30 day) );
+                        }]);
+
+To change elements while they are being imported, the predicate function can
+manipulate ``left`` and ``right``. Note that predicate functions are called
+before the change is committed to the table. Hence, when a table element is
+changed (``tpe`` is ``INPUT::EVENT_CHANGED``), ``left`` and ``right``
+contain the new values, but the destination (``blacklist`` in our example)
+still contains the old values. This allows predicate functions to examine
+the changes between the old and the new version before deciding if they
+should be allowed.
+
+Different readers
+-----------------
+
+The input framework supports different kinds of readers for different kinds
+of source data files. At the moment, the default reader reads ASCII files
+formatted in the Bro log file format (tab-separated values). At the moment,
+Bro comes with two other readers. The ``RAW`` reader reads a file that is
+split by a specified record separator (usually newline). The contents are
+returned line-by-line as strings; it can, for example, be used to read
+configuration files and the like and is probably
+only useful in the event mode and not for reading data to tables.
+
+Another included reader is the ``BENCHMARK`` reader, which is being used
+to optimize the speed of the input framework. It can generate arbitrary
+amounts of semi-random data in all Bro data types supported by the input
+framework.
+
+In the future, the input framework will get support for new data sources
+like, for example, different databases.
+
+Add_table options
+-----------------
+
+This section lists all possible options that can be used for the add_table
+function and gives a short explanation of their use. Most of the options
+already have been discussed in the previous sections.
+
+The possible fields that can be set for a table stream are:
+
+        ``source``
+                A mandatory string identifying the source of the data.
+                For the ASCII reader this is the filename.
+
+        ``name``
+                A mandatory name for the filter that can later be used
+                to manipulate it further.
+
+        ``idx``
+                Record type that defines the index of the table.
+
+        ``val``
+                Record type that defines the values of the table.
+
+        ``reader``
+                The reader used for this stream. Default is ``READER_ASCII``.
+
+        ``mode``
+                The mode in which the stream is opened. Possible values are
+                ``MANUAL``, ``REREAD`` and ``STREAM``.  Default is ``MANUAL``.
+                ``MANUAL`` means that the file is not updated after it has
+                been read. Changes to the file will not be reflected in the
+                data Bro knows.  ``REREAD`` means that the whole file is read
+                again each time a change is found. This should be used for
+                files that are mapped to a table where individual lines can
+                change.  ``STREAM`` means that the data from the file is
+                streamed. Events / table entries will be generated as new
+                data is appended to the file.
+
+        ``destination``
+                The destination table.
+
+        ``ev``
+                Optional event that is raised, when values are added to,
+                changed in, or deleted from the table.  Events are passed an
+                Input::Event description as the first argument, the index
+                record as the second argument and the values as the third
+                argument.
+
+        ``pred``
+                Optional predicate, that can prevent entries from being added
+                to the table and events from being sent.
+
+        ``want_record``
+                Boolean value, that defines if the event wants to receive the
+                fields inside of a single record value, or individually
+                (default).  This can be used if ``val`` is a record
+                containing only one type. In this case, if ``want_record`` is
+                set to false, the table will contain elements of the type
+                contained in ``val``.
+
+Reading Data to Events
+======================
+
+The second supported mode of the input framework is reading data to Bro
+events instead of reading them to a table using event streams.
+
+Event streams work very similarly to table streams that were already
+discussed in much detail. To read the blacklist of the previous example
+into an event stream, the following Bro code could be used:
+
+.. code:: bro
+
+        type Val: record {
+                ip: addr;
+                timestamp: time;
+                reason: string;
+        };
+
+        event blacklistentry(description: Input::EventDescription, tpe: Input::Event, ip: addr, timestamp: time, reason: string) {
+                # work with event data
+        }
+
+        event bro_init() {
+                Input::add_event([$source="blacklist.file", $name="blacklist", $fields=Val, $ev=blacklistentry]);
+        }
+
+
+The main difference in the declaration of the event stream is, that an event
+stream needs no separate index and value declarations -- instead, all source
+data types are provided in a single record definition.
+
+Apart from this, event streams work exactly the same as table streams and
+support most of the options that are also supported for table streams.
+
+The options that can be set when creating an event stream with
+``add_event`` are:
+
+        ``source``
+                A mandatory string identifying the source of the data.
+                For the ASCII reader this is the filename.
+
+        ``name``
+                A mandatory name for the stream that can later be used
+                to remove it.
+
+        ``fields``
+                Name of a record type containing the fields, which should be
+                retrieved from the input stream.
+
+        ``ev``
+                The event which is fired, after a line has been read from the
+                input source.  The first argument that is passed to the event
+                is an Input::Event structure, followed by the data, either
+                inside of a record (if ``want_record is set``) or as
+                individual fields.  The Input::Event structure can contain
+                information, if the received line is ``NEW``, has been
+                ``CHANGED`` or ``DELETED``. Since the ASCII reader cannot
+                track this information for event filters, the value is
+                always ``NEW`` at the moment.
+
+        ``mode``
+                The mode in which the stream is opened. Possible values are
+                ``MANUAL``, ``REREAD`` and ``STREAM``.  Default is ``MANUAL``.
+                ``MANUAL`` means that the file is not updated after it has
+                been read. Changes to the file will not be reflected in the
+                data Bro knows.  ``REREAD`` means that the whole file is read
+                again each time a change is found. This should be used for
+                files that are mapped to a table where individual lines can
+                change.  ``STREAM`` means that the data from the file is
+                streamed. Events / table entries will be generated as new
+                data is appended to the file.
+
+        ``reader``
+                The reader used for this stream. Default is ``READER_ASCII``.
+
+        ``want_record``
+                Boolean value, that defines if the event wants to receive the
+                fields inside of a single record value, or individually
+                (default). If this is set to true, the event will receive a
+                single record of the type provided in ``fields``.
+
+
+

doc/intel.rst

@@ -0,0 +1,119 @@
+Intel Framework
+===============
+
+Intro
+-----
+
+Intelligence data is critical to the process of monitoring for
+security purposes.  There is always data which will be discovered
+through the incident response process and data which is shared through
+private communities.  The goals of Bro's Intelligence Framework are to
+consume that data, make it available for matching, and provide
+infrastructure around improving performance, memory utilization, and
+generally making all of this easier.
+
+Data in the Intelligence Framework is the atomic piece of intelligence
+such as an IP address or an e-mail address along with a suite of
+metadata about it such as a freeform source field, a freeform
+descriptive field and a URL which might lead to more information about
+the specific item.  The metadata in the default scripts has been
+deliberately kept minimal so that the community can find the
+appropriate fields that need added by writing scripts which extend the
+base record using the normal record extension mechanism.
+
+Quick Start
+-----------
+
+Load the package of scripts that sends data into the Intelligence
+Framework to be checked by loading this script in local.bro::
+
+	@load policy/frameworks/intel/seen
+
+Refer to the "Loading Intelligence" section below to see the format
+for Intelligence Framework text files, then load those text files with
+this line in local.bro::
+
+	redef Intel::read_files += { "/somewhere/yourdata.txt" };
+
+The data itself only needs to reside on the manager if running in a
+cluster.
+
+Architecture
+------------
+
+The Intelligence Framework can be thought of as containing three
+separate portions.  The first part is how intelligence is loaded,
+followed by the mechanism for indicating to the intelligence framework
+that a piece of data which needs to be checked has been seen, and
+thirdly the part where a positive match has been discovered.
+
+Loading Intelligence
+********************
+
+Intelligence data can only be loaded through plain text files using
+the Input Framework conventions.  Additionally, on clusters the
+manager is the only node that needs the intelligence data.  The
+intelligence framework has distribution mechanisms which will push
+data out to all of the nodes that need it.
+
+Here is an example of the intelligence data format.  Note that all
+whitespace separators are literal tabs and fields containing only a
+hyphen are considered to be null values.::
+
+	#fields	indicator	indicator_type	meta.source	meta.desc	meta.url
+	1.2.3.4	Intel::ADDR	source1	Sending phishing email	http://source1.com/badhosts/1.2.3.4
+	a.b.com	Intel::DOMAIN	source2	Name used for data exfiltration	-
+
+For more examples of built in `indicator_type` values, please refer to the
+autogenerated documentation for the intelligence framework.
+
+To load the data once files are created, use the following example
+code to define files to load with your own file names of course::
+
+	redef Intel::read_files += {
+		"/somewhere/feed1.txt",
+		"/somewhere/feed2.txt",
+	};
+
+Remember, the files only need to be present on the file system of the
+manager node on cluster deployments.
+
+Seen Data
+*********
+
+When some bit of data is extracted (such as an email address in the
+"From" header in a message over SMTP), the Intelligence Framework
+needs to be informed that this data was discovered and it's presence
+should be checked within the intelligence data set.  This is
+accomplished through the Intel::seen function.
+
+Typically users won't need to work with this function due to built in
+hook scripts that Bro ships with that will "see" data and send it into
+the intelligence framework.  A user may only need to load the entire
+package of hook scripts as a module or pick and choose specific
+scripts to load.  Keep in mind that as more data is sent into the
+intelligence framework, the CPU load consumed by Bro will increase
+depending on how many times the Intel::seen function is being called
+which is heavily traffic dependent.
+
+The full package of hook scripts that Bro ships with for sending this
+"seen" data into the intelligence framework can be loading by adding
+this line to local.bro::
+
+	@load policy/frameworks/intel/seen
+
+Intelligence Matches
+********************
+
+Against all hopes, most networks will eventually have a hit on
+intelligence data which could indicate a possible compromise or other
+unwanted activity.  The Intelligence Framework provides an event that
+is generated whenever a match is discovered named Intel::match (TODO:
+make a link to inline docs).  Due to design restrictions placed upon
+the intelligence framework, there is no assurance as to where this
+event will be generated.  It could be generated on the worker where
+the data was seen or on the manager.  When the Intel::match event is
+handled, only the data given as event arguments to the event can be
+assured since the host where the data was seen may not be where
+Intel::match is handled.
+

doc/logging-dataseries.rst

@@ -0,0 +1,186 @@
+
+=============================
+Binary Output with DataSeries
+=============================
+
+.. rst-class:: opening
+
+   Bro's default ASCII log format is not exactly the most efficient
+   way for storing and searching large volumes of data. An an
+   alternative, Bro comes with experimental support for `DataSeries
+   <http://www.hpl.hp.com/techreports/2009/HPL-2009-323.html>`_
+   output, an efficient binary format for recording structured bulk
+   data. DataSeries is developed and maintained at HP Labs.
+
+.. contents::
+
+Installing DataSeries
+---------------------
+
+To use DataSeries, its libraries must be available at compile-time,
+along with the supporting *Lintel* package. Generally, both are
+distributed on `HP Labs' web site
+<http://tesla.hpl.hp.com/opensource/>`_. Currently, however, you need
+to use recent development versions for both packages, which you can
+download from github like this::
+
+    git clone http://github.com/dataseries/Lintel
+    git clone http://github.com/dataseries/DataSeries
+
+To build and install the two into ``<prefix>``, do::
+
+    ( cd Lintel     && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
+    ( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
+
+Please refer to the packages' documentation for more information about
+the installation process. In particular, there's more information on
+required and optional `dependencies for Lintel
+<https://raw.github.com/dataseries/Lintel/master/doc/dependencies.txt>`_
+and `dependencies for DataSeries
+<https://raw.github.com/dataseries/DataSeries/master/doc/dependencies.txt>`_.
+For users on RedHat-style systems, you'll need the following::
+
+    yum install libxml2-devel boost-devel
+
+Compiling Bro with DataSeries Support
+-------------------------------------
+
+Once you have installed DataSeries, Bro's ``configure`` should pick it
+up automatically as long as it finds it in a standard system location.
+Alternatively, you can specify the DataSeries installation prefix
+manually with ``--with-dataseries=<prefix>``. Keep an eye on
+``configure``'s summary output, if it looks like the following, Bro
+found DataSeries and will compile in the support::
+
+    # ./configure --with-dataseries=/usr/local
+    [...]
+    ====================|  Bro Build Summary  |=====================
+    [...]
+    DataSeries:        true
+    [...]
+    ================================================================
+
+Activating DataSeries
+---------------------
+
+The direct way to use DataSeries is to switch *all* log files over to
+the binary format. To do that, just add ``redef
+Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``.
+For testing, you can also just pass that on the command line::
+
+    bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES
+
+With that, Bro will now write all its output into DataSeries files
+``*.ds``. You can inspect these using DataSeries's set of command line
+tools, which its installation process installs into ``<prefix>/bin``.
+For example, to convert a file back into an ASCII representation::
+
+    $ ds2txt conn.log
+    [... We skip a bunch of metadata here ...]
+    ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
+    1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
+    1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp  0.000000 0 0 S0 F 0 D 1 199 0 0
+    1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp  0.000000 0 0 S0 F 0 D 1 179 0 0
+    1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117
+    1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127
+    1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211
+    [...]
+
+(``--skip-all`` suppresses the metadata.)
+
+Note that the ASCII conversion is *not* equivalent to Bro's default
+output format.
+
+You can also switch only individual files over to DataSeries by adding
+code like this to your ``local.bro``:
+
+.. code:: bro
+
+    event bro_init()
+        {
+        local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
+        f$writer = Log::WRITER_DATASERIES;               # Change writer type.
+        Log::add_filter(Conn::LOG, f);                   # Replace filter with adapted version.
+        }
+
+Bro's DataSeries writer comes with a few tuning options, see
+:doc:`scripts/base/frameworks/logging/writers/dataseries`.
+
+Working with DataSeries
+=======================
+
+Here are a few examples of using DataSeries command line tools to work
+with the output files.
+
+* Printing CSV::
+
+    $ ds2txt --csv conn.log
+    ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes
+    1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0
+    1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0
+    1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0
+    1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0
+    [...]
+
+  Add ``--separator=X`` to set a different separator.
+
+* Extracting a subset of columns::
+
+    $ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log
+    1258790493.773208 192.168.1.255 137
+    1258790451.402091 192.168.1.255 138
+    1258790493.787448 192.168.1.255 138
+    1258790615.268111 192.168.1.255 137
+    1258790615.289842 192.168.1.255 138
+    [...]
+
+* Filtering rows::
+
+    $ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all  conn.ds
+    1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp  15.004568 798 0 S0 F 0 D 6 966 0 0
+    1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp  15.004581 798 0 S0 F 0 D 6 966 0 0
+    1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp  15.005071 798 0 S0 F 0 D 6 966 0 0
+    1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp  15.004824 798 0 S0 F 0 D 6 966 0 0
+    1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp  15.005078 798 0 S0 F 0 D 6 966 0 0
+    1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp  15.004082 798 0 S0 F 0 D 6 966 0 0
+    [...]
+
+* Calculate some statistics:
+
+    Mean/stddev/min/max over a column::
+
+        $ dsstatgroupby '*' basic duration from conn.ds
+        # Begin DSStatGroupByModule
+        # processed 2159 rows, where clause eliminated 0 rows
+        # count(*), mean(duration), stddev, min, max
+        2159, 42.7938, 1858.34, 0, 86370
+        [...]
+
+    Quantiles of total connection volume::
+
+        $ dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds
+        [...]
+        2159 data points, mean 24616 +- 343295 [0,1.26615e+07]
+        quantiles about every 216 data points:
+        10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469
+        tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262
+        [...]
+
+The ``man`` pages for these tools show further options, and their
+``-h`` option gives some more information (either can be a bit cryptic
+unfortunately though).
+
+Deficiencies
+------------
+
+Due to limitations of the DataSeries format, one cannot inspect its
+files before they have been fully written. In other words, when using
+DataSeries, it's currently not possible to inspect the live log
+files inside the spool directory before they are rotated to their
+final location. It seems that this could be fixed with some effort,
+and we will work with DataSeries development team on that if the
+format gains traction among Bro users.
+
+Likewise, we're considering writing custom command line tools for
+interacting with DataSeries files, making that a bit more convenient
+than what the standard utilities provide.

doc/logging-elasticsearch.rst

@@ -0,0 +1,89 @@
+
+=========================================
+Indexed Logging Output with ElasticSearch
+=========================================
+
+.. rst-class:: opening
+
+   Bro's default ASCII log format is not exactly the most efficient
+   way for searching large volumes of data. ElasticSearch
+   is a new data storage technology for dealing with tons of data.
+   It's also a search engine built on top of Apache's Lucene
+   project. It scales very well, both for distributed indexing and 
+   distributed searching.
+
+.. contents::
+
+Warning
+-------
+
+This writer plugin is still in testing and is not yet recommended for
+production use!  The approach to how logs are handled in the plugin is "fire
+and forget" at this time, there is no error handling if the server fails to
+respond successfully to the insertion request.
+
+Installing ElasticSearch
+------------------------
+
+Download the latest version from: <http://www.elasticsearch.org/download/>.
+Once extracted, start ElasticSearch with::
+
+# ./bin/elasticsearch
+
+For more detailed information, refer to the ElasticSearch installation
+documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html
+
+Compiling Bro with ElasticSearch Support
+----------------------------------------
+
+First, ensure that you have libcurl installed the run configure.::
+
+    # ./configure
+    [...]
+    ====================|  Bro Build Summary  |=====================
+    [...]
+    cURL:              true
+    [...]
+    ElasticSearch:     true
+    [...]
+    ================================================================
+
+Activating ElasticSearch
+------------------------
+
+The easiest way to enable ElasticSearch output is to load the tuning/logs-to-
+elasticsearch.bro script.  If you are using BroControl, the following line in
+local.bro will enable it.
+
+.. console::
+
+    @load tuning/logs-to-elasticsearch
+
+With that, Bro will now write most of its logs into ElasticSearch in addition
+to maintaining the Ascii logs like it would do by default.  That script has
+some tunable options for choosing which logs to send to ElasticSearch, refer
+to the autogenerated script documentation for those options.
+
+There is an interface being written specifically to integrate with the data
+that Bro outputs into ElasticSearch named Brownian.  It can be found here::
+
+    https://github.com/grigorescu/Brownian
+
+Tuning
+------
+
+A common problem encountered with ElasticSearch is too many files being held
+open.  The ElasticSearch website has some suggestions on how to increase the
+open file limit.
+
+  - http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html
+
+TODO
+----
+
+Lots.
+
+- Perform multicast discovery for server.
+- Better error detection.
+- Better defaults (don't index loaded-plugins, for instance).
+- 

doc/logging.rst

@@ -43,13 +43,14 @@ Basics
 ======
 
 The data fields that a stream records are defined by a record type
-specified when it is created. Let's look at the script generating
+specified when it is created. Let's look at the script generating Bro's
-Bro's connection summaries as an example,
+connection summaries as an example,
-``base/protocols/conn/main.bro``. It defines a record ``Conn::Info``
+:doc:`scripts/base/protocols/conn/main`. It defines a record
-that lists all the fields that go into ``conn.log``, each marked with
+:bro:type:`Conn::Info` that lists all the fields that go into
-a ``&log`` attribute indicating that it is part of the information
+``conn.log``, each marked with a ``&log`` attribute indicating that it
-written out. To write a log record, the script then passes an instance
+is part of the information written out. To write a log record, the
-of ``Conn::Info`` to the logging framework's ``Log::write`` function.
+script then passes an instance of :bro:type:`Conn::Info` to the logging
+framework's :bro:id:`Log::write` function.
 
 By default, each stream automatically gets a filter named ``default``
 that generates the normal output by recording all record fields into a
@@ -62,11 +63,11 @@ to work with.
 Filtering
 ---------
 
-To create new a new output file for an existing stream, you can add a
+To create a new output file for an existing stream, you can add a
 new filter. A filter can, e.g., restrict the set of fields being
 logged:
 
-.. code:: bro:
+.. code:: bro
 
     event bro_init()
         {
@@ -85,14 +86,14 @@ Note the fields that are set for the filter:
     ``path``
         The filename for the output file, without any extension (which
         may be automatically added by the writer). Default path values
-        are generated by taking the stream's ID and munging it
+        are generated by taking the stream's ID and munging it slightly.
-        slightly. ``Conn::LOG`` is converted into ``conn``,
+        :bro:enum:`Conn::LOG` is converted into ``conn``,
-        ``PacketFilter::LOG`` is converted into ``packet_filter``, and 
+        :bro:enum:`PacketFilter::LOG` is converted into
-        ``Notice::POLICY_LOG`` is converted into ``notice_policy``.
+        ``packet_filter``.
 
     ``include``
         A set limiting the fields to the ones given. The names
-        correspond to those in the ``Conn::LOG`` record, with
+        correspond to those in the :bro:type:`Conn::Info` record, with
         sub-records unrolled by concatenating fields (separated with 
         dots).
 
@@ -158,10 +159,10 @@ further for example to log information by subnets or even by IP
 address. Be careful, however, as it is easy to create many files very
 quickly ...
 
-.. sidebar:
+.. sidebar:: A More Generic Path Function
 
-    The show ``split_log`` method has one draw-back: it can be used
+    The ``split_log`` method has one draw-back: it can be used
-    only with the ``Conn::Log`` stream as the record type is hardcoded
+    only with the :bro:enum:`Conn::LOG` stream as the record type is hardcoded
     into its argument list. However, Bro allows to do a more generic
     variant:
 
@@ -193,7 +194,7 @@ predicate that will be called for each log record:
         Log::add_filter(Conn::LOG, filter);
         }
 
-This will results in a log file ``conn-http.log`` that contains only
+This will result in a log file ``conn-http.log`` that contains only
 traffic detected and analyzed as HTTP traffic.
 
 Extending
@@ -201,8 +202,8 @@ Extending
 
 You can add further fields to a log stream by extending the record
 type that defines its content. Let's say we want to add a boolean
-field ``is_private`` to ``Conn::Info`` that indicates whether the
+field ``is_private`` to :bro:type:`Conn::Info` that indicates whether the
-originator IP address is part of the RFC1918 space:
+originator IP address is part of the :rfc:`1918` space:
 
 .. code:: bro
 
@@ -234,10 +235,10 @@ Notes:
 - For extending logs this way, one needs a bit of knowledge about how
   the script that creates the log stream is organizing its state
   keeping. Most of the standard Bro scripts attach their log state to
-  the ``connection`` record where it can then be accessed, just as the
+  the :bro:type:`connection` record where it can then be accessed, just
-  ``c$conn`` above. For example, the HTTP analysis adds a field ``http
+  as the ``c$conn`` above. For example, the HTTP analysis adds a field
-  : HTTP::Info`` to the ``connection`` record. See the script
+  ``http`` of type :bro:type:`HTTP::Info` to the :bro:type:`connection`
-  reference for more information.
+  record. See the script reference for more information.
 
 - When extending records as shown above, the new fields must always be
   declared either with a ``&default`` value or as ``&optional``.
@@ -251,8 +252,8 @@ Sometimes it is helpful to do additional analysis of the information
 being logged. For these cases, a stream can specify an event that will
 be generated every time a log record is written to it. All of Bro's
 default log streams define such an event. For example, the connection
-log stream raises the event ``Conn::log_conn(rec: Conn::Info)``: You
+log stream raises the event :bro:id:`Conn::log_conn`. You
-could use that for example for flagging when an a connection to
+could use that for example for flagging when a connection to a
 specific destination exceeds a certain duration:
 
 .. code:: bro
@@ -267,7 +268,7 @@ specific destination exceeds a certain duration:
         {
         if ( rec$duration > 5mins )
             NOTICE([$note=Long_Conn_Found, 
-                    $msg=fmt("unsually long conn to %s", rec$id$resp_h), 
+                    $msg=fmt("unusually long conn to %s", rec$id$resp_h), 
                     $id=rec$id]);
         }
 
@@ -279,11 +280,32 @@ real-time.
 Rotation
 --------
 
+By default, no log rotation occurs, but it's globally controllable for all
+filters by redefining the :bro:id:`Log::default_rotation_interval` option:
+
+.. code:: bro
+
+    redef Log::default_rotation_interval = 1 hr;
+
+Or specifically for certain :bro:type:`Log::Filter` instances by setting
+their ``interv`` field.  Here's an example of changing just the
+:bro:enum:`Conn::LOG` stream's default filter rotation.
+
+.. code:: bro
+
+    event bro_init()
+        {
+        local f = Log::get_filter(Conn::LOG, "default");
+        f$interv = 1 min;
+        Log::remove_filter(Conn::LOG, "default");
+        Log::add_filter(Conn::LOG, f);
+        }
+
 ASCII Writer Configuration
 --------------------------
 
 The ASCII writer has a number of options for customizing the format of
-its output, see XXX.bro.
+its output, see :doc:`scripts/base/frameworks/logging/writers/ascii`.
 
 Adding Streams
 ==============
@@ -296,7 +318,7 @@ example for the ``Foo`` module:
     module Foo;
 
     export {
-        # Create an ID for the our new stream. By convention, this is
+        # Create an ID for our new stream. By convention, this is
         # called "LOG".
         redef enum Log::ID += { LOG };
 
@@ -321,8 +343,8 @@ example for the ``Foo`` module:
         Log::create_stream(Foo::LOG, [$columns=Info, $ev=log_foo]);
         }
 
-You can also the state to the ``connection`` record to make it easily
+You can also add the state to the :bro:type:`connection` record to make
-accessible across event handlers:
+it easily accessible across event handlers:
 
 .. code:: bro
 
@@ -330,7 +352,7 @@ accessible across event handlers:
         foo: Info &optional;
         }
 
-Now you can use the ``Log::write`` method to output log records and 
+Now you can use the :bro:id:`Log::write` method to output log records and 
 save the logged ``Foo::Info`` record into the connection record:
 
 .. code:: bro
@@ -343,10 +365,21 @@ save the logged ``Foo::Info`` record into the connection record:
         }
 
 See the existing scripts for how to work with such a new connection
-field. A simple example is ``base/protocols/syslog/main.bro``. 
+field. A simple example is :doc:`scripts/base/protocols/syslog/main`. 
 
-When you are developing scripts that add data to the ``connection``
+When you are developing scripts that add data to the :bro:type:`connection`
 record, care must be given to when and how long data is stored.
 Normally data saved to the connection record will remain there for the
 duration of the connection and from a practical perspective it's not
 uncommon to need to delete that data before the end of the connection.
+
+Other Writers
+-------------
+
+Bro supports the following output formats other than ASCII:
+
+.. toctree::
+   :maxdepth: 1
+
+   logging-dataseries
+   logging-elasticsearch

doc/notice.rst

@@ -6,7 +6,7 @@ Notice Framework
 
     One of the easiest ways to customize Bro is writing a local notice
     policy. Bro can detect a large number of potentially interesting
-    situations, and the notice policy tells which of them the user wants to be
+    situations, and the notice policy hook which of them the user wants to be
     acted upon in some manner. In particular, the notice policy can specify
     actions to be taken, such as sending an email or compiling regular
     alarm emails.  This page gives an introduction into writing such a notice
@@ -21,11 +21,11 @@ Let's start with a little bit of background on Bro's philosophy on reporting
 things. Bro ships with a large number of policy scripts which perform a wide
 variety of analyses. Most of these scripts monitor for activity which might be
 of interest for the user. However, none of these scripts determines the
-importance of what it finds itself. Instead, the scripts only flags situations
+importance of what it finds itself. Instead, the scripts only flag situations
 as *potentially* interesting, leaving it to the local configuration to define
 which of them are in fact actionable. This decoupling of detection and
-reporting allows Bro to address the different needs that sites have:
+reporting allows Bro to address the different needs that sites have.
-definitions of what constitutes an attack or even a compromise differ quite a
+Definitions of what constitutes an attack or even a compromise differ quite a
 bit between environments, and activity deemed malicious at one site might be
 fully acceptable at another.
 
@@ -40,7 +40,7 @@ More information about raising notices can be found in the `Raising Notices`_
 section.
 
 Once a notice is raised, it can have any number of actions applied to it by
-the :bro:see:`Notice::policy` set which is described in the `Notice Policy`_
+writing :bro:see:`Notice::policy` hooks which is described in the `Notice Policy`_
 section below. Such actions can be to send a mail to the configured
 address(es) or to simply ignore the notice. Currently, the following actions
 are defined:
@@ -68,12 +68,6 @@ are defined:
       - Send an email to the email address or addresses given in the
         :bro:see:`Notice::mail_page_dest` variable.
 
-    * - Notice::ACTION_NO_SUPPRESS
-      - This action will disable the built in notice suppression for the
-        notice. Keep in mind that this action will need to be applied to
-        every notice that shouldn't be suppressed including each of the future
-        notices that would have normally been suppressed.
-
 How these notice actions are applied to notices is discussed in the
 `Notice Policy`_ and `Notice Policy Shortcuts`_ sections.
 
@@ -83,105 +77,46 @@ Processing Notices
 Notice Policy
 *************
 
-The predefined set :bro:see:`Notice::policy` provides the mechanism for
+The hook :bro:see:`Notice::policy` provides the mechanism for applying
-applying actions and other behavior modifications to notices. Each entry
+actions and generally modifying the notice before it's sent onward to
-of :bro:see:`Notice::policy` is a record of the type
+the action plugins.  Hooks can be thought of as multi-bodied functions
-:bro:see:`Notice::PolicyItem` which defines a condition to be matched
+and using them looks very similar to handling events.  The difference
-against all raised notices and one or more of a variety of behavior
+is that they don't go through the event queue like events.  Users should
-modifiers. The notice policy is defined by adding any number of
+directly make modifications to the :bro:see:`Notice::Info` record
-:bro:see:`Notice::PolicyItem` records to the :bro:see:`Notice::policy`
+given as the argument to the hook.
-set.
 
 Here's a simple example which tells Bro to send an email for all notices of
-type :bro:see:`SSH::Login` if the server is 10.0.0.1:
+type :bro:see:`SSH::Password_Guessing` if the server is 10.0.0.1:
 
 .. code:: bro
 
-    redef Notice::policy += {
+    hook Notice::policy(n: Notice::Info)
-      [$pred(n: Notice::Info) = {
+      {
-         return n$note == SSH::Login && n$id$resp_h == 10.0.0.1;
+      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
-       },
+        add n$actions[Notice::ACTION_EMAIL];
-       $action = Notice::ACTION_EMAIL]
+      }
-      };
 
 .. note::
 
-    Keep in mind that the semantics of the SSH::Login notice are
+    Keep in mind that the semantics of the SSH::Password_Guessing notice are
-    such that it is only raised when Bro heuristically detects a successful
+    such that it is only raised when Bro heuristically detects a failed
-    login. No apparently failed logins will raise this notice.
+    login.
-
-While the syntax might look a bit convoluted at first, it provides a lot of
-flexibility due to having access to Bro's full programming language.
-
-Predicate Field
-^^^^^^^^^^^^^^^
-
-The :bro:see:`Notice::PolicyItem` record type has a field name ``$pred``
-which defines the entry's condition in the form of a predicate written
-as a Bro function. The function is passed the notice as a
-:bro:see:`Notice::Info` record and it returns a boolean value indicating
-if the entry is applicable to that particular notice.
-
-.. note::
-
-    The lack of a predicate in a ``Notice::PolicyItem`` is implicitly true
-    (``T``) since an implicit false (``F``) value would never be used.
-
-Bro evaluates the predicates of each entry in the order defined by the
-``$priority`` field in :bro:see:`Notice::PolicyItem` records. The valid
-values are 0-10 with 10 being earliest evaluated. If ``$priority`` is
-omitted, the default priority is 5.
-
-Behavior Modification Fields
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-
-There are a set of fields in the :bro:see:`Notice::PolicyItem` record type that
-indicate ways that either the notice or notice processing should be modified
-if the predicate field (``$pred``) evaluated to true (``T``). Those fields are
-explained in more detail in the following table.
-
-.. list-table::
-    :widths: 20 30 20
-    :header-rows: 1
-
-    * - Field
-      - Description
-      - Example
-
-    * - ``$action=<Notice::Action>``
-      - Each :bro:see:`Notice::PolicyItem` can have a single action
-        applied to the notice with this field.
-      - ``$action = Notice::ACTION_EMAIL``
-
-    * - ``$suppress_for=<interval>`` 
-      - This field makes it possible for a user to modify the behavior of the
-        notice framework's automated suppression of intrinsically similar
-        notices. More information about the notice framework's automated
-        suppression can be found in the `Automated Suppression`_ section of
-        this document.
-      - ``$suppress_for = 10mins``
-
-    * - ``$halt=<bool>``
-      - This field can be used for modification of the notice policy
-        evaluation. To stop processing of notice policy items before
-        evaluating all of them, set this field to ``T`` and make the ``$pred``
-        field return ``T``. :bro:see:`Notice::PolicyItem` records defined at
-        a higher priority as defined by the ``$priority`` field will still be
-        evaluated but those at a lower priority won't.
-      - ``$halt = T``
-
 
+Hooks can also have priorities applied to order their execution like events
+with a default priority of 0.  Greater values are executed first.  Setting
+a hook body to run before default hook bodies might look like this:
 
 .. code:: bro
 
-    redef Notice::policy += {
+    hook Notice::policy(n: Notice::Info) &priority=5
-      [$pred(n: Notice::Info) = {
+      {
-         return n$note == SSH::Login && n$id$resp_h == 10.0.0.1;
+      if ( n$note == SSH::Password_Guessing && n$id$resp_h == 10.0.0.1 )
-       },
+        add n$actions[Notice::ACTION_EMAIL];
-       $action = Notice::ACTION_EMAIL,
+      }
-       $priority=5]
-      };
 
+Hooks can also abort later hook bodies with the ``break`` keyword. This
+is primarily useful if one wants to completely preempt processing by
+lower priority :bro:see:`Notice::policy` hooks.
 
 Notice Policy Shortcuts
 ***********************
@@ -189,7 +124,7 @@ Notice Policy Shortcuts
 Although the notice framework provides a great deal of flexibility and
 configurability there are many times that the full expressiveness isn't needed
 and actually becomes a hindrance to achieving results. The framework provides
-a default :bro:see:`Notice::policy` suite as a way of giving users the
+a default :bro:see:`Notice::policy` hook body as a way of giving users the
 shortcuts to easily apply many common actions to notices.
 
 These are implemented as sets and tables indexed with a
@@ -221,7 +156,7 @@ framework.
 
     * - :bro:see:`Notice::not_suppressed_types`
       - Adding a :bro:see:`Notice::Type` to this set results in that notice
-        no longer undergoes the normal notice suppression that would
+        no longer undergoing the normal notice suppression that would
         take place. Be careful when using this in production it could
         result in a dramatic increase in the number of notices being
         processed.
@@ -238,16 +173,16 @@ Raising Notices
 
 A script should raise a notice for any occurrence that a user may want
 to be notified about or take action on. For example, whenever the base
-SSH analysis scripts sees an SSH session where it is heuristically
+SSH analysis scripts sees enough failed logins to a given host, it
-guessed to be a successful login, it raises a Notice of the type
+raises a notice of the type :bro:see:`SSH::Password_Guessing`.  The code
-:bro:see:`SSH::Login`. The code in the base SSH analysis script looks
+in the base SSH analysis script which raises the notice looks like this:
-like this:
 
 .. code:: bro
 
-    NOTICE([$note=SSH::Login, 
+    NOTICE([$note=Password_Guessing,
-            $msg="Heuristically detected successful SSH login.",
+            $msg=fmt("%s appears to be guessing SSH passwords (seen in %d connections).", key$host, r$num),
-            $conn=c]);
+            $src=key$host,
+            $identifier=cat(key$host)]);
 
 :bro:see:`NOTICE` is a normal function in the global namespace which
 wraps a function within the ``Notice`` namespace. It takes a single
@@ -270,21 +205,21 @@ fields used when raising notices are described in the following table:
         information about this particular instance of the notice type.
 
     * - ``$sub``
-      - This is a sub-message which meant for human readability but will
+      - This is a sub-message meant for human readability but will
         frequently also be used to contain data meant to be matched with the
         ``Notice::policy``.
 
     * - ``$conn``
       - If a connection record is available when the notice is being raised
-        and the notice represents some attribute of the connection the
+        and the notice represents some attribute of the connection, then the
         connection record can be given here. Other fields such as ``$id`` and
         ``$src`` will automatically be populated from this value.
 
     * - ``$id``
       - If a conn_id record is available when the notice is being raised and
-        the notice represents some attribute of the connection, the connection
+        the notice represents some attribute of the connection, then the
-        be given here. Other fields such as ``$src`` will automatically be
+        connection can be given here. Other fields such as ``$src`` will
-        populated from this value.
+        automatically be populated from this value.
 
     * - ``$src``
       - If the notice represents an attribute of a single host then it's
@@ -313,7 +248,7 @@ of the notice the best information about the notice. If the notice is
 representative of many connections and is an attribute of a host (e.g. a
 scanning host) it probably makes most sense to fill out the ``$src`` field and
 not give a connection or conn_id. If a notice is representative of a
-connection attribute (e.g. an apparent SSH login) the it makes sense to fill
+connection attribute (e.g. an apparent SSH login) then it makes sense to fill
 out either ``$conn`` or ``$id`` based on the data that is available when the
 notice is raised. Using care when inserting data into a notice will make later
 analysis easier when only the data to fully represent the occurrence that
@@ -377,19 +312,45 @@ Setting the ``$identifier`` field is left to those raising notices because
 it's assumed that the script author who is raising the notice understands the
 full problem set and edge cases of the notice which may not be readily
 apparent to users. If users don't want the suppression to take place or simply
-want a different interval, they can always modify it with the
+want a different interval, they can set a notice's suppression
-:bro:see:`Notice::policy`.
+interval to ``0secs`` or delete the value from the ``$identifier`` field in
+a :bro:see:`Notice::policy` hook.
 
 
 Extending Notice Framework
 --------------------------
 
-Adding Custom Notice Actions
+There are a couple of mechanism currently for extending the notice framework
-****************************
+and adding new capability.
 
 Extending Notice Emails
 ***********************
 
+If there is extra information that you would like to add to emails, that is
+possible to add by writing :bro:see:`Notice::policy` hooks.
+
+There is a field in the :bro:see:`Notice::Info` record named
+``$email_body_sections`` which will be included verbatim when email is being
+sent. An example of including some information from an HTTP request is
+included below.
+
+.. code:: bro
+
+    hook Notice::policy(n: Notice::Info)
+      {
+      if ( n?$conn && n$conn?$http && n$conn$http?$host )
+        n$email_body_sections[|email_body_sections|] = fmt("HTTP host header: %s", n$conn$http$host);
+      }
+
+
 Cluster Considerations
 ----------------------
 
+As a user/developer of Bro, the main cluster concern with the notice framework
+is understanding what runs where. When a notice is generated on a worker, the
+worker checks to see if the notice shoudl be suppressed based on information
+locally maintained in the worker process. If it's not being
+suppressed, the worker forwards the notice directly to the manager and does no more
+local processing. The manager then runs the :bro:see:`Notice::policy` hook and
+executes all of the actions determined to be run.
+

doc/quickstart.rst

@@ -1,9 +1,3 @@
-.. _CMake: http://www.cmake.org
-.. _SWIG: http://www.swig.org
-.. _MacPorts: http://www.macports.org
-.. _Fink: http://www.finkproject.org
-.. _Homebrew: http://mxcl.github.com/homebrew
-.. _bro downloads page: http://bro-ids.org/download/index.html
 
 =================
 Quick Start Guide
@@ -22,166 +16,11 @@ Installation
 
 Bro works on most modern, Unix-based systems and requires no custom
 hardware.  It can be downloaded in either pre-built binary package or
-source code forms.
+source code forms.  See :doc:`Installing Bro <INSTALL>` for instructions
+on how to install Bro.
 
-Pre-Built Binary Release Packages
+.. note:: Below, ``$PREFIX`` is used to reference the Bro installation
----------------------------------
+   root directory.
-
-See the `bro downloads page`_ for currently supported/targeted platforms.
-
-* RPM
-
-.. console::
-
-   sudo yum localinstall Bro-all*.rpm
-
-* DEB
-
-.. console::
-
-   sudo gdebi Bro-all-*.deb
-
-* MacOS Disk Image with Installer
-
-  Just open the ``Bro-all-*.dmg`` and then run the ``.pkg`` installer.
-  Everything installed by the package will go into ``/opt/bro``.
-
-The primary install prefix for binary packages is ``/opt/bro``.
-Non-MacOS packages that include BroControl also put variable/runtime
-data (e.g. Bro logs) in ``/var/opt/bro``.
-
-Building From Source
---------------------
-
-Required Dependencies
-~~~~~~~~~~~~~~~~~~~~~
-
-* RPM/RedHat-based Linux:
-
-.. console::
-
-   sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
-
-* DEB/Debian-based Linux:
-
-.. console::
-
-   sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
-
-* FreeBSD
-
-  Most required dependencies should come with a minimal FreeBSD install
-  except for the following.
-
-.. console::
-
-   sudo pkg_add -r cmake swig bison python
-
-* Mac OS X
-
-  Snow Leopard (10.6) comes with all required dependencies except for CMake_.
-
-  Lion (10.7) comes with all required dependencies except for CMake_ and SWIG_.
-
-  Distributions of these dependencies can be obtained from the project websites
-  linked above, but they're also likely available from your preferred Mac OS X
-  package management system (e.g. MacPorts_, Fink_, or Homebrew_).
-
-  Note that the MacPorts ``swig`` package may not include any specific
-  language support so you may need to also install ``swig-ruby`` and
-  ``swig-python``.
-
-Optional Dependencies
-~~~~~~~~~~~~~~~~~~~~~
-
-Bro can use libGeoIP for geo-locating IP addresses and sendmail for
-sending emails.
-
-* RPM/RedHat-based Linux:
-
-.. console::
-
-   sudo yum install GeoIP-devel sendmail
-
-* DEB/Debian-based Linux:
-
-.. console::
-
-   sudo apt-get install libgeoip-dev sendmail
-
-* Ports-based FreeBSD
-
-.. console::
-
-   sudo pkg_add -r GeoIP
-
-  sendmail is typically already available.
-
-* Mac OS X
-
-  Vanilla OS X installations don't ship with libmagic or libGeoIP, but
-  if installed from your preferred package management system (e.g. MacPorts,
-  Fink Homebrew), they should be automatically detected and Bro will compile
-  against them.
-
-Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`
-
-Compiling Bro Source Code
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Bro releases are bundled into source packages for convenience and
-available from the `bro downloads page`_.
-
-The latest Bro development versions are obtainable through git
-repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_.  See
-our `git development documentation
-<http://bro-ids.org/development/process.html>`_ for comprehensive
-information on Bro's use of git revision control, but the short story
-for downloading the full source code experience for Bro via git is:
-
-.. console::
-
-    git clone --recursive git://git.bro-ids.org/bro
-
-.. note:: If you choose to clone the ``bro`` repository non-recursively for
-   a "minimal Bro experience", be aware that compiling it depends on
-   BinPAC, which has it's own ``binpac`` repository.  Either install it
-   first or initizalize/update the cloned ``bro`` repository's
-   ``aux/binpac`` submodule.
-
-See the ``INSTALL`` file included with the source code for more information
-on compiling, but this is the typical way to build and install from source
-(of course, changing the value of the ``--prefix`` option to point to the
-desired root install path):
-
-.. console::
-
-    ./configure --prefix=/desired/install/path
-    make
-    make install
-
-The default installation prefix is ``/usr/local/bro``, which would typically
-require root privileges when doing the ``make install``.
-
-Configure the Run-Time Environment
-----------------------------------
-
-Just remember that you may need to adjust your ``PATH`` environment variable
-according to the platform/shell/package you're using.  For example:
-
-Bourne-Shell Syntax:
-
-.. console::
-
-   export PATH=/usr/local/bro/bin:$PATH
-
-C-Shell Syntax:
-
-.. console::
-
-   setenv PATH /usr/local/bro/bin:$PATH
-
-Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
 
 Using BroControl
 ================
@@ -190,9 +29,6 @@ BroControl is an interactive shell for easily operating/managing Bro
 installations on a single system or even across multiple systems in a
 traffic-monitoring cluster.
 
-.. note:: Below, ``$PREFIX``, is used to reference the Bro installation
-   root directory.
-
 A Minimal Starting Configuration
 --------------------------------
 
@@ -275,7 +111,7 @@ protocol-dependent activity that's occurring.  E.g. ``http.log``'s next few
 columns (shortened for brevity) show a request to the root of Bro website::
 
     # method   host         uri  referrer  user_agent
-    GET        bro-ids.org  /    -         <...>Chrome/12.0.742.122<...>
+    GET        bro.org  /    -         <...>Chrome/12.0.742.122<...>
 
 Some logs are worth explicit mention:
 
@@ -443,7 +279,7 @@ that only takes the email action for SSH logins to a defined set of servers:
         ]
     };
 
-You'll just have to trust the syntax for now, but what we've done is first
+You'll just have to trust the syntax for now, but what we've done is
 first declare our own variable to hold a set of watched addresses,
 ``watched_servers``; then added a record to the policy that will generate
 an email on the condition that the predicate function evaluates to true, which

doc/reporting-problems.rst

@@ -4,7 +4,7 @@ Reporting Problems
 
 .. rst-class:: opening
 
-    Here we summarizes some steps to follow when you see Bro doing
+    Here we summarize some steps to follow when you see Bro doing
     something it shouldn't. To provide help, it is often crucial for
     us to have a way of reliably reproducing the effect you're seeing.
     Unfortunately, reproducing problems can be rather tricky with Bro
@@ -19,17 +19,17 @@ Reporting Problems
 
 Generally, when you encounter a problem with Bro, the best thing to do
 is opening a new ticket in `Bro's issue tracker
-<http://tracker.bro-ids.org/>`__ and include information on how to
+<http://tracker.bro.org/>`__ and include information on how to
-reproduce the issue. Ideallt, your ticket should come with the
+reproduce the issue. Ideally, your ticket should come with the
 following:
 
 * The Bro version you're using (if working directly from the git
   repository, the branch and revision number.)
 
-* The output you're seeing along with a description what you'd expect
+* The output you're seeing along with a description of what you'd expect
   Bro to do instead.
 
-* A *small* trace in `libpcap format <http://tcpdump.org>`__
+* A *small* trace in `libpcap format <http://www.tcpdump.org>`__
   demonstrating the effect (assuming the problem doesn't happen right
   at startup already).
 
@@ -39,7 +39,7 @@ following:
 
 * Any non-standard scripts you're using (but please only those really
   necessary; just a small code snippet triggering the problem would
-  perfect).
+  be perfect).
 
 * If you encounter a crash, information from the core dump, such as
   the stack backtrace, can be very helpful. See below for more on
@@ -51,10 +51,10 @@ How Do I Get a Trace File?
 
 As Bro is usually running live, coming up with a small trace file that
 reproduces a problem can turn out to be quite a challenge. Often it
-works to best to start with a large trace that triggers the problem,
+works best to start with a large trace that triggers the problem,
-and then successively thin it out as much a possible.
+and then successively thin it out as much as possible.
 
-To get to the initial large trace, here are few things you can try:
+To get to the initial large trace, here are a few things you can try:
 
 * Capture a trace with `tcpdump <http://www.tcpdump.org/>`__, either
   on the same interface Bro is running on, or on another host where
@@ -66,14 +66,14 @@ To get to the initial large trace, here are few things you can try:
   (e.g., for HTTP only, try ``port 80``). 
 
 * Bro's command-line option ``-w <trace>`` records all packets it
-  processes into the given the file. You can then later run Bro
+  processes into the given file. You can then later run Bro
   offline on this trace and it will process the packets in the same
   way as it did live. This is particularly helpful with problems that
   only occur after Bro has already been running for some time. For
   example, sometimes a crash may be triggered by a particular kind of
   traffic only occurring rarely. Running Bro live with ``-w`` and
   then, after the crash, offline on the recorded trace might, with a
-  little bit of luck, reproduce the the problem reliably. However, be
+  little bit of luck, reproduce the problem reliably. However, be
   careful with ``-w``: it can result in huge trace files, quickly
   filling up your disk. (One way to mitigate the space issues is to
   periodically delete the trace file by configuring
@@ -96,7 +96,7 @@ much as possible. Here are a few things you can try to this end:
 * Very often, a single connection is able to demonstrate the problem.
   If you can identify which one it is (e.g., from one of Bro's
   ``*.log`` files) you can extract the connection's packets from the
-  trace usong tcpdump by filtering for the corresponding 4-tuple of
+  trace using tcpdump by filtering for the corresponding 4-tuple of
   addresses and ports:
 
   .. console::
@@ -131,8 +131,8 @@ First, you should configure Bro with the option ``--enable-debug`` and
 recompile; this will disable all compiler optimizations and thus make
 the core dump more useful (don't expect great performance with this
 version though; compiling Bro without optimization has a noticeable
-impact on its CPU usage.). Then enable core dumps if you don't have
+impact on its CPU usage.). Then enable core dumps if you haven't
-already (e.g., ``ulimit -c unlimited`` if you're using a bash).
+already (e.g., ``ulimit -c unlimited`` if you're using bash).
 
 Once Bro has crashed, start gdb with the Bro binary and the file
 containing the core dump. (Alternatively, you can also run Bro
@@ -188,7 +188,7 @@ belonging to the ``Connection`` class. That class has members
 
 Note that these values are stored in `network byte order
 <http://en.wikipedia.org/wiki/Endianness#Endianness_in_networking>`__
-so you will need flip the bytes around if you are on a low-endian
+so you will need to flip the bytes around if you are on a low-endian
 machine (which is why the above example prints them in hex). For
 example, if an IP address prints as ``0100007f`` , that's 127.0.0.1 .
 

doc/scripts/CMakeLists.txt

@@ -15,11 +15,11 @@ endif ()
 #
 # srcDir: the directory which contains broInput
 # broInput: the file name of a bro policy script, any path prefix of this
-#     argument will be used to derive what path under policy/ the generated
+#     argument will be used to derive what path under scripts/ the generated
 #     documentation will be placed.
 # group: optional name of group that the script documentation will belong to.
-#     If this is not given, .bif files automatically get their own group or
+#     If this is not given, the group is automatically set to any path portion
-#     the group is automatically by any path portion of the broInput argument.
+#     of the broInput argument.
 #
 # In addition to adding the makefile target, several CMake variables are set:
 #
@@ -45,12 +45,6 @@ macro(REST_TARGET srcDir broInput)
 
     set(sumTextSrc ${absSrcPath})
     set(ogSourceFile ${absSrcPath})
-    if (${extension} STREQUAL ".bif.bro")
-        set(ogSourceFile ${BIF_SRC_DIR}/${basename})
-        # the summary text is taken at configure time, but .bif.bro files
-        # may not have been generated yet, so read .bif file instead
-        set(sumTextSrc ${ogSourceFile})
-    endif ()
 
     if (NOT relDstDir)
         set(docName "${basename}")
@@ -70,8 +64,6 @@ macro(REST_TARGET srcDir broInput)
 
     if (NOT "${ARGN}" STREQUAL "")
         set(group ${ARGN})
-    elseif (${extension} STREQUAL ".bif.bro")
-        set(group bifs)
     elseif (relDstDir)
         set(group ${relDstDir}/index)
         # add package index to master package list if not already in it
@@ -107,7 +99,7 @@ macro(REST_TARGET srcDir broInput)
         COMMAND "${CMAKE_COMMAND}"
         ARGS -E remove_directory .state
         # generate the reST documentation using bro
-        COMMAND BROPATH=${BROPATH}:${srcDir} ${CMAKE_BINARY_DIR}/src/bro
+        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
         ARGS -b -Z ${broInput} || (rm -rf .state *.log *.rst && exit 1)
         # move generated doc into a new directory tree that
         # defines the final structure of documents
@@ -132,6 +124,35 @@ endmacro(REST_TARGET)
 # Schedule Bro scripts for which to generate documentation.
 include(DocSourcesList.cmake)
 
+# Macro for generating reST docs that are independent of any particular Bro
+# script.
+macro(INDEPENDENT_REST_TARGET reST_file)
+    add_custom_command(OUTPUT ${reST_file}
+        # delete any leftover state from previous bro runs
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E remove_directory .state
+        # generate the reST documentation using bro
+        COMMAND BROPATH=${BROPATH}:${srcDir} BROMAGIC=${CMAKE_SOURCE_DIR}/magic/database ${CMAKE_BINARY_DIR}/src/bro
+        ARGS -b -Z base/init-bare.bro || (rm -rf .state *.log *.rst && exit 1)
+        # move generated doc into a new directory tree that
+        # defines the final structure of documents
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E make_directory ${dstDir}
+        COMMAND "${CMAKE_COMMAND}"
+        ARGS -E copy ${reST_file} ${dstDir}
+        # clean up the build directory
+        COMMAND rm
+        ARGS -rf .state *.log *.rst
+        DEPENDS bro
+        WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
+        COMMENT "[Bro] Generating reST docs for ${reST_file}"
+    )
+    list(APPEND ALL_REST_OUTPUTS ${reST_file})
+endmacro(INDEPENDENT_REST_TARGET)
+
+independent_rest_target(proto-analyzers.rst)
+independent_rest_target(file-analyzers.rst)
+
 # create temporary list of all docs to include in the master policy/index file
 file(WRITE ${MASTER_POLICY_INDEX} "${MASTER_POLICY_INDEX_TEXT}")
 

doc/scripts/DocSourcesList.cmake

@@ -16,13 +16,68 @@ rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
 rest_target(${psd} base/init-default.bro internal)
 rest_target(${psd} base/init-bare.bro internal)
 
-rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/analyzer.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bloom-filter.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/bro.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/const.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/event.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/file_analysis.bif.bro)
-rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/input.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ARP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_AYIYA.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BackDoor.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_BitTorrent.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ConnSize.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DCE_RPC.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DHCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_DNS.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FTP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_File.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_FileHash.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Finger.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_GTPv1.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Gnutella.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_HTTP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ICMP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_IRC.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Ident.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_InterConn.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Login.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_MIME.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Modbus.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetBIOS.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_NetFlow.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_PIA.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_POP3.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_RPC.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMB.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SMTP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SOCKS.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSH.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SSL.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_SteppingStone.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Syslog.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_TCP.functions.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_Teredo.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_UDP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/plugins/Bro_ZIP.events.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/top-k.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/scripts base/bif/types.bif.bro)
+rest_target(${psd} base/files/extract/main.bro)
+rest_target(${psd} base/files/hash/main.bro)
+rest_target(${psd} base/frameworks/analyzer/main.bro)
 rest_target(${psd} base/frameworks/cluster/main.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
 rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
@@ -31,13 +86,24 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
 rest_target(${psd} base/frameworks/communication/main.bro)
 rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
+rest_target(${psd} base/frameworks/files/main.bro)
+rest_target(${psd} base/frameworks/input/main.bro)
+rest_target(${psd} base/frameworks/input/readers/ascii.bro)
+rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
+rest_target(${psd} base/frameworks/input/readers/binary.bro)
+rest_target(${psd} base/frameworks/input/readers/raw.bro)
+rest_target(${psd} base/frameworks/input/readers/sqlite.bro)
+rest_target(${psd} base/frameworks/intel/cluster.bro)
+rest_target(${psd} base/frameworks/intel/input.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
+rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
-rest_target(${psd} base/frameworks/metrics/cluster.bro)
+rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
-rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
-rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
+rest_target(${psd} base/frameworks/logging/writers/none.bro)
+rest_target(${psd} base/frameworks/logging/writers/sqlite.bro)
 rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
 rest_target(${psd} base/frameworks/notice/actions/drop.bro)
 rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
@@ -46,72 +112,128 @@ rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
 rest_target(${psd} base/frameworks/notice/cluster.bro)
 rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
 rest_target(${psd} base/frameworks/notice/main.bro)
+rest_target(${psd} base/frameworks/notice/non-cluster.bro)
 rest_target(${psd} base/frameworks/notice/weird.bro)
 rest_target(${psd} base/frameworks/packet-filter/main.bro)
 rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
+rest_target(${psd} base/frameworks/packet-filter/utils.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/sumstats/cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/main.bro)
+rest_target(${psd} base/frameworks/sumstats/non-cluster.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/average.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/last.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/max.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/min.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sample.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/std-dev.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/sum.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/topk.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/unique.bro)
+rest_target(${psd} base/frameworks/sumstats/plugins/variance.bro)
+rest_target(${psd} base/frameworks/tunnels/main.bro)
+rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
+rest_target(${psd} base/protocols/conn/polling.bro)
+rest_target(${psd} base/protocols/dhcp/consts.bro)
+rest_target(${psd} base/protocols/dhcp/main.bro)
+rest_target(${psd} base/protocols/dhcp/utils.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
-rest_target(${psd} base/protocols/ftp/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/files.bro)
+rest_target(${psd} base/protocols/ftp/gridftp.bro)
+rest_target(${psd} base/protocols/ftp/info.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
-rest_target(${psd} base/protocols/http/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/utils.bro)
-rest_target(${psd} base/protocols/http/file-hash.bro)
+rest_target(${psd} base/protocols/http/entities.bro)
-rest_target(${psd} base/protocols/http/file-ident.bro)
+rest_target(${psd} base/protocols/http/files.bro)
 rest_target(${psd} base/protocols/http/main.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
+rest_target(${psd} base/protocols/irc/files.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
-rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
+rest_target(${psd} base/protocols/modbus/consts.bro)
+rest_target(${psd} base/protocols/modbus/main.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
+rest_target(${psd} base/protocols/smtp/files.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
+rest_target(${psd} base/protocols/socks/consts.bro)
+rest_target(${psd} base/protocols/socks/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
 rest_target(${psd} base/protocols/ssl/main.bro)
 rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
 rest_target(${psd} base/protocols/syslog/consts.bro)
 rest_target(${psd} base/protocols/syslog/main.bro)
+rest_target(${psd} base/utils/active-http.bro)
 rest_target(${psd} base/utils/addrs.bro)
 rest_target(${psd} base/utils/conn-ids.bro)
+rest_target(${psd} base/utils/dir.bro)
 rest_target(${psd} base/utils/directions-and-hosts.bro)
+rest_target(${psd} base/utils/exec.bro)
 rest_target(${psd} base/utils/files.bro)
 rest_target(${psd} base/utils/numbers.bro)
 rest_target(${psd} base/utils/paths.bro)
 rest_target(${psd} base/utils/patterns.bro)
+rest_target(${psd} base/utils/queue.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} base/utils/time.bro)
+rest_target(${psd} base/utils/urls.bro)
 rest_target(${psd} policy/frameworks/communication/listen.bro)
 rest_target(${psd} policy/frameworks/control/controllee.bro)
 rest_target(${psd} policy/frameworks/control/controller.bro)
 rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
 rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
-rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
+rest_target(${psd} policy/frameworks/files/detect-MHR.bro)
-rest_target(${psd} policy/frameworks/metrics/http-example.bro)
+rest_target(${psd} policy/frameworks/files/hash-all-files.bro)
-rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
+rest_target(${psd} policy/frameworks/intel/do_notice.bro)
+rest_target(${psd} policy/frameworks/intel/seen/conn-established.bro)
+rest_target(${psd} policy/frameworks/intel/seen/dns.bro)
+rest_target(${psd} policy/frameworks/intel/seen/http-host-header.bro)
+rest_target(${psd} policy/frameworks/intel/seen/http-url.bro)
+rest_target(${psd} policy/frameworks/intel/seen/http-user-agents.bro)
+rest_target(${psd} policy/frameworks/intel/seen/smtp-url-extraction.bro)
+rest_target(${psd} policy/frameworks/intel/seen/smtp.bro)
+rest_target(${psd} policy/frameworks/intel/seen/ssl.bro)
+rest_target(${psd} policy/frameworks/intel/seen/where-locations.bro)
+rest_target(${psd} policy/frameworks/packet-filter/shunt.bro)
 rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
-rest_target(${psd} policy/misc/analysis-groups.bro)
+rest_target(${psd} policy/integration/collective-intel/main.bro)
+rest_target(${psd} policy/misc/app-stats/main.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/facebook.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/gmail.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/google.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/netflix.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/pandora.bro)
+rest_target(${psd} policy/misc/app-stats/plugins/youtube.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
+rest_target(${psd} policy/misc/detect-traceroute/main.bro)
+rest_target(${psd} policy/misc/known-devices.bro)
+rest_target(${psd} policy/misc/load-balancing.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
 rest_target(${psd} policy/misc/profiling.bro)
+rest_target(${psd} policy/misc/scan.bro)
+rest_target(${psd} policy/misc/stats.bro)
 rest_target(${psd} policy/misc/trim-trace-file.bro)
 rest_target(${psd} policy/protocols/conn/known-hosts.bro)
 rest_target(${psd} policy/protocols/conn/known-services.bro)
 rest_target(${psd} policy/protocols/conn/weirds.bro)
+rest_target(${psd} policy/protocols/dhcp/known-devices-and-hostnames.bro)
 rest_target(${psd} policy/protocols/dns/auth-addl.bro)
 rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
+rest_target(${psd} policy/protocols/ftp/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
-rest_target(${psd} policy/protocols/http/detect-MHR.bro)
-rest_target(${psd} policy/protocols/http/detect-intel.bro)
 rest_target(${psd} policy/protocols/http/detect-sqli.bro)
 rest_target(${psd} policy/protocols/http/detect-webapps.bro)
 rest_target(${psd} policy/protocols/http/header-names.bro)
@@ -119,8 +241,11 @@ rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
+rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
+rest_target(${psd} policy/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
 rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
 rest_target(${psd} policy/protocols/ssh/geo-data.bro)
@@ -130,9 +255,11 @@ rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
 rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
 rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
 rest_target(${psd} policy/protocols/ssl/known-certs.bro)
+rest_target(${psd} policy/protocols/ssl/notary.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
+rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
 rest_target(${psd} site/local-manager.bro)
 rest_target(${psd} site/local-proxy.bro)

doc/scripts/bifs.rst

@@ -1,5 +0,0 @@
-.. This is a stub doc to which broxygen appends during the build process
-
-Built-In Functions (BIFs)
-=========================
-

doc/scripts/builtins.rst

@@ -6,113 +6,824 @@ Types
 
 The Bro scripting language supports the following built-in types.
 
-.. TODO: add documentation
-
 .. bro:type:: void
 
+    An internal Bro type representing an absence of a type.  Should
+    most often be seen as a possible function return type.
+
 .. bro:type:: bool
 
+    Reflects a value with one of two meanings: true or false.  The two
+    ``bool`` constants are ``T`` and ``F``.
+
 .. bro:type:: int
 
+    A numeric type representing a signed integer.  An ``int`` constant
+    is a string of digits preceded by a ``+`` or ``-`` sign, e.g.
+    ``-42`` or ``+5``.  When using type inferencing use care so that the
+    intended type is inferred, e.g. ``local size_difference = 0`` will
+    infer :bro:type:`count`, while ``local size_difference = +0``
+    will infer :bro:type:`int`.
+
 .. bro:type:: count
 
+    A numeric type representing an unsigned integer.  A ``count``
+    constant is a string of digits, e.g. ``1234`` or ``0``.
+
 .. bro:type:: counter
 
+    An alias to :bro:type:`count`.
+
+.. TODO: is there anything special about this type?
+
 .. bro:type:: double
 
+    A numeric type representing a double-precision floating-point
+    number.  Floating-point constants are written as a string of digits
+    with an optional decimal point, optional scale-factor in scientific
+    notation, and optional ``+`` or ``-`` sign.  Examples are ``-1234``,
+    ``-1234e0``, ``3.14159``, and ``.003e-23``.
+
 .. bro:type:: time
 
+    A temporal type representing an absolute time.  There is currently
+    no way to specify a ``time`` constant, but one can use the
+    :bro:id:`current_time` or :bro:id:`network_time` built-in functions
+    to assign a value to a ``time``-typed variable.
+
 .. bro:type:: interval
 
+    A temporal type representing a relative time.  An ``interval``
+    constant can be written as a numeric constant followed by a time
+    unit where the time unit is one of ``usec``, ``msec``, ``sec``, ``min``,
+    ``hr``, or ``day`` which respectively represent microseconds, milliseconds,
+    seconds, minutes, hours, and days.  Whitespace between the numeric
+    constant and time unit is optional.  Appending the letter "s" to the
+    time unit in order to pluralize it is also optional (to no semantic
+    effect).  Examples of ``interval`` constants are ``3.5 min`` and
+    ``3.5mins``.  An ``interval`` can also be negated, for example ``-
+    12 hr`` represents "twelve hours in the past".  Intervals also
+    support addition, subtraction, multiplication, division, and
+    comparison operations.
+
 .. bro:type:: string
 
+    A type used to hold character-string values which represent text.
+    String constants are created by enclosing text in double quotes (")
+    and the backslash character (\\) introduces escape sequences.
+
+    Note that Bro represents strings internally as a count and vector of
+    bytes rather than a NUL-terminated byte string (although string
+    constants are also automatically NUL-terminated).  This is because
+    network traffic can easily introduce NULs into strings either by
+    nature of an application, inadvertently, or maliciously.  And while
+    NULs are allowed in Bro strings, when present in strings passed as
+    arguments to many functions, a run-time error can occur as their
+    presence likely indicates a sort of problem.  In that case, the
+    string will also only be represented to the user as the literal
+    "<string-with-NUL>" string.
+
 .. bro:type:: pattern
 
+    A type representing regular-expression patterns which can be used
+    for fast text-searching operations.  Pattern constants are created
+    by enclosing text within forward slashes (/) and is the same syntax
+    as the patterns supported by the `flex lexical analyzer
+    <http://flex.sourceforge.net/manual/Patterns.html>`_.  The speed of
+    regular expression matching does not depend on the complexity or
+    size of the patterns.  Patterns support two types of matching, exact
+    and embedded.
+
+    In exact matching the ``==`` equality relational operator is used
+    with one :bro:type:`pattern` operand and one :bro:type:`string`
+    operand (order of operands does not matter) to check whether the full
+    string exactly matches the pattern.  In exact matching, the ``^``
+    beginning-of-line and ``$`` end-of-line anchors are redundant since
+    the pattern is implicitly anchored to the beginning and end of the
+    line to facilitate an exact match.  For example::
+
+        /foo|bar/ == "foo"
+
+    yields true, while::
+
+        /foo|bar/ == "foobar"
+
+    yields false.  The ``!=`` operator would yield the negation of ``==``.
+
+    In embedded matching the ``in`` operator is used with one
+    :bro:type:`pattern` operand (which must be on the left-hand side) and
+    one :bro:type:`string` operand, but tests whether the pattern
+    appears anywhere within the given string.  For example::
+
+        /foo|bar/ in "foobar"
+
+    yields true, while::
+
+        /^oob/ in "foobar"
+
+    is false since "oob" does not appear at the start of "foobar".  The
+    ``!in`` operator would yield the negation of ``in``.
+
 .. bro:type:: enum
 
+    A type allowing the specification of a set of related values that
+    have no further structure.  The only operations allowed on
+    enumerations are equality comparisons and they do not have
+    associated values or ordering.  An example declaration:
+
+    .. code:: bro
+
+        type color: enum { Red, White, Blue, };
+
+    The last comma after ``Blue`` is optional.
+
 .. bro:type:: timer
 
+.. TODO: is this a type that's exposed to users?
+
 .. bro:type:: port
 
+    A type representing transport-level port numbers.  Besides TCP and
+    UDP ports, there is a concept of an ICMP "port" where the source
+    port is the ICMP message type and the destination port the ICMP
+    message code.  A ``port`` constant is written as an unsigned integer
+    followed by one of ``/tcp``, ``/udp``, ``/icmp``, or ``/unknown``.
+
+    Ports can be compared for equality and also for ordering.  When
+    comparing order across transport-level protocols, ``unknown`` <
+    ``tcp`` < ``udp`` < ``icmp``, for example ``65535/tcp`` is smaller
+    than ``0/udp``.
+
 .. bro:type:: addr
 
-.. bro:type:: net
+    A type representing an IP address.
+
+    IPv4 address constants are written in "dotted quad" format,
+    ``A1.A2.A3.A4``, where Ai all lie between 0 and 255.
+
+    IPv6 address constants are written as colon-separated hexadecimal form
+    as described by :rfc:`2373`, but additionally encased in square brackets.
+    The mixed notation with embedded IPv4 addresses as dotted-quads in the
+    lower 32 bits is also allowed.
+    Some examples: ``[2001:db8::1]``, ``[::ffff:192.168.1.100]``, or
+    ``[aaaa:bbbb:cccc:dddd:eeee:ffff:1111:2222]``.
+
+    Hostname constants can also be used, but since a hostname can
+    correspond to multiple IP addresses, the type of such variable is a
+    :bro:type:`set` of :bro:type:`addr` elements. For example:
+
+    .. code:: bro
+
+        local a = www.google.com;
+
+    Addresses can be compared for (in)equality using ``==`` and ``!=``.
+    They can also be masked with ``/`` to produce a :bro:type:`subnet`:
+
+    .. code:: bro
+
+        local a: addr = 192.168.1.100;
+        local s: subnet = 192.168.0.0/16;
+        if ( a/16 == s )
+            print "true";
+
+    And checked for inclusion within a :bro:type:`subnet` using ``in`` :
+
+    .. code:: bro
+
+        local a: addr = 192.168.1.100;
+        local s: subnet = 192.168.0.0/16;
+        if ( a in s )
+            print "true";
 
 .. bro:type:: subnet
 
+    A type representing a block of IP addresses in CIDR notation.  A
+    ``subnet`` constant is written as an :bro:type:`addr` followed by a
+    slash (/) and then the network prefix size specified as a decimal
+    number.  For example, ``192.168.0.0/16`` or ``[fe80::]/64``.
+
 .. bro:type:: any
 
+    Used to bypass strong typing.  For example, a function can take an
+    argument of type ``any`` when it may be of different types.
+
 .. bro:type:: table
 
-.. bro:type:: union
+    An associate array that maps from one set of values to another.  The
+    values being mapped are termed the *index* or *indices* and the
+    result of the mapping is called the *yield*.  Indexing into tables
+    is very efficient, and internally it is just a single hash table
+    lookup.
 
-.. bro:type:: record
+    The table declaration syntax is::
 
-.. bro:type:: types
+        table [ type^+ ] of type
 
-.. bro:type:: func
+    where *type^+* is one or more types, separated by commas.  For example:
 
-.. bro:type:: file
+    .. code:: bro
 
-.. bro:type:: vector
+        global a: table[count] of string;
 
-.. TODO: below are kind of "special cases" that bro knows about?
+    declares a table indexed by :bro:type:`count` values and yielding
+    :bro:type:`string` values.  The yield type can also be more complex:
+
+    .. code:: bro
+
+        global a: table[count] of table[addr, port] of string;
+
+    which declares a table indexed by :bro:type:`count` and yielding
+    another :bro:type:`table` which is indexed by an :bro:type:`addr`
+    and :bro:type:`port` to yield a :bro:type:`string`.
+
+    Initialization of tables occurs by enclosing a set of initializers within
+    braces, for example:
+
+    .. code:: bro
+
+        global t: table[count] of string = {
+            [11] = "eleven",
+            [5] = "five",
+        };
+
+    A table constructor (equivalent to above example) can also be used
+    to create a table:
+
+    .. code:: bro
+
+        global t2: table[count] of string = table(
+            [11] = "eleven",
+            [5] = "five"
+        );
+
+    Table constructors can also be explicitly named by a type, which is
+    useful for when a more complex index type could otherwise be
+    ambiguous:
+
+    .. code:: bro
+
+        type MyRec: record {
+            a: count &optional;
+            b: count;
+        };
+
+        type MyTable: table[MyRec] of string;
+
+        global t3 = MyTable([[$b=5]] = "b5", [[$b=7]] = "b7");
+
+    Accessing table elements if provided by enclosing values within square
+    brackets (``[]``), for example:
+
+    .. code:: bro
+
+        t[13] = "thirteen";
+
+    And membership can be tested with ``in``:
+
+    .. code:: bro
+
+        if ( 13 in t )
+            ...
+
+    Iterate over tables with a ``for`` loop:
+
+    .. code:: bro
+
+        local t: table[count] of string;
+        for ( n in t )
+            ...
+
+        local services: table[addr, port] of string;
+        for ( [a, p] in services )
+            ...
+
+    Remove individual table elements with ``delete``:
+
+    .. code:: bro
+
+        delete t[13];
+
+    Nothing happens if the element with value ``13`` isn't present in
+    the table.
+
+    Table size can be obtained by placing the table identifier between
+    vertical pipe (|) characters:
+
+    .. code:: bro
+
+        |t|
 
 .. bro:type:: set
 
+    A set is like a :bro:type:`table`, but it is a collection of indices
+    that do not map to any yield value.  They are declared with the
+    syntax::
+
+        set [ type^+ ]
+
+    where *type^+* is one or more types separated by commas.
+
+    Sets are initialized by listing elements enclosed by curly braces:
+
+    .. code:: bro
+
+        global s: set[port] = { 21/tcp, 23/tcp, 80/tcp, 443/tcp };
+        global s2: set[port, string] = { [21/tcp, "ftp"], [23/tcp, "telnet"] };
+
+    The types are explicitly shown in the example above, but they could
+    have been left to type inference.
+
+    A set constructor (equivalent to above example) can also be used to
+    create a set:
+
+    .. code:: bro
+
+        global s3: set[port] = set(21/tcp, 23/tcp, 80/tcp, 443/tcp);
+
+    Set constructors can also be explicitly named by a type, which is
+    useful for when a more complex index type could otherwise be
+    ambiguous:
+
+    .. code:: bro
+
+        type MyRec: record {
+            a: count &optional;
+            b: count;
+        };
+
+        type MySet: set[MyRec];
+
+        global s4 = MySet([$b=1], [$b=2]);
+
+    Set membership is tested with ``in``:
+
+    .. code:: bro
+
+        if ( 21/tcp in s )
+            ...
+
+    Elements are added with ``add``:
+
+    .. code:: bro
+
+        add s[22/tcp];
+
+    And removed with ``delete``:
+
+    .. code:: bro
+
+        delete s[21/tcp];
+
+    Set size can be obtained by placing the set identifier between
+    vertical pipe (|) characters:
+
+    .. code:: bro
+
+        |s|
+
+.. bro:type:: vector
+
+    A vector is like a :bro:type:`table`, except it's always indexed by a
+    :bro:type:`count`.  A vector is declared like:
+
+    .. code:: bro
+
+        global v: vector of string;
+
+    And can be initialized with the vector constructor:
+
+    .. code:: bro
+
+        global v: vector of string = vector("one", "two", "three");
+
+    Vector constructors can also be explicitly named by a type, which
+    is useful for when a more complex yield type could otherwise be
+    ambiguous.
+
+    .. code:: bro
+
+        type MyRec: record {
+            a: count &optional;
+            b: count;
+        };
+
+        type MyVec: vector of MyRec;
+
+        global v2 = MyVec([$b=1], [$b=2], [$b=3]);
+
+    Adding an element to a vector involves accessing/assigning it:
+
+    .. code:: bro
+
+        v[3] = "four"
+
+    Note how the vector indexing is 0-based.
+
+    Vector size can be obtained by placing the vector identifier between
+    vertical pipe (|) characters:
+
+    .. code:: bro
+
+        |v|
+
+.. bro:type:: record
+
+    A ``record`` is a collection of values.  Each value has a field name
+    and a type.  Values do not need to have the same type and the types
+    have no restrictions.  An example record type definition:
+
+    .. code:: bro
+
+        type MyRecordType: record {
+            c: count;
+            s: string &optional;
+        };
+
+    Access to a record field uses the dollar sign (``$``) operator:
+
+    .. code:: bro
+
+        global r: MyRecordType;
+        r$c = 13;
+
+    Record assignment can be done field by field or as a whole like:
+
+    .. code:: bro
+
+        r = [$c = 13, $s = "thirteen"];
+
+    When assigning a whole record value, all fields that are not
+    :bro:attr:`&optional` or have a :bro:attr:`&default` attribute must
+    be specified.
+
+    To test for existence of a field that is :bro:attr:`&optional`, use the
+    ``?$`` operator:
+
+    .. code:: bro
+
+        if ( r?$s )
+            ...
+
+    Records can also be created using a constructor syntax:
+
+    .. code:: bro
+
+        global r2: MyRecordType = record($c = 7);
+
+    And the constructor can be explicitly named by type, too, which
+    is arguably more readable code:
+
+    .. code:: bro
+
+        global r3 = MyRecordType($c = 42);
+
+.. bro:type:: opaque
+
+    A data type whose actual representation/implementation is
+    intentionally hidden, but whose values may be passed to certain
+    functions that can actually access the internal/hidden resources.
+    Opaque types are differentiated from each other by qualifying them
+    like ``opaque of md5`` or ``opaque of sha1``.  Any valid identifier
+    can be used as the type qualifier.
+
+    An example use of this type is the set of built-in functions which
+    perform hashing:
+
+    .. code:: bro
+
+        local handle: opaque of md5 = md5_hash_init();
+        md5_hash_update(handle, "test");
+        md5_hash_update(handle, "testing");
+        print md5_hash_finish(handle);
+
+    Here the opaque type is used to provide a handle to a particular
+    resource which is calculating an MD5 checksum incrementally over
+    time, but the details of that resource aren't relevant, it's only
+    necessary to have a handle as a way of identifying it and
+    distinguishing it from other such resources.
+
+.. bro:type:: file
+
+    Bro supports writing to files, but not reading from them.  For
+    example, declare, open, and write to a file and finally close it
+    like:
+
+    .. code:: bro
+
+        global f: file = open("myfile");
+        print f, "hello, world";
+        close(f);
+
+    Writing to files like this for logging usually isn't recommended, for better
+    logging support see :doc:`/logging`.
+
 .. bro:type:: function
 
+    Function types in Bro are declared using::
+
+        function( argument*  ): type
+
+    where *argument* is a (possibly empty) comma-separated list of
+    arguments, and *type* is an optional return type.  For example:
+
+    .. code:: bro
+
+        global greeting: function(name: string): string;
+
+    Here ``greeting`` is an identifier with a certain function type.
+    The function body is not defined yet and ``greeting`` could even
+    have different function body values at different times.  To define
+    a function including a body value, the syntax is like:
+
+    .. code:: bro
+
+        function greeting(name: string): string
+            {
+            return "Hello, " + name;
+            }
+
+    Note that in the definition above, it's not necessary for us to have
+    done the first (forward) declaration of ``greeting`` as a function
+    type, but when it is, the argument list and return type much match
+    exactly.
+
+    Function types don't need to have a name and can be assigned anonymously:
+
+    .. code:: bro
+
+        greeting = function(name: string): string { return "Hi, " + name; };
+
+    And finally, the function can be called like:
+
+    .. code:: bro
+
+        print greeting("Dave");
+
+    Function parameters may specify default values as long as they appear
+    last in the parameter list:
+
+    .. code:: bro
+
+        global foo: function(s: string, t: string &default="abc", u: count &default=0);
+
+    If a function was previously declared with default parameters, the
+    default expressions can be omitted when implementing the function
+    body and they will still be used for function calls that lack those
+    arguments.
+
+    .. code:: bro
+
+        function foo(s: string, t: string, u: count)
+            {
+            print s, t, u;
+            }
+
+    And calls to the function may omit the defaults from the argument list:
+
+    .. code:: bro
+
+        foo("test");
+
 .. bro:type:: event
 
+    Event handlers are nearly identical in both syntax and semantics to
+    a :bro:type:`function`, with the two differences being that event
+    handlers have no return type since they never return a value, and
+    you cannot call an event handler.  Instead of directly calling an
+    event handler from a script, event handler bodies are executed when
+    they are invoked by one of three different methods:
+
+    - From the event engine
+
+        When the event engine detects an event for which you have
+        defined a corresponding event handler, it queues an event for
+        that handler.  The handler is invoked as soon as the event
+        engine finishes processing the current packet and flushing the
+        invocation of other event handlers that were queued first.
+
+    - With the ``event`` statement from a script
+
+        Immediately queuing invocation of an event handler occurs like:
+
+        .. code:: bro
+
+            event password_exposed(user, password);
+
+        This assumes that ``password_exposed`` was previously declared
+        as an event handler type with compatible arguments.
+
+    - Via the ``schedule`` expression in a script
+
+        This delays the invocation of event handlers until some time in
+        the future.  For example:
+
+        .. code:: bro
+
+            schedule 5 secs { password_exposed(user, password) };
+
+    Multiple event handler bodies can be defined for the same event handler
+    identifier and the body of each will be executed in turn.  Ordering
+    of execution can be influenced with :bro:attr:`&priority`.
+
+.. bro:type:: hook
+
+    A hook is another flavor of function that shares characteristics of
+    both a :bro:type:`function` and a :bro:type:`event`.  They are like
+    events in that many handler bodies can be defined for the same hook
+    identifier and the order of execution can be enforced with
+    :bro:attr:`&priority`.  They are more like functions in the way they
+    are invoked/called, because, unlike events, their execution is
+    immediate and they do not get scheduled through an event queue.
+    Also, a unique feature of a hook is that a given hook handler body
+    can short-circuit the execution of remaining hook handlers simply by
+    exiting from the body as a result of a ``break`` statement (as
+    opposed to a ``return`` or just reaching the end of the body).
+
+    A hook type is declared like::
+
+        hook( argument* )
+
+    where *argument* is a (possibly empty) comma-separated list of
+    arguments.  For example:
+
+    .. code:: bro
+
+        global myhook: hook(s: string)
+
+    Here ``myhook`` is the hook type identifier and no hook handler
+    bodies have been defined for it yet.  To define some hook handler
+    bodies the syntax looks like:
+
+    .. code:: bro
+
+        hook myhook(s: string) &priority=10
+            {
+            print "priority 10 myhook handler", s;
+            s = "bye";
+            }
+
+        hook myhook(s: string)
+            {
+            print "break out of myhook handling", s;
+            break;
+            }
+
+        hook myhook(s: string) &priority=-5
+            {
+            print "not going to happen", s;
+            }
+
+    Note that the first (forward) declaration of ``myhook`` as a hook
+    type isn't strictly required.  Argument types must match for all
+    hook handlers and any forward declaration of a given hook.
+
+    To invoke immediate execution of all hook handler bodies, they
+    are called similarly to a function, except preceded by the ``hook``
+    keyword:
+
+    .. code:: bro
+
+        hook myhook("hi");
+
+    or
+
+    .. code:: bro
+
+        if ( hook myhook("hi") )
+            print "all handlers ran";
+
+    And the output would look like::
+
+        priority 10 myhook handler, hi
+        break out of myhook handling, bye
+
+    Note how the modification to arguments can be seen by remaining
+    hook handlers.
+
+    The return value of a hook call is an implicit :bro:type:`bool`
+    value with ``T`` meaning that all handlers for the hook were
+    executed and ``F`` meaning that only some of the handlers may have
+    executed due to one handler body exiting as a result of a ``break``
+    statement.
+
 Attributes
 ----------
 
-The Bro scripting language supports the following built-in attributes.
+Attributes occur at the end of type/event declarations and change their
-
+behavior. The syntax is ``&key`` or ``&key=val``, e.g., ``type T:
-.. TODO: add documentation
+set[count] &read_expire=5min`` or ``event foo() &priority=-3``.  The Bro
+scripting language supports the following built-in attributes.
 
 .. bro:attr:: &optional
 
+    Allows a record field to be missing. For example the type ``record {
+    a: int, b: port &optional }`` could be instantiated both as
+    singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``.
+
 .. bro:attr:: &default
 
+    Uses a default value for a record field, a function/hook/event
+    parameter, or container elements.  For example, ``table[int] of
+    string &default="foo" }`` would create a table that returns the
+    :bro:type:`string` ``"foo"`` for any non-existing index.
+
 .. bro:attr:: &redef
 
+    Allows for redefinition of initial object values. This is typically
+    used with constants, for example, ``const clever = T &redef;`` would
+    allow the constant to be redefined at some later point during script
+    execution.
+
 .. bro:attr:: &rotate_interval
 
+    Rotates a file after a specified interval.
+
 .. bro:attr:: &rotate_size
 
+    Rotates a file after it has reached a given size in bytes.
+
 .. bro:attr:: &add_func
 
+.. TODO: needs to be documented.
+
 .. bro:attr:: &delete_func
 
+.. TODO: needs to be documented.
+
 .. bro:attr:: &expire_func
 
+    Called right before a container element expires.  The function's
+    first parameter is of the same type of the container and the second
+    parameter the same type of the container's index.  The return
+    value is a :bro:type:`interval` indicating the amount of additional
+    time to wait before expiring the container element at the given
+    index (which will trigger another execution of this function).
+
 .. bro:attr:: &read_expire
 
+    Specifies a read expiration timeout for container elements. That is,
+    the element expires after the given amount of time since the last
+    time it has been read. Note that a write also counts as a read.
+
 .. bro:attr:: &write_expire
 
+    Specifies a write expiration timeout for container elements. That
+    is, the element expires after the given amount of time since the
+    last time it has been written.
+
 .. bro:attr:: &create_expire
 
+    Specifies a creation expiration timeout for container elements. That
+    is, the element expires after the given amount of time since it has
+    been inserted into the container, regardless of any reads or writes.
+
 .. bro:attr:: &persistent
 
+    Makes a variable persistent, i.e., its value is writen to disk (per
+    default at shutdown time).
+
 .. bro:attr:: &synchronized
 
-.. bro:attr:: &postprocessor
+    Synchronizes variable accesses across nodes. The value of a
+    ``&synchronized`` variable is automatically propagated to all peers
+    when it changes.
 
 .. bro:attr:: &encrypt
 
-.. bro:attr:: &match
+    Encrypts files right before writing them to disk.
 
-.. bro:attr:: &disable_print_hook
+.. TODO: needs to be documented in more detail.
 
 .. bro:attr:: &raw_output
 
+    Opens a file in raw mode, i.e., non-ASCII characters are not
+    escaped.
+
 .. bro:attr:: &mergeable
 
+    Prefers set union to assignment for synchronized state. This
+    attribute is used in conjunction with :bro:attr:`&synchronized`
+    container types: when the same container is updated at two peers
+    with different value, the propagation of the state causes a race
+    condition, where the last update succeeds. This can cause
+    inconsistencies and can be avoided by unifying the two sets, rather
+    than merely overwriting the old value.
+
 .. bro:attr:: &priority
 
+    Specifies the execution priority of an event handler. Higher values
+    are executed before lower ones. The default value is 0.
+
 .. bro:attr:: &group
 
+    Groups event handlers such that those in the same group can be
+    jointly activated or deactivated.
+
 .. bro:attr:: &log
 
-.. bro:attr:: (&tracked)
+    Writes a record field to the associated log stream.
+
+.. bro:attr:: &error_handler
+
+.. TODO: needs documented
+

doc/scripts/example.bro

@@ -54,11 +54,11 @@ global example_ports = {
     443/tcp, 562/tcp,
 } &redef;
 
-# redefinitions of "dpd_config" are self-documenting and
+
-# go into the generated doc's "Port Analysis" section
+event bro_init()
-redef dpd_config += {
+	{
-    [ANALYZER_SSL] = [$ports = example_ports]
+	Analyzer::register_for_ports(Analyzer::ANALYZER_SSL, example_ports);
-};
+	}
 
 # redefinitions of "Notice::Type" are self-documenting, but
 # more information can be supplied in two different ways

doc/scripts/genDocSourcesList.sh

@@ -67,12 +67,12 @@ sourcedir=${thisdir}/../..
 
 echo "$statictext" > $outfile
 
-bifs=`( cd ${sourcedir}/src && find . -name \*\.bif | sort )`
+bifs=`( cd ${sourcedir}/build/scripts/base && find . -name \*\.bif.bro | sort )`
 
 for file in $bifs
 do
-    f=${file:2}.bro
+    f=${file:2}
-    echo "rest_target(\${CMAKE_BINARY_DIR}/src base/$f)" >> $outfile
+    echo "rest_target(\${CMAKE_BINARY_DIR}/scripts base/$f)" >> $outfile
 done
 
 scriptfiles=`( cd ${sourcedir}/scripts && find . -name \*\.bro | sort )`

doc/signatures.rst

@@ -34,7 +34,7 @@ Let's look at an example signature first:
 
 This signature asks Bro to match the regular expression ``.*root`` on
 all TCP connections going to port 80. When the signature triggers, Bro
-will raise an event ``signature_match`` of the form:
+will raise an event :bro:id:`signature_match` of the form:
 
 .. code:: bro
 
@@ -45,19 +45,24 @@ triggered the match, ``msg`` is the string specified by the
 signature's event statement (``Found root!``), and data is the last
 piece of payload which triggered the pattern match.
 
-To turn such ``signature_match`` events into actual alarms, you can
+To turn such :bro:id:`signature_match` events into actual alarms, you can
-load Bro's ``signature.bro`` script. This script contains a default
+load Bro's :doc:`/scripts/base/frameworks/signatures/main` script.
-event handler that raises ``SensitiveSignature`` :doc:`Notices <notice>`
+This script contains a default event handler that raises
+:bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
 (as well as others; see the beginning of the script).
 
-As signatures are independent of Bro's policy scripts, they are put
+As signatures are independent of Bro's policy scripts, they are put into
-into their own file(s). There are two ways to specify which files
+their own file(s). There are three ways to specify which files contain
-contain signatures: By using the ``-s`` flag when you invoke Bro, or
+signatures: By using the ``-s`` flag when you invoke Bro, or by
-by extending the Bro variable ``signatures_files`` using the ``+=``
+extending the Bro variable :bro:id:`signature_files` using the ``+=``
-operator. If a signature file is given without a path, it is searched
+operator, or by using the ``@load-sigs`` directive inside a Bro script.
-along the normal ``BROPATH``. The default extension of the file name
+If a signature file is given without a full path, it is searched for
-is ``.sig``, and Bro appends that automatically when neccesary.
+along the normal ``BROPATH``.  Additionally, the ``@load-sigs``
-
+directive can be used to load signature files in a path relative to the
+Bro script in which it's placed, e.g. ``@load-sigs ./mysigs.sig`` will
+expect that signature file in the same directory as the Bro script. The
+default extension of the file name is ``.sig``, and Bro appends that
+automatically when necessary.
 
 Signature language
 ==================
@@ -78,9 +83,8 @@ Header Conditions
 ~~~~~~~~~~~~~~~~~
 
 Header conditions limit the applicability of the signature to a subset
-of traffic that contains matching packet headers. For TCP, this match
+of traffic that contains matching packet headers.  This type of matching
-is performed only for the first packet of a connection. For other
+is performed only for the first packet of a connection.
-protocols, it is done on each individual packet.
 
 There are pre-defined header conditions for some of the most used
 header fields. All of them generally have the format ``<keyword> <cmp>
@@ -90,14 +94,22 @@ one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``; and
 against. The following keywords are defined:
 
 ``src-ip``/``dst-ip <cmp> <address-list>``
-    Source and destination address, repectively. Addresses can be
+    Source and destination address, respectively. Addresses can be given
-    given as IP addresses or CIDR masks.
+    as IPv4 or IPv6 addresses or CIDR masks.  For IPv6 addresses/masks
+    the colon-hexadecimal representation of the address must be enclosed
+    in square brackets (e.g. ``[fe80::1]`` or ``[fe80::0]/16``).
 
-``src-port``/``dst-port`` ``<int-list>``
+``src-port``/``dst-port <cmp> <int-list>``
-    Source and destination port, repectively.
+    Source and destination port, respectively.
 
-``ip-proto tcp|udp|icmp``
+``ip-proto <cmp> tcp|udp|icmp|icmp6|ip|ip6``
-    IP protocol.
+    IPv4 header's Protocol field or the Next Header field of the final
+    IPv6 header (i.e. either Next Header field in the fixed IPv6 header
+    if no extension headers are present or that field from the last
+    extension header in the chain).  Note that the IP-in-IP forms of
+    tunneling are automatically decapsulated by default and signatures
+    apply to only the inner-most packet, so specifying ``ip`` or ``ip6``
+    is a no-op.
 
 For lists of multiple values, they are sequentially compared against
 the corresponding header field. If at least one of the comparisons
@@ -111,30 +123,32 @@ condition can be defined either as
 
     header <proto>[<offset>:<size>] [& <integer>] <cmp> <value-list>
 
-This compares the value found at the given position of the packet
+This compares the value found at the given position of the packet header
-header with a list of values. ``offset`` defines the position of the
+with a list of values. ``offset`` defines the position of the value
-value within the header of the protocol defined by ``proto`` (which
+within the header of the protocol defined by ``proto`` (which can be
-can be ``ip``, ``tcp``, ``udp`` or ``icmp``). ``size`` is either 1, 2,
+``ip``, ``ip6``, ``tcp``, ``udp``, ``icmp`` or ``icmp6``). ``size`` is
-or 4 and specifies the value to have a size of this many bytes. If the
+either 1, 2, or 4 and specifies the value to have a size of this many
-optional ``& <integer>`` is given, the packet's value is first masked
+bytes. If the optional ``& <integer>`` is given, the packet's value is
-with the integer before it is compared to the value-list. ``cmp`` is
+first masked with the integer before it is compared to the value-list.
-one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. ``value-list`` is
+``cmp`` is one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``.
-a list of comma-separated integers similar to those described above.
+``value-list`` is a list of comma-separated integers similar to those
-The integers within the list may be followed by an additional ``/
+described above.  The integers within the list may be followed by an
-mask`` where ``mask`` is a value from 0 to 32. This corresponds to the
+additional ``/ mask`` where ``mask`` is a value from 0 to 32. This
-CIDR notation for netmasks and is translated into a corresponding
+corresponds to the CIDR notation for netmasks and is translated into a
-bitmask applied to the packet's value prior to the comparison (similar
+corresponding bitmask applied to the packet's value prior to the
-to the optional ``& integer``).
+comparison (similar to the optional ``& integer``).  IPv6 address values
+are not allowed in the value-list, though you can still inspect any 1,
+2, or 4 byte section of an IPv6 header using this keyword.
 
-Putting all together, this is an example conditiation that is
+Putting it all together, this is an example condition that is
-equivalent to ``dst- ip == 1.2.3.4/16, 5.6.7.8/24``:
+equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
 
 .. code:: bro-sig
 
     header ip[16:4] == 1.2.3.4/16, 5.6.7.8/24
 
-Internally, the predefined header conditions are in fact just
+Note that the analogous example for IPv6 isn't currently possible since
-short-cuts and mappend into a generic condition.
+4 bytes is the max width of a value that can be compared.
 
 Content Conditions
 ~~~~~~~~~~~~~~~~~~
@@ -143,7 +157,7 @@ Content conditions are defined by regular expressions. We
 differentiate two kinds of content conditions: first, the expression
 may be declared with the ``payload`` statement, in which case it is
 matched against the raw payload of a connection (for reassembled TCP
-streams) or of a each packet (for ICMP, UDP, and non-reassembled TCP).
+streams) or of each packet (for ICMP, UDP, and non-reassembled TCP).
 Second, it may be prefixed with an analyzer-specific label, in which
 case the expression is matched against the data as extracted by the
 corresponding analyzer.
@@ -208,7 +222,7 @@ To define dependencies between signatures, there are two conditions:
 
 ``requires-reverse-signature [!] <id>``
     Similar to ``requires-signature``, but ``id`` has to match for the
-    opposite direction of the same connection, compared the current
+    opposite direction of the same connection, compared to the current
     signature. This allows to model the notion of requests and
     replies.
 
@@ -224,20 +238,10 @@ matched. The following context conditions are defined:
     confirming the match. If false is returned, no signature match is
     going to be triggered. The function has to be of type ``function
     cond(state: signature_state, data: string): bool``. Here,
-    ``content`` may contain the most recent content chunk available at
+    ``data`` may contain the most recent content chunk available at
     the time the signature was matched. If no such chunk is available,
-    ``content`` will be the empty string. ``signature_state`` is
+    ``data`` will be the empty string. See :bro:type:`signature_state`
-    defined as follows:
+    for its definition.
-
-    .. code:: bro
-
-        type signature_state: record {
-            id: string;          # ID of the signature
-            conn: connection;    # Current connection
-            is_orig: bool;       # True if current endpoint is originator
-            payload_size: count; # Payload size of the first packet
-            };
-
 
 ``payload-size <cmp> <integer>``
     Compares the integer to the size of the payload of a packet. For
@@ -265,7 +269,7 @@ Actions define what to do if a signature matches. Currently, there are
 two actions defined:
 
 ``event <string>``
-    Raises a ``signature_match`` event. The event handler has the
+    Raises a :bro:id:`signature_match` event. The event handler has the
     following type:
 
     .. code:: bro
@@ -338,11 +342,11 @@ Things to keep in mind when writing signatures
   signature engine and can be matched with ``\r`` and ``\n``,
   respectively. Generally, Bro follows `flex's regular expression
   syntax
-  <http://www.gnu.org/software/flex/manual/html_chapter/flex_7.html>`_.
+  <http://flex.sourceforge.net/manual/Patterns.html>`_.
-  See the DPD signatures in ``policy/sigs/dpd.bro`` for some examples
+  See the DPD signatures in ``base/frameworks/dpd/dpd.sig`` for some examples
   of fairly complex payload patterns.
 
-* The data argument of the ``signature_match`` handler might not carry
+* The data argument of the :bro:id:`signature_match` handler might not carry
   the full text matched by the regular expression. Bro performs the
   matching incrementally as packets come in; when the signature
   eventually fires, it can only pass on the most recent chunk of data.

doc/upgrade.rst

@@ -1,19 +1,80 @@
 
-=============================
+==========================================
-Upgrading From Bro 1.5 to 2.0
+Upgrading From the Previous Version of Bro
-=============================
+==========================================
 
 .. rst-class:: opening
 
-   This guide details differences between Bro versions 1.5 and 2.0
+   This guide details specific differences between Bro versions
    that may be important for users to know as they work on updating
    their Bro deployment/configuration to the later version.
 
 .. contents::
 
 
-Introduction
+Upgrading From Bro 2.0 to 2.1
-============
+=============================
+
+In Bro 2.1, IPv6 is enabled by default.  Therefore, when building Bro from
+source, the "--enable-brov6" configure option has been removed because it
+is no longer relevant.  
+
+Other configure changes include renaming the "--enable-perftools" option
+to "--enable-perftools-debug" to indicate that the option is only relevant
+for debugging the heap.  One other change involves what happens when
+tcmalloc (part of Google perftools) is found at configure time.  On Linux,
+it will automatically be linked with Bro, but on other platforms you
+need to use the "--enable-perftools" option to enable linking to tcmalloc.
+
+There are a couple of changes to the Bro scripting language to better
+support IPv6. First, IPv6 literals appearing in a Bro script must now be
+enclosed in square brackets (for example, ``[fe80::db15]``). For subnet
+literals, the slash "/" appears after the closing square bracket (for
+example, ``[fe80:1234::]/32``). Second, when an IP address variable or IP
+address literal is enclosed in pipes (for example, ``|[fe80::db15]|``) the
+result is now the size of the address in bits (32 for IPv4 and 128 for IPv6).
+
+In the Bro scripting language, "match" and "using" are no longer reserved
+keywords.
+
+Some built-in functions have been removed: "addr_to_count" (use
+"addr_to_counts" instead), "bro_has_ipv6" (this is no longer relevant
+because Bro now always supports IPv6), "active_connection" (use
+"connection_exists" instead), and "connection_record" (use "lookup_connection"
+instead).
+
+The "NFS3::mode2string" built-in function has been renamed to "file_mode".
+
+Some built-in functions have been changed:  "exit" (now takes the exit code
+as a parameter), "to_port" (now takes a string as parameter instead
+of a count and transport protocol, but "count_to_port" is still available),
+"connect" (now takes an additional string parameter specifying the zone of
+a non-global IPv6 address), and "listen" (now takes three additional
+parameters to enable listening on IPv6 addresses).
+
+Some Bro script variables have been renamed:  "LogAscii::header_prefix"
+has been renamed to "LogAscii::meta_prefix", "LogAscii::include_header"
+has been renamed to "LogAscii::include_meta".
+
+Some Bro script variables have been removed: "tunnel_port",
+"parse_udp_tunnels", "use_connection_compressor", "cc_handle_resets",
+"cc_handle_only_syns", and "cc_instantiate_on_data".
+
+A couple events have changed:  the "icmp_redirect" event now includes
+the target and destination addresses and any Neighbor Discovery options
+in the message, and the last parameter of the "dns_AAAA_reply" event has
+been removed because it was unused.
+
+The format of the ASCII log files has changed very slightly.  Two new lines
+are automatically added, one to record the time when the log was opened,
+and the other to record the time when the log was closed.
+
+In BroControl, the option (in broctl.cfg) "CFlowAddr" was renamed
+to "CFlowAddress".
+
+
+Upgrading From Bro 1.5 to 2.0
+=============================
 
 As the version number jump suggests, Bro 2.0 is a major upgrade and
 lots of things have changed. Most importantly, we have rewritten
@@ -55,13 +116,13 @@ renamed to ``scripts/`` and contains major subdirectories ``base/``,
 further.
 
 The contents of the new ``scripts/`` directory, like the old/flat
-``policy/`` still gets installed under under the ``share/bro``
+``policy/`` still gets installed under the ``share/bro``
 subdirectory of the installation prefix path just like previous
 versions.  For example, if Bro was compiled like ``./configure
 --prefix=/usr/local/bro && make && make install``, then the script
 hierarchy can be found in ``/usr/local/bro/share/bro``.
 
-THe main
+The main
 subdirectories of that hierarchy are as follows:
 
 - ``base/`` contains all scripts that are loaded by Bro by default
@@ -132,7 +193,7 @@ Logging Framework
 
 - The new logging framework makes it possible to extend, customize,
   and filter logs very easily. See the :doc:`logging framework <logging>`
-  more information on usage.
+  for more information on usage.
 
 - A common pattern found in the new scripts is to store logging stream
   records for protocols inside the ``connection`` records so that
@@ -193,7 +254,7 @@ Variable Naming
 
 - Identifiers may have been renamed to conform to new `scripting
   conventions
-  <http://www.bro-ids.org/development/script-conventions.html>`_
+  <http://www.bro.org/development/script-conventions.html>`_
 
 
 BroControl
@@ -209,8 +270,8 @@ live analysis.
 BroControl now has an extensive plugin interface for adding new
 commands and options. Note that this is still considered experimental.
 
-We have remove the ``analysis`` command, and BroControl does currently
+We have removed the ``analysis`` command, and BroControl currently
-not not send daily alarm summaries anymore (this may be restored
+does not send daily alarm summaries anymore (this may be restored
 later).
 
 Removed Functionality
@@ -233,11 +294,11 @@ Development Infrastructure
 ==========================
 
 Bro development has moved from using SVN to Git for revision control.
-Users that like to use the latest Bro developments by checking it out
+Users that want to use the latest Bro development snapshot by checking it out
 from the source repositories should see the `development process
-<http://www.bro-ids.org/development/process.html>`_. Note that all the various
+<http://www.bro.org/development/process.html>`_. Note that all the various
-sub-components now reside on their own repositories. However, the
+sub-components now reside in their own repositories. However, the
-top-level Bro repository includes them as git submodules so it's easu
+top-level Bro repository includes them as git submodules so it's easy
 to check them all out simultaneously.
 
 Bro now uses `CMake <http://www.cmake.org>`_ for its build system so

magic

@@ -0,0 +1 @@
+Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed

pkg/make-mac-packages

@@ -3,7 +3,13 @@
 # This script creates binary packages for Mac OS X.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
+cmake -P /dev/stdin << "EOF"
+if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
+    message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
+endif ()
+EOF
+
+[ $? -ne 0 ] && exit 1;
 
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
@@ -34,26 +40,26 @@ prefix=/opt/bro
 cd ..
 
 # Minimum Bro
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \
     --binary-package
 ( cd build && make package )
 
 # Full Bro package
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
 cd aux/broccoli
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..
 
 # Broctl
 cd aux/broctl
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..

scripts/base/files/extract/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/extract/main.bro

@@ -0,0 +1,38 @@
+@load base/frameworks/files
+@load base/utils/paths
+
+module FileExtract;
+
+export {
+	## The prefix where files are extracted to.
+	const prefix = "./extract_files/" &redef;
+
+	redef record Files::Info += {
+		## Local filenames of extracted file.
+		extracted: string &optional &log;
+	};
+
+	redef record Files::AnalyzerArgs += {
+		## The local filename to which to write an extracted file.
+		## This field is used in the core by the extraction plugin
+		## to know where to write the file to.  It's also optional
+		extract_filename: string &optional;
+	};
+}
+
+function on_add(f: fa_file, args: Files::AnalyzerArgs)
+	{
+	if ( ! args?$extract_filename )
+		args$extract_filename = cat("extract-", f$source, "-", f$id);
+
+	f$info$extracted = args$extract_filename;
+	args$extract_filename = build_path_compressed(prefix, args$extract_filename);
+	}
+
+event bro_init() &priority=10
+	{
+	Files::register_analyzer_add_callback(Files::ANALYZER_EXTRACT, on_add);
+
+	# Create the extraction directory.
+	mkdir(prefix);
+	}

scripts/base/files/hash/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/files/hash/main.bro

@@ -0,0 +1,32 @@
+@load base/frameworks/files
+
+module FileHash;
+
+export {
+	redef record Files::Info += {
+		## An MD5 digest of the file contents.
+		md5: string &log &optional;
+
+		## A SHA1 digest of the file contents.
+		sha1: string &log &optional;
+
+		## A SHA256 digest of the file contents.
+		sha256: string &log &optional;
+	};
+
+}
+
+event file_hash(f: fa_file, kind: string, hash: string) &priority=5
+	{
+	switch ( kind ) {
+	case "md5":
+		f$info$md5 = hash;
+		break;
+	case "sha1":
+		f$info$sha1 = hash;
+		break;
+	case "sha256":
+		f$info$sha256 = hash;
+		break;
+	}
+	}

scripts/base/frameworks/analyzer/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/analyzer/main.bro

@@ -0,0 +1,229 @@
+##! Framework for managing Bro's protocol analyzers.
+##!
+##! The analyzer framework allows to dynamically enable or disable analyzers, as
+##! well as to manage the well-known ports which automatically activate a
+##! particular analyzer for new connections.
+##!
+##! Protocol analyzers are identified by unique tags of type
+##! :bro:type:`Analyzer::Tag`, such as :bro:enum:`Analyzer::ANALYZER_HTTP` and
+##! :bro:enum:`Analyzer::ANALYZER_HTTP`. These tags are defined internally by
+##! the analyzers themselves, and documented in their analyzer-specific
+##! description along with the events that they generate.
+
+@load base/frameworks/packet-filter/utils
+
+module Analyzer;
+
+export {
+	## If true, all available analyzers are initially disabled at startup. One
+	## can then selectively enable them with
+	## :bro:id:`Analyzer::enable_analyzer`.
+	global disable_all = F &redef;
+
+	## Enables an analyzer. Once enabled, the analyzer may be used for analysis
+	## of future connections as decided by Bro's dynamic protocol detection.
+	##
+	## tag: The tag of the analyzer to enable.
+	##
+	## Returns: True if the analyzer was successfully enabled.
+	global enable_analyzer: function(tag: Analyzer::Tag) : bool;
+
+	## Disables an analyzer. Once disabled, the analyzer will not be used
+	## further for analysis of future connections.
+	##
+	## tag: The tag of the analyzer to disable.
+	##
+	## Returns: True if the analyzer was successfully disabled.
+	global disable_analyzer: function(tag: Analyzer::Tag) : bool;
+
+	## Registers a set of well-known ports for an analyzer. If a future
+	## connection on one of these ports is seen, the analyzer will be
+	## automatically assigned to parsing it. The function *adds* to all ports
+	## already registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## ports: The set of well-known ports to associate with the analyzer.
+	##
+	## Returns: True if the ports were sucessfully registered.
+	global register_for_ports: function(tag: Analyzer::Tag, ports: set[port]) : bool;
+
+	## Registers an individual well-known port for an analyzer. If a future
+	## connection on this port is seen, the analyzer will be automatically
+	## assigned to parsing it. The function *adds* to all ports already
+	## registered, it doesn't replace them.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## p: The well-known port to associate with the analyzer.
+	##
+	## Returns: True if the port was sucessfully registered.
+	global register_for_port: function(tag: Analyzer::Tag, p: port) : bool;
+
+	## Returns a set of all well-known ports currently registered for a
+	## specific analyzer.
+	##
+	## tag: The tag of the analyzer.
+	##
+	## Returns: The set of ports.
+	global registered_ports: function(tag: Analyzer::Tag) : set[port];
+
+	## Returns a table of all ports-to-analyzer mappings currently registered.
+	##
+	## Returns: A table mapping each analyzer to the set of ports
+	##          registered for it.
+	global all_registered_ports: function() : table[Analyzer::Tag] of set[port];
+
+	## Translates an analyzer type to a string with the analyzer's name.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: The analyzer name corresponding to the tag.
+	global name: function(tag: Analyzer::Tag) : string;
+
+	## Translates an analyzer's name to a tag enum value.
+	##
+	## name: The analyzer name.
+	##
+	## Returns: The analyzer tag corresponding to the name.
+	global get_tag: function(name: string): Analyzer::Tag;
+
+	## Schedules an analyzer for a future connection originating from a given IP
+	## address and port.
+	##
+	## orig: The IP address originating a connection in the future.
+	##       0.0.0.0 can be used as a wildcard to match any originator address.
+	##
+	## resp: The IP address responding to a connection from *orig*.
+	##
+	## resp_p: The destination port at *resp*.
+	##
+	## analyzer: The analyzer ID.
+	##
+	## tout: A timeout interval after which the scheduling request will be
+	##       discarded if the connection has not yet been seen.
+	##
+	## Returns: True if succesful.
+	global schedule_analyzer: function(orig: addr, resp: addr, resp_p: port,
+	                                   analyzer: Analyzer::Tag, tout: interval) : bool;
+
+	## Automatically creates a BPF filter for the specified protocol based
+	## on the data supplied for the protocol through the
+	## :bro:see:`Analyzer::register_for_ports` function.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: BPF filter string.
+	global analyzer_to_bpf: function(tag: Analyzer::Tag): string;
+
+	## Create a BPF filter which matches all of the ports defined
+	## by the various protocol analysis scripts as "registered ports"
+	## for the protocol.
+	global get_bpf: function(): string;
+
+	## A set of analyzers to disable by default at startup. The default set
+	## contains legacy analyzers that are no longer supported.
+	global disabled_analyzers: set[Analyzer::Tag] = {
+		ANALYZER_INTERCONN,
+		ANALYZER_STEPPINGSTONE,
+		ANALYZER_BACKDOOR,
+		ANALYZER_TCPSTATS,
+	} &redef;
+}
+
+@load base/bif/analyzer.bif
+
+global ports: table[Analyzer::Tag] of set[port];
+
+event bro_init() &priority=5
+	{
+	if ( disable_all )
+		__disable_all_analyzers();
+
+	for ( a in disabled_analyzers )
+		disable_analyzer(a);
+	}
+
+function enable_analyzer(tag: Analyzer::Tag) : bool
+	{
+	return __enable_analyzer(tag);
+	}
+
+function disable_analyzer(tag: Analyzer::Tag) : bool
+	{
+	return __disable_analyzer(tag);
+	}
+
+function register_for_ports(tag: Analyzer::Tag, ports: set[port]) : bool
+	{
+	local rc = T;
+
+	for ( p in ports )
+		{
+		if ( ! register_for_port(tag, p) )
+			rc = F;
+		}
+
+	return rc;
+	}
+
+function register_for_port(tag: Analyzer::Tag, p: port) : bool
+	{
+	if ( ! __register_for_port(tag, p) )
+		return F;
+
+	if ( tag !in ports )
+		ports[tag] = set();
+
+	add ports[tag][p];
+	return T;
+	}
+
+function registered_ports(tag: Analyzer::Tag) : set[port]
+	{
+	return tag in ports ? ports[tag] : set();
+	}
+
+function all_registered_ports(): table[Analyzer::Tag] of set[port]
+	{
+	return ports;
+	}
+
+function name(atype: Analyzer::Tag) : string
+	{
+	return __name(atype);
+	}
+
+function get_tag(name: string): Analyzer::Tag
+	{
+	return __tag(name);
+	}
+
+function schedule_analyzer(orig: addr, resp: addr, resp_p: port,
+			   analyzer: Analyzer::Tag, tout: interval) : bool
+	{
+	return __schedule_analyzer(orig, resp, resp_p, analyzer, tout);
+	}
+
+function analyzer_to_bpf(tag: Analyzer::Tag): string
+	{
+	# Return an empty string if an undefined analyzer was given.
+	if ( tag !in ports )
+		return "";
+
+	local output = "";
+	for ( p in ports[tag] )
+		output = PacketFilter::combine_filters(output, "or", PacketFilter::port_to_bpf(p));
+	return output;
+	}
+
+function get_bpf(): string
+	{
+	local output = "";
+	for ( tag in ports )
+		{
+		output = PacketFilter::combine_filters(output, "or", analyzer_to_bpf(tag));
+		}
+	return output;
+	}
+

scripts/base/frameworks/cluster/__load__.bro

@@ -9,10 +9,10 @@ redef peer_description = Cluster::node;
 # Add a cluster prefix.
 @prefixes += cluster
 
-## If this script isn't found anywhere, the cluster bombs out.
+# If this script isn't found anywhere, the cluster bombs out.
-## Loading the cluster framework requires that a script by this name exists
+# Loading the cluster framework requires that a script by this name exists
-## somewhere in the BROPATH.  The only thing in the file should be the
+# somewhere in the BROPATH.  The only thing in the file should be the
-## cluster definition in the :bro:id:`Cluster::nodes` variable.
+# cluster definition in the :bro:id:`Cluster::nodes` variable.
 @load cluster-layout
 
 @if ( Cluster::node in Cluster::nodes )

scripts/base/frameworks/cluster/main.bro

@@ -1,21 +1,45 @@
+##! A framework for establishing and controlling a cluster of Bro instances.
+##! In order to use the cluster framework, a script named
+##! ``cluster-layout.bro`` must exist somewhere in Bro's script search path
+##! which has a cluster definition of the :bro:id:`Cluster::nodes` variable.
+##! The ``CLUSTER_NODE`` environment variable or :bro:id:`Cluster::node`
+##! must also be sent and the cluster framework loaded as a package like
+##! ``@load base/frameworks/cluster``.
+
 @load base/frameworks/control
 
 module Cluster;
 
 export {
+	## The cluster logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## The record type which contains the column fields of the cluster log.
 	type Info: record {
+		## The time at which a cluster message was generated.
 		ts:       time;
+		## A message indicating information about the cluster's operation.
 		message:  string;
 	} &log;
 
+	## Types of nodes that are allowed to participate in the cluster
+	## configuration.
 	type NodeType: enum {
+		## A dummy node type indicating the local node is not operating
+		## within a cluster.
 		NONE,
+		## A node type which is allowed to view/manipulate the configuration
+		## of other nodes in the cluster.
 		CONTROL,
+		## A node type responsible for log and policy management.
 		MANAGER,
+		## A node type for relaying worker node communication and synchronizing
+		## worker node state.
 		PROXY,
+		## The node type doing all the actual traffic analysis.
 		WORKER,
+		## A node acting as a traffic recorder using the
+		## `Time Machine <http://tracker.bro.org/time-machine>`_ software.
 		TIME_MACHINE,
 	};
 	
@@ -49,30 +73,41 @@ export {
 	
 	## Record type to indicate a node in a cluster.
 	type Node: record {
+		## Identifies the  type of cluster node in this node's configuration.
 		node_type:    NodeType;
+		## The IP address of the cluster node.
 		ip:           addr;
+		## If the *ip* field is a non-global IPv6 address, this field
+		## can specify a particular :rfc:`4007` ``zone_id``.
+		zone_id:      string      &default="";
+		## The port to which the this local node can connect when
+		## establishing communication.
 		p:            port;
-		
 		## Identifier for the interface a worker is sniffing.
 		interface:    string      &optional;
-		
+		## Name of the manager node this node uses.  For workers and proxies.
-		## Manager node this node uses.  For workers and proxies.
 		manager:      string      &optional;
-		## Proxy node this node uses.  For workers and managers.
+		## Name of the proxy node this node uses.  For workers and managers.
 		proxy:        string      &optional;
-		## Worker nodes that this node connects with.  For managers and proxies.
+		## Names of worker nodes that this node connects with.
+		## For managers and proxies.
 		workers:      set[string] &optional;
+		## Name of a time machine node with which this node connects.
 		time_machine: string      &optional;
 	};
 	
 	## This function can be called at any time to determine if the cluster
 	## framework is being enabled for this run.
+	##
+	## Returns: True if :bro:id:`Cluster::node` has been set.
 	global is_enabled: function(): bool;
 	
 	## This function can be called at any time to determine what type of
 	## cluster node the current Bro instance is going to be acting as.
 	## If :bro:id:`Cluster::is_enabled` returns false, then
 	## :bro:enum:`Cluster::NONE` is returned.
+	##
+	## Returns: The :bro:type:`Cluster::NodeType` the calling node acts as.
 	global local_node_type: function(): NodeType;
 	
 	## This gives the value for the number of workers currently connected to,

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -13,8 +13,12 @@
 ## Turn off remote logging since this is the manager and should only log here.
 redef Log::enable_remote_logging = F;
 
+## Log rotation interval.
 redef Log::default_rotation_interval = 1 hrs;
 
+## Alarm summary mail interval.
+redef Log::default_mail_alarms_interval = 24 hrs;
+
 ## Use the cluster's archive logging script.
 redef Log::default_rotation_postprocessor_cmd = "archive-log";
 

scripts/base/frameworks/cluster/nodes/proxy.bro

@@ -1,3 +1,7 @@
+##! Redefines the options common to all proxy nodes within a Bro cluster.
+##! In particular, proxies are not meant to produce logs locally and they
+##! do not forward events anywhere, they mainly synchronize state between
+##! worker nodes.
 
 @prefixes += cluster-proxy
 

scripts/base/frameworks/cluster/nodes/worker.bro

@@ -1,3 +1,7 @@
+##! Redefines some options common to all worker nodes within a Bro cluster.
+##! In particular, worker nodes do not produce logs locally, instead they
+##! send them off to a manager node for processing.
+
 @prefixes += cluster-worker
 
 ## Don't do any local logging.

scripts/base/frameworks/cluster/setup-connections.bro

@@ -1,3 +1,6 @@
+##! This script establishes communication among all nodes in a cluster
+##! as defined by :bro:id:`Cluster::nodes`.
+
 @load ./main
 @load base/frameworks/communication
 
@@ -16,23 +19,26 @@ event bro_init() &priority=9
 		# Connections from the control node for runtime control and update events.
 		# Every node in a cluster is eligible for control from this host.
 		if ( n$node_type == CONTROL )
-			Communication::nodes["control"] = [$host=n$ip, $connect=F,
+			Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id,
-			                                   $class="control", $events=control_events];
+			                                   $connect=F, $class="control",
+			                                   $events=control_events];
 		
 		if ( me$node_type == MANAGER )
 			{
 			if ( n$node_type == WORKER && n$manager == node )
 				Communication::nodes[i] =
-				    [$host=n$ip, $connect=F,
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=worker2manager_events, $request_logs=T];
 			
 			if ( n$node_type == PROXY && n$manager == node )
 				Communication::nodes[i] =
-				    [$host=n$ip, $connect=F,
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=proxy2manager_events, $request_logs=T];
 				
 			if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
-				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
+				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
+				                                        $zone_id=nodes[i]$zone_id,
+				                                        $p=nodes[i]$p,
 				                                        $connect=T, $retry=1min,
 				                                        $events=tm2manager_events];
 			}
@@ -41,7 +47,8 @@ event bro_init() &priority=9
 			{
 			if ( n$node_type == WORKER && n$proxy == node )
 				Communication::nodes[i] =
-					[$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events];
+					[$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i,
+					 $sync=T, $auth=T, $events=worker2proxy_events];
 			
 			# accepts connections from the previous one. 
 			# (This is not ideal for setups with many proxies)
@@ -50,16 +57,18 @@ event bro_init() &priority=9
 				{
 				if ( n?$proxy )
 					Communication::nodes[i]
-					     = [$host=n$ip, $p=n$p,
+					     = [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
 					        $connect=T, $auth=F, $sync=T, $retry=1mins];
 				else if ( me?$proxy && me$proxy == i )
 					Communication::nodes[me$proxy]
-					     = [$host=nodes[i]$ip, $connect=F, $auth=T, $sync=T];
+					     = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,
+					        $connect=F, $auth=T, $sync=T];
 				}
 			
 			# Finally the manager, to send it status updates.
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				                                   $zone_id=nodes[i]$zone_id, 
 				                                   $p=nodes[i]$p, 
 				                                   $connect=T, $retry=1mins, 
 				                                   $class=node,
@@ -69,6 +78,7 @@ event bro_init() &priority=9
 			{
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				                                   $zone_id=nodes[i]$zone_id,
 				                                   $p=nodes[i]$p,
 				                                   $connect=T, $retry=1mins, 
 				                                   $class=node, 
@@ -76,6 +86,7 @@ event bro_init() &priority=9
 			
 			if ( n$node_type == PROXY && me$proxy == i )
 				Communication::nodes["proxy"] = [$host=nodes[i]$ip, 
+				                                 $zone_id=nodes[i]$zone_id,
 				                                 $p=nodes[i]$p,
 				                                 $connect=T, $retry=1mins, 
 				                                 $sync=T, $class=node, 
@@ -84,6 +95,7 @@ event bro_init() &priority=9
 			if ( n$node_type == TIME_MACHINE && 
 			     me?$time_machine && me$time_machine == i )
 				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, 
+				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
 				                                        $connect=T, 
 				                                        $retry=1min,

scripts/base/frameworks/communication/main.bro

@@ -1,14 +1,18 @@
-##! Connect to remote Bro or Broccoli instances to share state and/or transfer
+##! Facilitates connecting to remote Bro or Broccoli instances to share state
-##! events.
+##! and/or transfer events.
 
 @load base/frameworks/packet-filter
+@load base/utils/addrs
 
 module Communication;
 
 export {
+
+	## The communication logging stream identifier.
 	redef enum Log::ID += { LOG };
 
-	## Which interface to listen on (0.0.0.0 for any interface).
+	## Which interface to listen on. The addresses ``0.0.0.0`` and ``[::]``
+	## are wildcards.
 	const listen_interface = 0.0.0.0 &redef;
 
 	## Which port to listen on.
@@ -17,18 +21,42 @@ export {
 	## This defines if a listening socket should use SSL.
 	const listen_ssl = F &redef;
 
+	## Defines if a listening socket can bind to IPv6 addresses.
+	const listen_ipv6 = F &redef;
+
+	## If :bro:id:`Communication::listen_interface` is a non-global
+	## IPv6 address and requires a specific :rfc:`4007` ``zone_id``,
+	## it can be specified here.
+	const listen_ipv6_zone_id = "" &redef;
+
+	## Defines the interval at which to retry binding to
+	## :bro:id:`Communication::listen_interface` on
+	## :bro:id:`Communication::listen_port` if it's already in use.
+	const listen_retry = 30 secs &redef;
+
 	## Default compression level.  Compression level is 0-9, with 0 = no
 	## compression.
 	global compression_level = 0 &redef;
 
+	## A record type containing the column fields of the communication log.
 	type Info: record {
+		## The network time at which a communication event occurred.
 		ts:                  time   &log;
+		## The peer name (if any) with which a communication event is concerned.
 		peer:                string &log &optional;
+		## Where the communication event message originated from, that is,
+		## either from the scripting layer or inside the Bro process.
 		src_name:            string &log &optional;
+		## .. todo:: currently unused.
 		connected_peer_desc: string &log &optional;
+		## .. todo:: currently unused.
 		connected_peer_addr: addr   &log &optional;
+		## .. todo:: currently unused.
 		connected_peer_port: port   &log &optional;
+		## The severity of the communication event message.
 		level:               string &log &optional;
+		## A message describing the communication event between Bro or
+		## Broccoli instances.
 		message:             string &log;
 	};
 
@@ -39,6 +67,10 @@ export {
 		## Remote address.
 		host: addr;
 
+		## If the *host* field is a non-global IPv6 address, this field
+		## can specify a particular :rfc:`4007` ``zone_id``.
+		zone_id: string &optional;
+
 		## Port of the remote Bro communication endpoint if we are initiating
 		## the connection based on the :bro:id:`connect` field.
 		p: port &optional;
@@ -77,7 +109,7 @@ export {
 		auth: bool &default = F;
 
 		## If not set, no capture filter is sent.
-		## If set to "", the default cature filter is sent.
+		## If set to "", the default capture filter is sent.
 		capture_filter: string &optional;
 
 		## Whether to use SSL-based communication.
@@ -97,10 +129,24 @@ export {
 	## to or respond to connections from.
 	global nodes: table[string] of Node &redef;
 
+	## A table of peer nodes for which this node issued a
+	## :bro:id:`Communication::connect_peer` call but with which a connection
+	## has not yet been established or with which a connection has been
+	## closed and is currently in the process of retrying to establish.
+	## When a connection is successfully established, the peer is removed
+	## from the table.
 	global pending_peers: table[peer_id] of Node;
+
+	## A table of peer nodes for which this node has an established connection.
+	## Peers are automatically removed if their connection is closed and
+	## automatically added back if a connection is re-established later.
 	global connected_peers: table[peer_id] of Node;
 
-	## Connect to nodes[node], independent of its "connect" flag.
+	## Connect to a node in :bro:id:`Communication::nodes` independent
+	## of its "connect" flag.
+	##
+	## peer: the string used to index a particular node within the
+	##      :bro:id:`Communication::nodes` table.
 	global connect_peer: function(peer: string);
 }
 
@@ -133,7 +179,7 @@ event remote_log(level: count, src: count, msg: string)
 # This is a core generated event.
 event remote_log_peer(p: event_peer, level: count, src: count, msg: string)
 	{
-	local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg);
+	local rmsg = fmt("[#%d/%s:%d] %s", p$id, addr_to_uri(p$host), p$p, msg);
 	do_script_log_common(level, src, rmsg);
 	}
 
@@ -151,7 +197,8 @@ function connect_peer(peer: string)
 		p = node$p;
 
 	local class = node?$class ? node$class : "";
-	local id = connect(node$host, p, class, node$retry, node$ssl);
+	local zone_id = node?$zone_id ? node$zone_id : "";
+	local id = connect(node$host, zone_id, p, class, node$retry, node$ssl);
 
 	if ( id == PEER_ID_NONE )
 		Log::write(Communication::LOG, [$ts = network_time(),
@@ -169,12 +216,9 @@ function setup_peer(p: event_peer, node: Node)
 		request_remote_events(p, node$events);
 		}
 
-	if ( node?$capture_filter )
+	if ( node?$capture_filter && node$capture_filter != "" )
 		{
 		local filter = node$capture_filter;
-		if ( filter == "" )
-			filter = PacketFilter::default_filter;
-
 		do_script_log(p, fmt("sending capture_filter: %s", filter));
 		send_capture_filter(p, filter);
 		}

scripts/base/frameworks/control/main.bro

@@ -1,43 +1,34 @@
-##! This is a utility script that sends the current values of all &redef'able
+##! The control framework provides the foundation for providing "commands"
-##! consts to a remote Bro then sends the :bro:id:`configuration_update` event
+##! that can be taken remotely at runtime to modify a running Bro instance
-##! and terminates processing.
+##! or collect information from the running instance.
-##!
-##! Intended to be used from the command line like this when starting a controller::
-##!
-##!     bro <scripts> frameworks/control/controller Control::host=<host_addr> Control::port=<host_port> Control::cmd=<command> [Control::arg=<arg>]
-##!
-##! A controllee only needs to load the controllee script in addition
-##! to the specific analysis scripts desired.  It may also need a node
-##! configured as a controller node in the communications nodes configuration::
-##!
-##!     bro <scripts> frameworks/control/controllee
-##!
-##! To use the framework as a controllee, it only needs to be loaded and
-##! the controlled node need to accept all events in the "Control::" namespace
-##! from the host where the control actions will be performed from along with
-##! using the "control" class.
 
 module Control;
 
 export {
-	## This is the address of the host that will be controlled.
+	## The address of the host that will be controlled.
 	const host = 0.0.0.0 &redef;
 
-	## This is the port of the host that will be controlled.
+	## The port of the host that will be controlled.
 	const host_port = 0/tcp &redef;
 
-	## This is the command that is being done.  It's typically set on the
+	## If :bro:id:`Control::host` is a non-global IPv6 address and
-	## command line and influences whether this instance starts up as a
+	## requires a specific :rfc:`4007` ``zone_id``, it can be set here.
-	## controller or controllee.
+	const zone_id = "" &redef;
+
+	## The command that is being done.  It's typically set on the
+	## command line.
 	const cmd = "" &redef;
 
 	## This can be used by commands that take an argument.
 	const arg = "" &redef;
 
+	## Events that need to be handled by controllers.
 	const controller_events = /Control::.*_request/ &redef;
+	
+	## Events that need to be handled by controllees.
 	const controllee_events = /Control::.*_response/ &redef;
 
-	## These are the commands that can be given on the command line for
+	## The commands that can currently be given on the command line for
 	## remote control.
 	const commands: set[string] = {
 		"id_value",
@@ -45,15 +36,15 @@ export {
 		"net_stats",
 		"configuration_update",
 		"shutdown",
-	};
+	} &redef;
 
 	## Variable IDs that are to be ignored by the update process.
-	const ignore_ids: set[string] = {
+	const ignore_ids: set[string] = { };
-	};
 
 	## Event for requesting the value of an ID (a variable).
 	global id_value_request: event(id: string);
-	## Event for returning the value of an ID after an :bro:id:`id_request` event.
+	## Event for returning the value of an ID after an
+	## :bro:id:`Control::id_value_request` event.
 	global id_value_response: event(id: string, val: string);
 
 	## Requests the current communication status.
@@ -68,7 +59,8 @@ export {
 
 	## Inform the remote Bro instance that it's configuration may have been updated.
 	global configuration_update_request: event();
-	## This event is a wrapper and alias for the :bro:id:`configuration_update_request` event.
+	## This event is a wrapper and alias for the
+	## :bro:id:`Control::configuration_update_request` event.
 	## This event is also a primary hooking point for the control framework.
 	global configuration_update: event();
 	## Message in response to a configuration update request.

scripts/base/frameworks/dpd/dpd.sig

@@ -1,151 +0,0 @@
-# Signatures to initiate dynamic protocol detection.
-
-signature dpd_ftp_client {
-  ip-proto == tcp
-  payload /(|.*[\n\r]) *[uU][sS][eE][rR] /
-  tcp-state originator
-}
-
-# Match for server greeting (220, 120) and for login or passwd
-# required (230, 331).
-signature dpd_ftp_server {
-  ip-proto == tcp
-  payload /[\n\r ]*(120|220)[^0-9].*[\n\r] *(230|331)[^0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_ftp_client
-  enable "ftp"
-}
-
-signature dpd_http_client {
-  ip-proto == tcp
-  payload /^[[:space:]]*(GET|HEAD|POST)[[:space:]]*/
-  tcp-state originator
-}
-
-signature dpd_http_server {
-  ip-proto == tcp
-  payload /^HTTP\/[0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_http_client
-  enable "http"
-}
-
-signature dpd_bittorrenttracker_client {
-  ip-proto == tcp
-  payload /^.*\/announce\?.*info_hash/
-  tcp-state originator
-}
-
-signature dpd_bittorrenttracker_server {
-  ip-proto == tcp
-  payload /^HTTP\/[0-9]/
-  tcp-state responder
-  requires-reverse-signature dpd_bittorrenttracker_client
-  enable "bittorrenttracker"
-}
-
-signature dpd_bittorrent_peer1 {
-  ip-proto == tcp
-  payload /^\x13BitTorrent protocol/
-  tcp-state originator
-}
-
-signature dpd_bittorrent_peer2 {
-  ip-proto == tcp
-  payload /^\x13BitTorrent protocol/
-  tcp-state responder
-  requires-reverse-signature dpd_bittorrent_peer1
-  enable "bittorrent"
-}
-
-signature irc_client1 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Uu][Ss][Ee][Rr] +.+[\n\r]+ *[Nn][Ii][Cc][Kk] +.*[\r\n]/
-  requires-reverse-signature irc_server_reply
-  tcp-state originator
-  enable "irc"
-}
-
-signature irc_client2 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Nn][Ii][Cc][Kk] +.+[\r\n]+ *[Uu][Ss][Ee][Rr] +.+[\r\n]/
-  requires-reverse-signature irc_server_reply
-  tcp-state originator
-  enable "irc"
-}
-
-signature irc_server_reply {
-  ip-proto == tcp
-  payload /^(|.*[\n\r])(:[^ \n\r]+ )?[0-9][0-9][0-9] /
-  tcp-state responder
-}
-
-signature irc_server_to_server1 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
-}
-
-signature irc_server_to_server2 {
-  ip-proto == tcp
-  payload /(|.*[\r\n]) *[Ss][Ee][Rr][Vv][Ee][Rr] +[^ ]+ +[0-9]+ +:.+[\r\n]/
-  requires-reverse-signature irc_server_to_server1
-  enable "irc"
-}
-
-signature dpd_smtp_client {
-  ip-proto == tcp
-  payload /(|.*[\n\r])[[:space:]]*([hH][eE][lL][oO]|[eE][hH][lL][oO])/
-  requires-reverse-signature dpd_smtp_server
-  enable "smtp"
-  tcp-state originator
-}
-
-signature dpd_smtp_server {
-  ip-proto == tcp
-  payload /^[[:space:]]*220[[:space:]-]/
-  tcp-state responder
-}
-
-signature dpd_ssh_client {
-  ip-proto == tcp
-  payload /^[sS][sS][hH]-/
-  requires-reverse-signature dpd_ssh_server
-  enable "ssh"
-  tcp-state originator
-}
-
-signature dpd_ssh_server {
-  ip-proto == tcp
-  payload /^[sS][sS][hH]-/
-  tcp-state responder
-}
-
-signature dpd_pop3_server {
-  ip-proto == tcp
-  payload /^\+OK/
-  requires-reverse-signature dpd_pop3_client
-  enable "pop3"
-  tcp-state responder
-}
-
-signature dpd_pop3_client {
-  ip-proto == tcp
-  payload /(|.*[\r\n])[[:space:]]*([uU][sS][eE][rR][[:space:]]|[aA][pP][oO][pP][[:space:]]|[cC][aA][pP][aA]|[aA][uU][tT][hH])/
-  tcp-state originator
-}
-
-signature dpd_ssl_server {
-  ip-proto == tcp
-  # Server hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x02...\x03[\x00\x01\x02]|...?\x04..\x00\x02).*/
-  requires-reverse-signature dpd_ssl_client
-  enable "ssl"
-  tcp-state responder
-}
-
-signature dpd_ssl_client {
-  ip-proto == tcp
-  # Client hello.
-  payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
-  tcp-state originator
-}

scripts/base/frameworks/dpd/main.bro

@@ -3,18 +3,17 @@
 
 module DPD;
 
-## Add the DPD signatures to the signature framework.
-redef signature_files += "base/frameworks/dpd/dpd.sig";
-
 export {
+	## Add the DPD logging stream identifier.
 	redef enum Log::ID += { LOG };
 
+	## The record type defining the columns to log in the DPD logging stream.
 	type Info: record {
 		## Timestamp for when protocol analysis failed.
 		ts:             time            &log;
 		## Connection unique ID.
 		uid:            string          &log;
-		## Connection ID.
+		## Connection ID containing the 4-tuple which identifies endpoints.
 		id:             conn_id         &log;
 		## Transport protocol for the violation.
 		proto:          transport_proto &log;
@@ -40,22 +39,11 @@ redef record connection += {
 event bro_init() &priority=5
 	{
 	Log::create_stream(DPD::LOG, [$columns=Info]);
-	
-	# Populate the internal DPD analysis variable.
-	for ( a in dpd_config )
-		{
-		for ( p in dpd_config[a]$ports )
-			{
-			if ( p !in dpd_analyzer_ports )
-				dpd_analyzer_ports[p] = set();
-			add dpd_analyzer_ports[p][a];
-			}
-		}
 	}
 
-event protocol_confirmation(c: connection, atype: count, aid: count) &priority=10
+event protocol_confirmation(c: connection, atype: Analyzer::Tag, aid: count) &priority=10
 	{
-	local analyzer = analyzer_name(atype);
+	local analyzer = Analyzer::name(atype);
 
 	if ( fmt("-%s",analyzer) in c$service )
 		delete c$service[fmt("-%s", analyzer)];
@@ -63,10 +51,10 @@ event protocol_confirmation(c: connection, atype: count, aid: count) &priority=1
 	add c$service[analyzer];
 	}
 
-event protocol_violation(c: connection, atype: count, aid: count,
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
                          reason: string) &priority=10
 	{
-	local analyzer = analyzer_name(atype);
+	local analyzer = Analyzer::name(atype);
 	# If the service hasn't been confirmed yet, don't generate a log message
 	# for the protocol violation.
 	if ( analyzer !in c$service )
@@ -85,7 +73,7 @@ event protocol_violation(c: connection, atype: count, aid: count,
 	c$dpd = info;
 	}
 
-event protocol_violation(c: connection, atype: count, aid: count, reason: string) &priority=5
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count, reason: string) &priority=5
 	{
 	if ( !c?$dpd || aid in c$dpd$disabled_aids )
 		return;
@@ -99,9 +87,12 @@ event protocol_violation(c: connection, atype: count, aid: count, reason: string
 	add c$dpd$disabled_aids[aid];
 	}
 
-event protocol_violation(c: connection, atype: count, aid: count,
+event protocol_violation(c: connection, atype: Analyzer::Tag, aid: count,
 				reason: string) &priority=-5
 	{
 	if ( c?$dpd )
+		{
 		Log::write(DPD::LOG, c$dpd);
+		delete c$dpd;
+		}
 	}

scripts/base/frameworks/files/__load__.bro

@@ -0,0 +1 @@
+@load ./main.bro

scripts/base/frameworks/files/main.bro

@@ -0,0 +1,363 @@
+##! An interface for driving the analysis of files, possibly independent of
+##! any network protocol over which they're transported.
+
+@load base/bif/file_analysis.bif
+@load base/frameworks/analyzer
+@load base/frameworks/logging
+@load base/utils/site
+
+module Files;
+
+export {
+	redef enum Log::ID += {
+		## Logging stream for file analysis.
+		LOG
+	};
+
+	## A structure which represents a desired type of file analysis.
+	type AnalyzerArgs: record {
+		## An event which will be generated for all new file contents,
+		## chunk-wise.  Used when *tag* is
+		## :bro:see:`Files::ANALYZER_DATA_EVENT`.
+		chunk_event: event(f: fa_file, data: string, off: count) &optional;
+
+		## An event which will be generated for all new file contents,
+		## stream-wise.  Used when *tag* is
+		## :bro:see:`Files::ANALYZER_DATA_EVENT`.
+		stream_event: event(f: fa_file, data: string) &optional;
+	} &redef;
+
+	## Contains all metadata related to the analysis of a given file.
+	## For the most part, fields here are derived from ones of the same name
+	## in :bro:see:`fa_file`.
+	type Info: record {
+		## The time when the file was first seen.
+		ts: time &log;
+
+		## An identifier associated with a single file.
+		fuid: string &log;
+
+		## If this file was transferred over a network
+		## connection this should show the host or hosts that
+		## the data sourced from.
+		tx_hosts: set[addr] &log;
+
+		## If this file was transferred over a network
+		## connection this should show the host or hosts that
+		## the data traveled to.
+		rx_hosts: set[addr] &log;
+
+		## Connection UIDS over which the file was transferred.
+		conn_uids: set[string] &log;
+
+		## An identification of the source of the file data.  E.g. it may be
+		## a network protocol over which it was transferred, or a local file
+		## path which was read, or some other input source.
+		source: string &log &optional;
+
+		## A value to represent the depth of this file in relation 
+		## to its source.  In SMTP, it is the depth of the MIME
+		## attachment on the message.  In HTTP, it is the depth of the
+		## request within the TCP connection.
+		depth: count &default=0 &log;
+
+		## A set of analysis types done during the file analysis.
+		analyzers: set[string] &log;
+
+		## A mime type provided by libmagic against the *bof_buffer*, or
+		## in the cases where no buffering of the beginning of file occurs,
+		## an initial guess of the mime type based on the first data seen.
+		mime_type: string &log &optional;
+
+		## A filename for the file if one is available from the source
+		## for the file.  These will frequently come from 
+		## "Content-Disposition" headers in network protocols.
+		filename: string &log &optional;
+
+		## The duration the file was analyzed for.
+		duration: interval &log &default=0secs;
+
+		## If the source of this file is a network connection, this field
+		## indicates if the data originated from the local network or not as
+		## determined by the configured bro:see:`Site::local_nets`.
+		local_orig: bool &log &optional;
+
+		## If the source of this file is a network connection, this field
+		## indicates if the file is being sent by the originator of the connection
+		## or the responder.
+		is_orig: bool &log &optional;
+
+		## Number of bytes provided to the file analysis engine for the file.
+		seen_bytes: count &log &default=0;
+
+		## Total number of bytes that are supposed to comprise the full file.
+		total_bytes: count &log &optional;
+
+		## The number of bytes in the file stream that were completely missed
+		## during the process of analysis e.g. due to dropped packets.
+		missing_bytes: count &log &default=0;
+
+		## The number of not all-in-sequence bytes in the file stream that
+		## were delivered to file analyzers due to reassembly buffer overflow.
+		overflow_bytes: count &log &default=0;
+
+		## Whether the file analysis timed out at least once for the file.
+		timedout: bool &log &default=F;
+
+		## Identifier associated with a container file from which this one was
+		## extracted as part of the file analysis.
+		parent_fuid: string &log &optional;
+	} &redef;
+
+	## A table that can be used to disable file analysis completely for
+	## any files transferred over given network protocol analyzers.
+	const disable: table[Files::Tag] of bool = table() &redef;
+
+	## The salt concatenated to unique file handle strings generated by
+	## :bro:see:`get_file_handle` before hashing them in to a file id
+	## (the *id* field of :bro:see:`fa_file`).
+	## Provided to help mitigate the possiblility of manipulating parts of
+	## network connections that factor in to the file handle in order to
+	## generate two handles that would hash to the same file id.
+	const salt = "I recommend changing this." &redef;
+
+	## Sets the *timeout_interval* field of :bro:see:`fa_file`, which is
+	## used to determine the length of inactivity that is allowed for a file
+	## before internal state related to it is cleaned up.  When used within a
+	## :bro:see:`file_timeout` handler, the analysis will delay timing out
+	## again for the period specified by *t*.
+	##
+	## f: the file.
+	##
+	## t: the amount of time the file can remain inactive before discarding.
+	##
+	## Returns: true if the timeout interval was set, or false if analysis
+	##          for the *id* isn't currently active.
+	global set_timeout_interval: function(f: fa_file, t: interval): bool;
+
+	## Adds an analyzer to the analysis of a given file.
+	##
+	## f: the file.
+	##
+	## tag: the analyzer type.
+	##
+	## args: any parameters the analyzer takes.
+	##
+	## Returns: true if the analyzer will be added, or false if analysis
+	##          for the *id* isn't currently active or the *args*
+	##          were invalid for the analyzer type.
+	global add_analyzer: function(f: fa_file, 
+	                              tag: Files::Tag, 
+	                              args: AnalyzerArgs &default=AnalyzerArgs()): bool;
+
+	## Removes an analyzer from the analysis of a given file.
+	##
+	## f: the file.
+	##
+	## args: the analyzer (type and args) to remove.
+	##
+	## Returns: true if the analyzer will be removed, or false if analysis
+	##          for the *id* isn't currently active.
+	global remove_analyzer: function(f: fa_file,
+	                                 tag: Files::Tag,
+	                                 args: AnalyzerArgs &default=AnalyzerArgs()): bool;
+
+	## Stops/ignores any further analysis of a given file.
+	##
+	## f: the file.
+	##
+	## Returns: true if analysis for the given file will be ignored for the
+	##          rest of it's contents, or false if analysis for the *id*
+	##          isn't currently active.
+	global stop: function(f: fa_file): bool;
+
+	## Translates an file analyzer enum value to a string with the analyzer's name.
+	##
+	## tag: The analyzer tag.
+	##
+	## Returns: The analyzer name corresponding to the tag.
+	global analyzer_name: function(tag: Files::Tag): string;
+
+	## Provides a text description regarding metadata of the file.
+	## For example, with HTTP it would return a URL.
+	##
+	## f: The file to be described.
+	##
+	## Returns a text description regarding metadata of the file.
+	global describe: function(f: fa_file): string;
+
+	type ProtoRegistration: record {
+		## A callback to generate a file handle on demand when
+		## one is needed by the core.
+		get_file_handle: function(c: connection, is_orig: bool): string;
+		
+		## A callback to "describe" a file.  In the case of an HTTP
+		## transfer the most obvious description would be the URL.
+		## It's like an extremely compressed version of the normal log.
+		describe: function(f: fa_file): string
+				&default=function(f: fa_file): string { return ""; };
+	};
+
+	## Register callbacks for protocols that work with the Files framework.  
+	## The callbacks must uniquely identify a file and each protocol can 
+	## only have a single callback registered for it.
+	## 
+	## tag: Tag for the protocol analyzer having a callback being registered.
+	##
+	## reg: A :bro:see:`Files::ProtoRegistration` record.
+	##
+	## Returns: true if the protocol being registered was not previously registered.
+	global register_protocol: function(tag: Analyzer::Tag, reg: ProtoRegistration): bool;
+
+	## Register a callback for file analyzers to use if they need to do some manipulation
+	## when they are being added to a file before the core code takes over.  This is 
+	## unlikely to be interesting for users and should only be called by file analyzer
+	## authors but it *not required*.
+	## 
+	## tag: Tag for the file analyzer.
+	##
+	## callback: Function to execute when the given file analyzer is being added.
+	global register_analyzer_add_callback: function(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs));
+
+	## Event that can be handled to access the Info record as it is sent on
+	## to the logging framework.
+	global log_files: event(rec: Info);
+}
+
+redef record fa_file += {
+	info: Info &optional;
+};
+
+# Store the callbacks for protocol analyzers that have files.
+global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
+
+global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Files::LOG, [$columns=Info, $ev=log_files]);
+	}
+
+function set_info(f: fa_file)
+	{
+	if ( ! f?$info )
+		{
+		local tmp: Info = Info($ts=f$last_active,
+		                       $fuid=f$id);
+		f$info = tmp;
+		}
+
+	if ( f?$parent_id )
+		f$info$parent_fuid = f$parent_id;
+	if ( f?$source )
+		f$info$source = f$source;
+	f$info$duration = f$last_active - f$info$ts;
+	f$info$seen_bytes = f$seen_bytes;
+	if ( f?$total_bytes ) 
+		f$info$total_bytes = f$total_bytes;
+	f$info$missing_bytes = f$missing_bytes;
+	f$info$overflow_bytes = f$overflow_bytes;
+	if ( f?$is_orig )
+		f$info$is_orig = f$is_orig;
+	if ( f?$mime_type ) 
+		f$info$mime_type = f$mime_type;
+	}
+
+function set_timeout_interval(f: fa_file, t: interval): bool
+	{
+	return __set_timeout_interval(f$id, t);
+	}
+
+function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
+	{
+	add f$info$analyzers[Files::analyzer_name(tag)];
+
+	if ( tag in analyzer_add_callbacks )
+		analyzer_add_callbacks[tag](f, args);
+
+	if ( ! __add_analyzer(f$id, tag, args) )
+		{
+		Reporter::warning(fmt("Analyzer %s not added successfully to file %s.", tag, f$id));
+		return F;
+		}
+	return T;
+	}
+
+function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
+	{
+	analyzer_add_callbacks[tag] = callback;
+	}
+
+function remove_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
+	{
+	return __remove_analyzer(f$id, tag, args);
+	}
+
+function stop(f: fa_file): bool
+	{
+	return __stop(f$id);
+	}
+
+function analyzer_name(tag: Files::Tag): string
+	{
+	return __analyzer_name(tag);
+	}
+
+event file_new(f: fa_file) &priority=10
+	{
+	set_info(f);
+	}
+
+event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10
+	{
+	set_info(f);
+	add f$info$conn_uids[c$uid];
+	local cid = c$id;
+	add f$info$tx_hosts[f$is_orig ? cid$orig_h : cid$resp_h];
+	if( |Site::local_nets| > 0 )
+		f$info$local_orig=Site::is_local_addr(f$is_orig ? cid$orig_h : cid$resp_h);
+
+	add f$info$rx_hosts[f$is_orig ? cid$resp_h : cid$orig_h];
+	}
+
+event file_timeout(f: fa_file) &priority=10
+	{
+	set_info(f);
+	f$info$timedout = T;
+	}
+
+event file_state_remove(f: fa_file) &priority=10
+	{
+	set_info(f);
+	}
+
+event file_state_remove(f: fa_file) &priority=-10
+	{
+	Log::write(Files::LOG, f$info);
+	}
+
+function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
+	{
+	local result = (tag !in registered_protocols);
+	registered_protocols[tag] = reg;
+	return result;
+	}
+
+function describe(f: fa_file): string
+	{
+	local tag = Analyzer::get_tag(f$source);
+	if ( tag !in registered_protocols )
+		return "";
+
+	local handler = registered_protocols[tag];
+	return handler$describe(f);
+	}
+
+event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
+	{
+	if ( tag !in registered_protocols )
+		return;
+
+	local handler = registered_protocols[tag];
+	set_file_handle(handler$get_file_handle(c, is_orig));
+	}

scripts/base/frameworks/input/__load__.bro

@@ -0,0 +1,6 @@
+@load ./main
+@load ./readers/ascii
+@load ./readers/raw
+@load ./readers/benchmark
+@load ./readers/binary
+@load ./readers/sqlite

scripts/base/frameworks/input/main.bro

@@ -0,0 +1,217 @@
+##! The input framework provides a way to read previously stored data either
+##! as an event stream or into a bro table.
+
+module Input;
+
+export {
+
+	## The default input reader used. Defaults to `READER_ASCII`.
+	const default_reader = READER_ASCII &redef;
+
+	## The default reader mode used. Defaults to `MANUAL`.
+	const default_mode = MANUAL &redef;
+
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;
+
+	## Flag that controls if the input framework accepts records
+	## that contain types that are not supported (at the moment
+	## file and function). If true, the input framework will
+	## warn in these cases, but continue. If false, it will
+	## abort. Defaults to false (abort)
+	const accept_unsupported_types = F &redef;
+
+	## TableFilter description type used for the `table` method.
+	type TableDescription: record {
+		## Common definitions for tables and events
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this stream
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a stream at a later time
+		name: string;
+
+		# Special definitions for tables
+
+		## Table which will receive the data read by the input framework
+		destination: any;
+
+		## Record that defines the values used as the index of the table
+		idx: any;
+
+		## Record that defines the values used as the elements of the table
+		## If val is undefined, destination has to be a set.
+		val: any &optional;
+
+		## Defines if the value of the table is a record (default), or a single  value. Val
+		## can only contain one element when this is set to false.
+		want_record: bool &default=T;
+
+		## The event that is raised each time a value is added to, changed in or removed
+		## from the table. The event will receive an Input::Event enum as the first
+		## argument, the idx record as the second argument and the value (record) as the
+		## third argument.
+		ev: any &optional; # event containing idx, val as values.
+
+		## Predicate function that can decide if an insertion, update or removal should
+		## really be executed. Parameters are the same as for the event. If true is
+		## returned, the update is performed. If false is returned, it is skipped.
+		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+                config: table[string] of string &default=table();
+	};
+
+	## EventFilter description type used for the `event` method.
+	type EventDescription: record {
+		## Common definitions for tables and events
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this steam
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a stream at a later time
+		name: string;
+
+		# Special definitions for events
+
+		## Record describing the fields to be retrieved from the source input.
+		fields: any;
+
+		## If want_record if false, the event receives each value in fields as a separate argument.
+		## If it is set to true (default), the event receives all fields in a single record value.
+		want_record: bool &default=T;
+
+		## The event that is raised each time a new line is received from the reader.
+		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
+		ev: any;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
+	};
+
+	## A file analyis input stream type used to forward input data to the
+	## file analysis framework.
+	type AnalysisDescription: record {
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this steam.  Compatible readers must be
+		## able to accept a filter of a single string type (i.e.
+		## they read a byte stream).
+		reader: Reader &default=Input::READER_BINARY;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name that uniquely identifies the input source.
+		## Can be used used to remove a stream at a later time.
+		## This will also be used for the unique *source* field of
+		## :bro:see:`fa_file`.  Most of the time, the best choice for this
+		## field will be the same value as the *source* field.
+		name: string;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
+	};
+
+	## Create a new table input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_table: function(description: Input::TableDescription) : bool;
+
+	## Create a new event input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_event: function(description: Input::EventDescription) : bool;
+
+	## Create a new file analysis input from a given source.  Data read from
+	## the source is automatically forwarded to the file analysis framework.
+	##
+	## description: A record describing the source
+	##
+	## Returns: true on sucess.
+	global add_analysis: function(description: Input::AnalysisDescription) : bool;
+
+	## Remove a input stream. Returns true on success and false if the named stream was
+	## not found.
+	##
+	## id: string value identifying the stream to be removed
+	global remove: function(id: string) : bool;
+
+	## Forces the current input to be checked for changes.
+	## Returns true on success and false if the named stream was not found
+	##
+	## id: string value identifying the stream
+	global force_update: function(id: string) : bool;
+
+	## Event that is called, when the end of a data source has been reached, including
+	## after an update.
+	global end_of_data: event(name: string, source:string);
+}
+
+@load base/bif/input.bif
+
+
+module Input;
+
+function add_table(description: Input::TableDescription) : bool
+	{
+	return __create_table_stream(description);
+	}
+
+function add_event(description: Input::EventDescription) : bool
+	{
+	return __create_event_stream(description);
+	}
+
+function add_analysis(description: Input::AnalysisDescription) : bool
+	{
+	return __create_analysis_stream(description);
+	}
+
+function remove(id: string) : bool
+	{
+	return __remove_stream(id);
+	}
+
+function force_update(id: string) : bool
+	{
+	return __force_update(id);
+	}
+

scripts/base/frameworks/input/readers/ascii.bro

@@ -0,0 +1,21 @@
+##! Interface for the ascii input reader.
+##!
+##! The defaults are set to match Bro's ASCII output.
+
+module InputAscii;
+
+export {
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long
+	const separator = Input::separator &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long
+	const set_separator = Input::set_separator &redef;
+
+	## String to use for empty fields.
+	const empty_field = Input::empty_field &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Input::unset_field &redef;
+}

scripts/base/frameworks/input/readers/benchmark.bro

@@ -0,0 +1,23 @@
+##! Interface for the ascii input reader.
+
+module InputBenchmark;
+
+export {
+	## multiplication factor for each second
+	const factor = 1.0 &redef;
+
+	## spread factor between lines
+	const spread = 0 &redef;
+
+	## spreading where usleep = 1000000 / autospread * num_lines
+	const autospread = 0.0 &redef;
+
+	## addition factor for each heartbeat
+	const addfactor = 0 &redef;
+
+	## stop spreading at x lines per heartbeat
+	const stopspreadat = 0 &redef;
+
+	## 1 -> enable timed spreading
+	const timedspread = 0.0 &redef;
+}

scripts/base/frameworks/input/readers/binary.bro

@@ -0,0 +1,8 @@
+##! Interface for the binary input reader.
+
+module InputBinary;
+
+export {
+	## Size of data chunks to read from the input file at a time.
+	const chunk_size = 1024 &redef;
+}

scripts/base/frameworks/input/readers/raw.bro

@@ -0,0 +1,17 @@
+##! Interface for the raw input reader.
+
+module InputRaw;
+
+export {
+	## Separator between input records.
+	## Please note that the separator has to be exactly one character long
+	const record_separator = "\n" &redef;
+
+	## Event that is called when a process created by the raw reader exits.
+	##
+	## name: name of the input stream
+	## source: source of the input stream
+	## exit_code: exit code of the program, or number of the signal that forced the program to exit
+	## signal_exit: false when program exitted normally, true when program was forced to exit by a signal
+	global process_finished: event(name: string, source:string, exit_code:count, signal_exit:bool);
+}

scripts/base/frameworks/input/readers/sqlite.bro

@@ -0,0 +1,17 @@
+##! Interface for the SQLite input reader.
+##!
+##! The defaults are set to match Bro's ASCII output.
+
+module InputSQLite;
+
+export {
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long.
+	const set_separator = Input::set_separator &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Input::unset_field &redef;
+
+	## String to use for empty fields.
+	const empty_field = Input::empty_field &redef;
+}

scripts/base/frameworks/intel/__load__.bro

@@ -1 +1,11 @@
 @load ./main
+
+# The cluster framework must be loaded first.
+@load base/frameworks/cluster
+
+@if ( Cluster::is_enabled() )
+@load ./cluster
+@endif
+
+# This needs cluster support to only read on the manager.
+@load ./input

scripts/base/frameworks/intel/cluster.bro

@@ -0,0 +1,61 @@
+##! Cluster transparency support for the intelligence framework.  This is mostly oriented
+##! toward distributing intelligence information across clusters.
+
+@load base/frameworks/cluster
+@load ./input
+
+module Intel;
+
+redef record Item += {
+	## This field is used internally for cluster transparency to avoid 
+	## re-dispatching intelligence items over and over from workers.
+	first_dispatch: bool &default=T;
+};
+
+# If this process is not a manager process, we don't want the full metadata
+@if ( Cluster::local_node_type() != Cluster::MANAGER )
+redef have_full_data = F;
+@endif
+
+global cluster_new_item: event(item: Item);
+
+# Primary intelligence distribution comes from manager.
+redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
+# If a worker finds intelligence and adds it, it should share it back to the manager.
+redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+event Intel::match_no_items(s: Seen) &priority=5
+	{
+	event Intel::match(s, Intel::get_items(s));
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# When a worker connects, send it the complete minimal data store.
+	# It will be kept up to date after this by the cluster_new_item event.
+	if ( Cluster::nodes[p$descr]$node_type == Cluster::WORKER )
+		{
+		send_id(p, "Intel::min_data_store");
+		}
+	}
+@endif
+
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
+
+event Intel::new_item(item: Intel::Item) &priority=5
+	{
+	# The cluster manager always rebroadcasts intelligence.
+	# Workers redistribute it if it was locally generated.
+	if ( Cluster::local_node_type() == Cluster::MANAGER ||
+	     item$first_dispatch )
+		{
+		item$first_dispatch=F;
+		event Intel::cluster_new_item(item);
+		}
+	}

scripts/base/frameworks/intel/input.bro

@@ -0,0 +1,33 @@
+@load ./main
+
+module Intel;
+
+export {
+	## Intelligence files that will be read off disk.  The files are
+	## reread everytime they are updated so updates much be atomic with
+	## "mv" instead of writing the file in place.
+	const read_files: set[string] = {} &redef;
+}
+
+event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item)
+	{
+	Intel::insert(item);
+	}
+
+event bro_init() &priority=5
+	{
+	if ( ! Cluster::is_enabled() ||
+	     Cluster::local_node_type() == Cluster::MANAGER )
+		{
+		for ( a_file in read_files )
+			{
+			Input::add_event([$source=a_file,
+			                  $reader=Input::READER_ASCII,
+			                  $mode=Input::REREAD,
+			                  $name=cat("intel-", a_file),
+			                  $fields=Intel::Item,
+			                  $ev=Intel::read_entry]);
+			}
+		}
+	}
+

scripts/base/frameworks/intel/main.bro

@@ -1,24 +1,7 @@
 ##! The intelligence framework provides a way to store and query IP addresses,
-##! strings (with a subtype), and numeric (with a subtype) data.  Metadata 
+##! and strings (with a str_type).  Metadata can
-##! also be associated with the intelligence like tags which are arbitrary
+##! also be associated with the intelligence like for making more informed
-##! strings, time values, and longer descriptive strings.
+##! decisions about matching and handling of intelligence.
-
-# Example string subtypes:
-#   url
-#   email
-#   domain
-#   software
-#   user_name
-#   file_name
-#   file_md5
-#   x509_cert - DER encoded, not PEM (ascii armored)
-
-# Example tags: 
-#   infrastructure
-#   malicious
-#   sensitive
-#   canary
-#   friend
 
 @load base/frameworks/notice
 
@@ -27,251 +10,318 @@ module Intel;
 export {
 	redef enum Log::ID += { LOG };
 	
-	redef enum Notice::Type += {
+	## Enum type to represent various types of intelligence data.
-		## This notice should be used in all detector scripts to indicate 
+	type Type: enum {
-		## an intelligence based detection.
+		## An IP address.
-		Detection,
+		ADDR,
-	};
+		## A complete URL without the prefix "http://".
-	
+		URL,
-	type Info: record {
+		## Software name.
-		ts:      time   &log;
+		SOFTWARE,
-		## This value should be one of: "info", "warn", "error"
+		## Email address.
-		level:   string &log;
+		EMAIL,
-		message: string &log;
+		## DNS domain name.
+		DOMAIN,
+		## A user name.
+		USER_NAME,
+		## File hash which is non-hash type specific.  It's up to the user to query
+		## for any relevant hash types.
+		FILE_HASH,
+		## Certificate SHA-1 hash.
+		CERT_HASH,
 	};
 	
+	## Data about an :bro:type:`Intel::Item`
 	type MetaData: record {
+		## An arbitrary string value representing the data source.  Typically,
+		## the convention for this field will be the source name and feed name
+		## separated by a hyphen.  For example: "source1-c&c".
+		source:      string;
+		## A freeform description for the data.
 		desc:        string      &optional;
+		## A URL for more information about the data.
 		url:         string      &optional;
-		first_seen:  time        &optional;
-		latest_seen: time        &optional;
-		tags:        set[string];
 	};
 	
+	## Represents a piece of intelligence.
 	type Item: record {
-		ip:          addr        &optional;
+		## The intelligence indicator.
-		str:         string      &optional;
+		indicator:      string;
-		num:         int         &optional;
-		subtype:     string      &optional;
 
-		desc:        string      &optional;
+		## The type of data that the indicator field represents.
-		url:         string      &optional;
+		indicator_type: Type;
-		first_seen:  time        &optional;
-		latest_seen: time        &optional;
-		tags:        set[string];
 		
-		## These single string tags are throw away until pybroccoli supports sets
+		## Metadata for the item.  Typically represents more deeply
-		tag1: string &optional;
+		## descriptive data for a piece of intelligence.
-		tag2: string &optional;
+		meta:           MetaData;
-		tag3: string &optional;
 	};
 	
-	type QueryItem: record {
+	## Enum to represent where data came from when it was discovered.
-		ip:          addr        &optional;
+	## The convention is to prefix the name with ``IN_``.
-		str:         string      &optional;
+	type Where: enum {
-		num:         int         &optional;
+		## A catchall value to represent data of unknown provenance.
-		subtype:     string      &optional;
+		IN_ANYWHERE,
-		
-		or_tags:     set[string] &optional;
-		and_tags:    set[string] &optional; 
-		
-		## The predicate can be given when searching for a match.  It will
-		## be tested against every :bro:type:`MetaData` item associated with 
-		## the data being matched on.  If it returns T a single time, the 
-		## matcher will consider that the item has matched.
-		pred:    function(meta: Intel::MetaData): bool &optional;
 	};
 
+	type Seen: record {
+		## The string if the data is about a string.
+		indicator:       string        &log &optional;
 
-	global insert: function(item: Item): bool;
+		## The type of data that the indicator represents.
-	global insert_event: event(item: Item);
+		indicator_type:  Type          &log &optional;
-	global matcher: function(item: QueryItem): bool;
 
-	type MetaDataStore: table[count] of MetaData;
+		## If the indicator type was :bro:enum:`Intel::ADDR`, then this 
-	type DataStore: record {
+		## field will be present.
-		ip_data:     table[addr] of MetaDataStore;
+		host:            addr          &optional;
-		## The first string is the actual value and the second string is the subtype.
+
-		string_data: table[string, string] of MetaDataStore;
+		## Where the data was discovered.
-		int_data:    table[int, string] of MetaDataStore;
+		where:           Where         &log;
+		
+		## If the data was discovered within a connection, the 
+		## connection record should go into get to give context to the data.
+		conn:            connection    &optional;
 	};
-	global data_store: DataStore;
 
+	## Record used for the logging framework representing a positive
+	## hit within the intelligence framework.
+	type Info: record {
+		## Timestamp when the data was discovered.
+		ts:       time           &log;
 
+		## If a connection was associated with this intelligence hit,
+		## this is the uid for the connection
+		uid:      string         &log &optional;
+		## If a connection was associated with this intelligence hit,
+		## this is the conn_id for the connection.
+		id:       conn_id        &log &optional;
+
+		## Where the data was seen.
+		seen:     Seen           &log;
+		## Sources which supplied data that resulted in this match.
+		sources:  set[string]    &log &default=string_set();
+	};
+
+	## Intelligence data manipulation functions.
+	global insert: function(item: Item);
+
+	## Function to declare discovery of a piece of data in order to check
+	## it against known intelligence for matches.
+	global seen: function(s: Seen);
+
+	## Event to represent a match in the intelligence data from data that was seen.  
+	## On clusters there is no assurance as to where this event will be generated 
+	## so do not assume that arbitrary global state beyond the given data
+	## will be available.
+	##
+	## This is the primary mechanism where a user will take actions based on data
+	## within the intelligence framework.
+	global match: event(s: Seen, items: set[Item]);
+
+	global log_intel: event(rec: Info);
 }
 
-event bro_init()
+# Internal handler for matches with no metadata available.
+global match_no_items: event(s: Seen);
+
+# Internal events for cluster data distribution
+global new_item: event(item: Item);
+global updated_item: event(item: Item);
+
+# Optionally store metadata.  This is used internally depending on
+# if this is a cluster deployment or not.
+const have_full_data = T &redef;
+
+# The in memory data structure for holding intelligence.
+type DataStore: record {
+	host_data:    table[addr] of set[MetaData];
+	string_data:  table[string, Type] of set[MetaData];
+};
+global data_store: DataStore &redef;
+
+# The in memory data structure for holding the barest matchable intelligence.
+# This is primarily for workers to do the initial quick matches and store
+# a minimal amount of data for the full match to happen on the manager.
+type MinDataStore: record {
+	host_data:    set[addr];
+	string_data:  set[string, Type];
+};
+global min_data_store: MinDataStore &redef;
+
+
+event bro_init() &priority=5
 	{
-	Log::create_stream(Intel::LOG, [$columns=Info]);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_intel]);
 	}
 
-
+function find(s: Seen): bool
-function insert(item: Item): bool
 	{
-	local err_msg = "";
+	if ( s?$host )
-	if ( (item?$str || item?$num) && ! item?$subtype )
-		err_msg = "You must provide a subtype to insert_sync or this item doesn't make sense.";
-	
-	if ( err_msg == "" )
 		{
-		# Create and fill out the meta data item.
+		return ((s$host in min_data_store$host_data) || 
-		local meta: MetaData;
+		        (have_full_data && s$host in data_store$host_data));
-		if ( item?$first_seen )
+		}
-			meta$first_seen = item$first_seen;
+	else if ( ([to_lower(s$indicator), s$indicator_type] in min_data_store$string_data) ||
-		if ( item?$latest_seen )
+	           (have_full_data && [to_lower(s$indicator), s$indicator_type] in data_store$string_data) )
-			meta$latest_seen = item$latest_seen;
+		{
-		if ( item?$tags )
+		return T;
-			meta$tags = item$tags;
+		}
-		if ( item?$desc )
+	else
-			meta$desc = item$desc;
+		{
-		if ( item?$url )
+		return F;
-			meta$url = item$url;
+		}
+	}
 
+function get_items(s: Seen): set[Item]
+	{
+	local return_data: set[Item];
 
-		# This is hopefully only temporary until pybroccoli supports sets.
+	if ( ! have_full_data )
-		if ( item?$tag1 )
+		{
-			add item$tags[item$tag1];
+		# A reporter warning should be generated here because this function
-		if ( item?$tag2 )
+		# should never be called from a host that doesn't have the full data.
-			add item$tags[item$tag2];
+		# TODO: do a reporter warning.
-		if ( item?$tag3 )
+		return return_data;
-			add item$tags[item$tag3];
+		}
 
-		if ( item?$ip )
+	if ( s?$host )
+		{
+		# See if the host is known about and it has meta values
+		if ( s$host in data_store$host_data )
 			{
-			if ( item$ip !in data_store$ip_data )
+			for ( m in data_store$host_data[s$host] )
-				data_store$ip_data[item$ip] = table();
+				{
-			data_store$ip_data[item$ip][|data_store$ip_data[item$ip]|] = meta;
+				add return_data[Item($indicator=cat(s$host), $indicator_type=ADDR, $meta=m)];
-			return T;
+				}
 			}
-		else if ( item?$str )
+		}
+	else
+		{
+		local lower_indicator = to_lower(s$indicator);
+		# See if the string is known about and it has meta values
+		if ( [lower_indicator, s$indicator_type] in data_store$string_data )
 			{
-			if ( [item$str, item$subtype] !in data_store$string_data )
+			for ( m in data_store$string_data[lower_indicator, s$indicator_type] )
-				data_store$string_data[item$str, item$subtype] = table();
+				{
-			
+				add return_data[Item($indicator=s$indicator, $indicator_type=s$indicator_type, $meta=m)];
-			data_store$string_data[item$str, item$subtype][|data_store$string_data[item$str, item$subtype]|] = meta;
+				}
-			return T;
 			}
-		else if ( item?$num )
+		}
-			{
-			if ( [item$num, item$subtype] !in data_store$int_data )
-				data_store$int_data[item$num, item$subtype] = table();
 
-			data_store$int_data[item$num, item$subtype][|data_store$int_data[item$num, item$subtype]|] = meta;
+	return return_data;
-			return T;
+	}
+
+function Intel::seen(s: Seen)
+	{
+	if ( find(s) )
+		{
+		if ( s?$host )
+			{
+			s$indicator = cat(s$host);
+			s$indicator_type = Intel::ADDR;
+			}
+
+		if ( have_full_data )
+			{
+			local items = get_items(s);
+			event Intel::match(s, items);
 			}
 		else
-			err_msg = "Failed to insert intelligence item for some unknown reason.";
-		}
-	
-	if ( err_msg != "" )
-		Log::write(Intel::LOG, [$ts=network_time(), $level="warn", $message=fmt(err_msg)]);
-	return F;
-	}
-	
-event insert_event(item: Item)
-	{
-	insert(item);
-	}
-	
-function match_item_with_metadata(item: QueryItem, meta: MetaData): bool
-	{
-	if ( item?$and_tags )
-		{
-		local matched = T;
-		# Every tag given has to match in a single MetaData entry.
-		for ( tag in item$and_tags )
 			{
-			if ( tag !in meta$tags )
+			event Intel::match_no_items(s);
-				matched = F;
 			}
-		if ( matched )
+		}
+	}
+
+
+function has_meta(check: MetaData, metas: set[MetaData]): bool
+	{
+	local check_hash = md5_hash(check);
+	for ( m in metas )
+		{
+		if ( check_hash == md5_hash(m) )
 			return T;
 		}
-	else if ( item?$or_tags )
-		{
-		# For OR tags, only a single tag has to match.
-		for ( tag in item$or_tags )
-			{
-			if ( tag in meta$tags )
-				return T;
-			}
-		}
-	else if ( item?$pred )
-		return item$pred(meta);
 
-	# This indicates some sort of failure in the query
+	# The records must not be equivalent if we made it this far.
 	return F;
 	}
 
-function matcher(item: QueryItem): bool
+event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
-	local err_msg = "";
+	local info: Info = [$ts=network_time(), $seen=s];
-	if ( ! (item?$ip || item?$str || item?$num) )
-		err_msg = "You must supply one of the $ip, $str, or $num fields to search on";
-	else if ( (item?$or_tags || item?$and_tags) && item?$pred )
-		err_msg = "You can't match with both tags and a predicate.";
-	else if ( item?$or_tags && item?$and_tags )
-		err_msg = "You can't match with both OR'd together tags and AND'd together tags";
-	else if ( (item?$str || item?$num) && ! item?$subtype )
-		err_msg = "You must provide a subtype to matcher or this item doesn't make sense.";
-	else if ( item?$str && item?$num )
-		err_msg = "You must only provide $str or $num, not both.";
 
-	local meta: MetaData;
+	if ( s?$conn )
-
-	if ( err_msg == "" )
 		{
-		if ( item?$ip )
+		info$uid = s$conn$uid;
-			{
+		info$id  = s$conn$id;
-			if ( item$ip in data_store$ip_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-			
-				for ( i in data_store$ip_data[item$ip] )
-					{
-					meta = data_store$ip_data[item$ip][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		
-		else if ( item?$str )
-			{
-			if ( [item$str, item$subtype] in data_store$string_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$string_data[item$str, item$subtype] )
-					{
-					meta = data_store$string_data[item$str, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		
-		else if ( item?$num )
-			{
-			if ( [item$num, item$subtype] in data_store$int_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$int_data[item$num, item$subtype] )
-					{
-					meta = data_store$int_data[item$num, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		else
-			err_msg = "Failed to query intelligence data for some unknown reason.";
 		}
 
-	if ( err_msg != "" )
+	for ( item in items )
-		Log::write(Intel::LOG, [$ts=network_time(), $level="error", $message=fmt(err_msg)]);
+		add info$sources[item$meta$source];
-	return F;
+
+	Log::write(Intel::LOG, info);
 	}
+
+function insert(item: Item)
+	{
+	# Create and fill out the meta data item.
+	local meta = item$meta;
+	local metas: set[MetaData];
+
+	# All intelligence is case insensitive at the moment.
+	local lower_indicator = to_lower(item$indicator);
+
+	if ( item$indicator_type == ADDR )
+		{
+		local host = to_addr(item$indicator);
+		if ( have_full_data )
+			{
+			if ( host !in data_store$host_data )
+				data_store$host_data[host] = set();
+
+			metas = data_store$host_data[host];
+			}
+
+		add min_data_store$host_data[host];
+		}
+	else
+		{
+		if ( have_full_data )
+			{
+			if ( [lower_indicator, item$indicator_type] !in data_store$string_data )
+				data_store$string_data[lower_indicator, item$indicator_type] = set();
+
+			metas = data_store$string_data[lower_indicator, item$indicator_type];
+			}
+
+		add min_data_store$string_data[lower_indicator, item$indicator_type];
+		}
+
+	local updated = F;
+	if ( have_full_data )
+		{
+		for ( m in metas )
+			{
+			if ( meta$source == m$source )
+				{
+				if ( has_meta(meta, metas) )
+					{
+					# It's the same item being inserted again.
+					return;
+					}
+				else
+					{
+					# Same source, different metadata means updated item.
+					updated = T;
+					}
+				}
+			}
+		add metas[item$meta];
+		}
+	
+	if ( updated )
+		event Intel::updated_item(item);
+	else
+		event Intel::new_item(item);
+	}
+	

scripts/base/frameworks/logging/__load__.bro

@@ -1,3 +1,7 @@
 @load ./main
 @load ./postprocessors
 @load ./writers/ascii
+@load ./writers/dataseries
+@load ./writers/sqlite
+@load ./writers/elasticsearch
+@load ./writers/none

scripts/base/frameworks/logging/main.bro

@@ -1,43 +1,63 @@
 ##! The Bro logging interface.
 ##!
-##! See XXX for a introduction to Bro's logging framework.
+##! See :doc:`/logging` for a introduction to Bro's logging framework.
 
 module Log;
 
-# Log::ID and Log::Writer are defined in bro.init due to circular dependencies.
+# Log::ID and Log::Writer are defined in types.bif due to circular dependencies.
 
 export {
-	## If true, is local logging is by default enabled for all filters.
+	## If true, local logging is by default enabled for all filters.
 	const enable_local_logging = T &redef;
 
-	## If true, is remote logging is by default enabled for all filters.
+	## If true, remote logging is by default enabled for all filters.
 	const enable_remote_logging = T &redef;
 
 	## Default writer to use if a filter does not specify
 	## anything else.
 	const default_writer = WRITER_ASCII &redef;
 
+	## Default separator between fields for logwriters.
+	## Can be overwritten by individual writers.
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Can be overwritten by individual writers.
+	const set_separator = "," &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious. 
+	## Can be overwritten by individual writers.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	## Can be overwritten by individual writers.
+	const unset_field = "-" &redef;	
+
 	## Type defining the content of a logging stream.
 	type Stream: record {
 		## A record type defining the log's columns.
 		columns: any;
 
 		## Event that will be raised once for each log entry.
-		## The event receives a single same parameter, an instance of type ``columns``.
+		## The event receives a single same parameter, an instance of type
+		## ``columns``.
 		ev: any &optional;
 	};
 
-	## Default function for building the path values for log filters if not
+	## Builds the default path values for log filters if not otherwise
-	## speficied otherwise by a filter. The default implementation uses ``id``
+	## specified by a filter. The default implementation uses *id*
 	## to derive a name.
 	##
-	## id: The log stream.
+	## id: The ID associated with the log stream.
+	##
 	## path: A suggested path value, which may be either the filter's
 	##       ``path`` if defined, else a previous result from the function.
 	##       If no ``path`` is defined for the filter, then the first call
 	##       to the function will contain an empty string.
+	##
 	## rec: An instance of the streams's ``columns`` type with its
-	##      fields set to the values to logged.
+	##      fields set to the values to be logged.
 	##
 	## Returns: The path to be used for the filter.
 	global default_path_func: function(id: ID, path: string, rec: any) : string &redef;
@@ -46,7 +66,7 @@ export {
 
 	## Information passed into rotation callback functions.
 	type RotationInfo: record {
-		writer: Writer;		##< Writer.
+		writer: Writer;		##< The :bro:type:`Log::Writer` being used.
 		fname: string;		##< Full name of the rotated file.
 		path: string;		##< Original path value.
 		open: time;		##< Time when opened.
@@ -57,25 +77,29 @@ export {
 	## Default rotation interval. Zero disables rotation.
 	const default_rotation_interval = 0secs &redef;
 
-	## Default naming format for timestamps embedded into filenames. Uses a strftime() style.
+	## Default alarm summary mail interval. Zero disables alarm summary mails.
+	const default_mail_alarms_interval = 0secs &redef;
+
+	## Default naming format for timestamps embedded into filenames.
+	## Uses a ``strftime()`` style.
 	const default_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;
 
 	## Default shell command to run on rotated files. Empty for none.
 	const default_rotation_postprocessor_cmd = "" &redef;
 
-	## Specifies the default postprocessor function per writer type. Entries in this
+	## Specifies the default postprocessor function per writer type.
-	## table are initialized by each writer type.
+	## Entries in this table are initialized by each writer type.
 	const default_rotation_postprocessors: table[Writer] of function(info: RotationInfo) : bool &redef;
 
-	## Filter customizing logging.
+	## A filter type describes how to customize logging streams.
 	type Filter: record {
 		## Descriptive name to reference this filter.
 		name: string;
 
-		## The writer to use.
+		## The logging writer implementation to use.
 		writer: Writer &default=default_writer;
 
-		## Predicate indicating whether a log entry should be recorded.
+		## Indicates whether a log entry should be recorded.
 		## If not given, all entries are recorded.
 		##
 		## rec: An instance of the streams's ``columns`` type with its
@@ -92,6 +116,12 @@ export {
 		## file name. Generally, filenames are expected to given
 		## without any extensions; writers will add appropiate
 		## extensions automatically.
+		##
+		## If this path is found to conflict with another filter's
+		## for the same writer type, it is automatically corrected
+		## by appending "-N", where N is the smallest integer greater
+		## or equal to 2 that allows the corrected path name to not
+		## conflict with another filter's.
 		path: string &optional;
 
 		## A function returning the output path for recording entries
@@ -101,15 +131,20 @@ export {
 		## easy to flood the disk by returning a new string for each
 		## connection ...
 		##
-		## id: The log stream.
+		## id: The ID associated with the log stream.
+		##
 		## path: A suggested path value, which may be either the filter's
 		##       ``path`` if defined, else a previous result from the function.
 		##       If no ``path`` is defined for the filter, then the first call
 		##       to the function will contain an empty string.
-		## rec: An instance of the streams's ``columns`` type with its
-		##      fields set to the values to logged.
 		##
-		## Returns: The path to be used for the filter.
+		## rec: An instance of the streams's ``columns`` type with its
+		##      fields set to the values to be logged.
+		##
+		## Returns: The path to be used for the filter, which will be subject
+		##          to the same automatic correction rules as the *path*
+		##          field of :bro:type:`Log::Filter` in the case of conflicts
+		##          with other filters trying to use the same writer/path pair.
 		path_func: function(id: ID, path: string, rec: any): string &optional;
 
 		## Subset of column names to record. If not given, all
@@ -129,34 +164,209 @@ export {
 		## Rotation interval.
 		interv: interval &default=default_rotation_interval;
 
-		## Callback function to trigger for rotated files. If not set,
+		## Callback function to trigger for rotated files. If not set, the
-		## the default comes out of default_rotation_postprocessors.
+		## default comes out of :bro:id:`Log::default_rotation_postprocessors`.
 		postprocessor: function(info: RotationInfo) : bool &optional;
+
+		## A key/value table that will be passed on to the writer.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
 	};
 
 	## Sentinel value for indicating that a filter was not found when looked up.
-	const no_filter: Filter = [$name="<not found>"]; # Sentinel.
+	const no_filter: Filter = [$name="<not found>"];
 
-	# TODO: Document.
+	## Creates a new logging stream with the default filter.
+	##
+	## id: The ID enum to be associated with the new logging stream.
+	##
+	## stream: A record defining the content that the new stream will log.
+	##
+	## Returns: True if a new logging stream was successfully created and
+	##          a default filter added to it.
+	##
+	## .. bro:see:: Log::add_default_filter Log::remove_default_filter
 	global create_stream: function(id: ID, stream: Stream) : bool;
+
+	## Removes a logging stream completely, stopping all the threads.
+	##
+	## id: The ID enum to be associated with the new logging stream.
+	##
+	## Returns: True if a new stream was successfully removed.
+	##
+	## .. bro:see:: Log::create_stream
+	global remove_stream: function(id: ID) : bool;
+
+	## Enables a previously disabled logging stream.  Disabled streams
+	## will not be written to until they are enabled again.  New streams
+	## are enabled by default.
+	##
+	## id: The ID associated with the logging stream to enable.
+	##
+	## Returns: True if the stream is re-enabled or was not previously disabled.
+	##
+	## .. bro:see:: Log::disable_stream
 	global enable_stream: function(id: ID) : bool;
+
+	## Disables a currently enabled logging stream.  Disabled streams
+	## will not be written to until they are enabled again.  New streams
+	## are enabled by default.
+	##
+	## id: The ID associated with the logging stream to disable.
+	##
+	## Returns: True if the stream is now disabled or was already disabled.
+	##
+	## .. bro:see:: Log::enable_stream
 	global disable_stream: function(id: ID) : bool;
+
+	## Adds a custom filter to an existing logging stream.  If a filter
+	## with a matching ``name`` field already exists for the stream, it
+	## is removed when the new filter is successfully added.
+	##
+	## id: The ID associated with the logging stream to filter.
+	##
+	## filter: A record describing the desired logging parameters.
+	##
+	## Returns: True if the filter was sucessfully added, false if
+	##          the filter was not added or the *filter* argument was not
+	##          the correct type.
+	##
+	## .. bro:see:: Log::remove_filter Log::add_default_filter
+	##    Log::remove_default_filter
 	global add_filter: function(id: ID, filter: Filter) : bool;
+
+	## Removes a filter from an existing logging stream.
+	##
+	## id: The ID associated with the logging stream from which to
+	##     remove a filter.
+	##
+	## name: A string to match against the ``name`` field of a
+	##       :bro:type:`Log::Filter` for identification purposes.
+	##
+	## Returns: True if the logging stream's filter was removed or
+	##          if no filter associated with *name* was found.
+	##
+	## .. bro:see:: Log::remove_filter Log::add_default_filter
+	##    Log::remove_default_filter
 	global remove_filter: function(id: ID, name: string) : bool;
-	global get_filter: function(id: ID, name: string) : Filter; # Returns no_filter if not found.
+
+	## Gets a filter associated with an existing logging stream.
+	##
+	## id: The ID associated with a logging stream from which to
+	##     obtain one of its filters.
+	##
+	## name: A string to match against the ``name`` field of a
+	##       :bro:type:`Log::Filter` for identification purposes.
+	##
+	## Returns: A filter attached to the logging stream *id* matching
+	##          *name* or, if no matches are found returns the
+	##          :bro:id:`Log::no_filter` sentinel value.
+	##
+	## .. bro:see:: Log::add_filter Log::remove_filter Log::add_default_filter
+	##              Log::remove_default_filter
+	global get_filter: function(id: ID, name: string) : Filter;
+
+	## Writes a new log line/entry to a logging stream.
+	##
+	## id: The ID associated with a logging stream to be written to.
+	##
+	## columns: A record value describing the values of each field/column
+	##          to write to the log stream.
+	##
+	## Returns: True if the stream was found and no error occurred in writing
+	##          to it or if the stream was disabled and nothing was written.
+	##          False if the stream was was not found, or the *columns*
+	##          argument did not match what the stream was initially defined
+	##          to handle, or one of the stream's filters has an invalid
+	##          ``path_func``.
+	##
+	## .. bro:see: Log::enable_stream Log::disable_stream
 	global write: function(id: ID, columns: any) : bool;
+
+	## Sets the buffering status for all the writers of a given logging stream.
+	## A given writer implementation may or may not support buffering and if it
+	## doesn't then toggling buffering with this function has no effect.
+	##
+	## id: The ID associated with a logging stream for which to
+	##     enable/disable buffering.
+	##
+	## buffered: Whether to enable or disable log buffering.
+	##
+	## Returns: True if buffering status was set, false if the logging stream
+	##          does not exist.
+	##
+	## .. bro:see:: Log::flush
 	global set_buf: function(id: ID, buffered: bool): bool;
+
+	## Flushes any currently buffered output for all the writers of a given
+	## logging stream.
+	##
+	## id: The ID associated with a logging stream for which to flush buffered
+	##     data.
+	##
+	## Returns: True if all writers of a log stream were signalled to flush
+	##          buffered data or if the logging stream is disabled,
+	##          false if the logging stream does not exist.
+	##
+	## .. bro:see:: Log::set_buf Log::enable_stream Log::disable_stream
 	global flush: function(id: ID): bool;
+
+	## Adds a default :bro:type:`Log::Filter` record with ``name`` field
+	## set as "default" to a given logging stream.
+	##
+	## id: The ID associated with a logging stream for which to add a default
+	##     filter.
+	##
+	## Returns: The status of a call to :bro:id:`Log::add_filter` using a
+	##          default :bro:type:`Log::Filter` argument with ``name`` field
+	##          set to "default".
+	##
+	## .. bro:see:: Log::add_filter Log::remove_filter
+	##    Log::remove_default_filter
 	global add_default_filter: function(id: ID) : bool;
+
+	## Removes the :bro:type:`Log::Filter` with ``name`` field equal to
+	## "default".
+	##
+	## id: The ID associated with a logging stream from which to remove the
+	##     default filter.
+	##
+	## Returns: The status of a call to :bro:id:`Log::remove_filter` using
+	##          "default" as the argument.
+	##
+	## .. bro:see:: Log::add_filter Log::remove_filter Log::add_default_filter
 	global remove_default_filter: function(id: ID) : bool;
 
+	## Runs a command given by :bro:id:`Log::default_rotation_postprocessor_cmd`
+	## on a rotated file.  Meant to be called from postprocessor functions
+	## that are added to :bro:id:`Log::default_rotation_postprocessors`.
+	##
+	## info: A record holding meta-information about the log being rotated.
+	##
+	## npath: The new path of the file (after already being rotated/processed
+	##        by writer-specific postprocessor as defined in
+	##        :bro:id:`Log::default_rotation_postprocessors`.
+	##
+	## Returns: True when :bro:id:`Log::default_rotation_postprocessor_cmd`
+	##          is empty or the system command given by it has been invoked
+	##          to postprocess a rotated log file.
+	##
+	## .. bro:see:: Log::default_rotation_date_format
+	##    Log::default_rotation_postprocessor_cmd
+	##    Log::default_rotation_postprocessors
 	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
+
+	## The streams which are currently active and not disabled.
+	## This table is not meant to be modified by users!  Only use it for
+	## examining which streams are active.
+	global active_streams: table[ID] of Stream = table();
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
 global filters: table[ID, string] of Filter;
 
-@load base/logging.bif # Needs Filter and Stream defined.
+@load base/bif/logging.bif # Needs Filter and Stream defined.
 
 module Log;
 
@@ -165,20 +375,23 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
 	{
 	if ( info$writer in default_rotation_postprocessors )
 		return default_rotation_postprocessors[info$writer](info);
+	else
+		# Return T by default so that postprocessor-less writers don't shutdown.
+		return T;
 	}
 
 function default_path_func(id: ID, path: string, rec: any) : string
 	{
+	# The suggested path value is a previous result of this function
+	# or a filter path explicitly set by the user, so continue using it.
+	if ( path != "" )
+		return path;
+
 	local id_str = fmt("%s", id);
 
 	local parts = split1(id_str, /::/);
 	if ( |parts| == 2 )
 		{
-		# The suggested path value is a previous result of this function
-		# or a filter path explicitly set by the user, so continue using it.
-		if ( path != "" )
-			return path;
-		
 		# Example: Notice::LOG -> "notice"
 		if ( parts[2] == "LOG" )
 			{
@@ -214,13 +427,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo
 	if ( pp_cmd == "" )
 		return T;
 
+	# Turn, e.g., Log::WRITER_ASCII into "ascii".
+	local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", "");
+
 	# The date format is hard-coded here to provide a standardized
 	# script interface.
-	system(fmt("%s %s %s %s %s %d",
+	system(fmt("%s %s %s %s %s %d %s",
                pp_cmd, npath, info$path,
                strftime("%y-%m-%d_%H.%M.%S", info$open),
                strftime("%y-%m-%d_%H.%M.%S", info$close),
-               info$terminating));
+               info$terminating, writer));
 
 	return T;
 	}
@@ -230,11 +446,21 @@ function create_stream(id: ID, stream: Stream) : bool
 	if ( ! __create_stream(id, stream) )
 		return F;
 
+	active_streams[id] = stream;
+
 	return add_default_filter(id);
 	}
 
+function remove_stream(id: ID) : bool
+	{
+	delete active_streams[id];
+	return __remove_stream(id);
+	}
+
 function disable_stream(id: ID) : bool
 	{
+	delete active_streams[id];
+
 	return __disable_stream(id);
 	}
 

scripts/base/frameworks/logging/postprocessors/__load__.bro

@@ -1 +1,2 @@
 @load ./scp
+@load ./sftp

scripts/base/frameworks/logging/postprocessors/scp.bro

@@ -1,30 +1,56 @@
 ##! This script defines a postprocessing function that can be applied
 ##! to a logging filter in order to automatically SCP (secure copy)
 ##! a log stream (or a subset of it) to a remote host at configurable
-##! rotation time intervals.
+##! rotation time intervals.  Generally, to use this functionality
+##! you must handle the :bro:id:`bro_init` event and do the following
+##! in your handler:
+##!
+##! 1) Create a new :bro:type:`Log::Filter` record that defines a name/path,
+##!    rotation interval, and set the ``postprocessor`` to
+##!    :bro:id:`Log::scp_postprocessor`.
+##! 2) Add the filter to a logging stream using :bro:id:`Log::add_filter`.
+##! 3) Add a table entry to :bro:id:`Log::scp_destinations` for the filter's
+##!    writer/path pair which defines a set of :bro:type:`Log::SCPDestination`
+##!    records.
 
 module Log;
 
 export {
-	## This postprocessor SCP's the rotated-log to all the remote hosts
+	## Secure-copies the rotated-log to all the remote hosts
 	## defined in :bro:id:`Log::scp_destinations` and then deletes
 	## the local copy of the rotated-log.  It's not active when
 	## reading from trace files.
+	##
+	## info: A record holding meta-information about the log file to be
+	##       postprocessed.
+	##
+	## Returns: True if secure-copy system command was initiated or
+	##          if no destination was configured for the log as described
+	##          by *info*.
 	global scp_postprocessor: function(info: Log::RotationInfo): bool;
 
 	## A container that describes the remote destination for the SCP command
 	## argument as ``user@host:path``.
 	type SCPDestination: record {
+		## The remote user to log in as.  A trust mechanism should be
+		## pre-established.
 		user: string;
+		## The remote host to which to transfer logs.
 		host: string;
+		## The path/directory on the remote host to send logs.
 		path: string;
 	};
 
 	## A table indexed by a particular log writer and filter path, that yields
 	## a set remote destinations.  The :bro:id:`Log::scp_postprocessor`
 	## function queries this table upon log rotation and performs a secure
-	## copy of the rotated-log to each destination in the set.
+	## copy of the rotated-log to each destination in the set.  This
+	## table can be modified at run-time.
 	global scp_destinations: table[Writer, string] of set[SCPDestination];
+
+	## Default naming format for timestamps embedded into log filenames
+	## that use the SCP rotator.
+	const scp_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;
 }
 
 function scp_postprocessor(info: Log::RotationInfo): bool
@@ -34,7 +60,11 @@ function scp_postprocessor(info: Log::RotationInfo): bool
 
 	local command = "";
 	for ( d in scp_destinations[info$writer, info$path] )
-		command += fmt("scp %s %s@%s:%s;", info$fname, d$user, d$host, d$path);
+		{
+		local dst = fmt("%s/%s.%s.log", d$path, info$path,
+                        strftime(Log::scp_rotation_date_format, info$open));
+		command += fmt("scp %s %s@%s:%s;", info$fname, d$user, d$host, dst);
+		}
 
 	command += fmt("/bin/rm %s", info$fname);
 	system(command);

scripts/base/frameworks/logging/postprocessors/sftp.bro

@@ -0,0 +1,73 @@
+##! This script defines a postprocessing function that can be applied
+##! to a logging filter in order to automatically SFTP
+##! a log stream (or a subset of it) to a remote host at configurable
+##! rotation time intervals.  Generally, to use this functionality
+##! you must handle the :bro:id:`bro_init` event and do the following
+##! in your handler:
+##!
+##! 1) Create a new :bro:type:`Log::Filter` record that defines a name/path,
+##!    rotation interval, and set the ``postprocessor`` to
+##!    :bro:id:`Log::sftp_postprocessor`.
+##! 2) Add the filter to a logging stream using :bro:id:`Log::add_filter`.
+##! 3) Add a table entry to :bro:id:`Log::sftp_destinations` for the filter's
+##!    writer/path pair which defines a set of :bro:type:`Log::SFTPDestination`
+##!    records.
+
+module Log;
+
+export {
+	## Securely transfers the rotated-log to all the remote hosts
+	## defined in :bro:id:`Log::sftp_destinations` and then deletes
+	## the local copy of the rotated-log.  It's not active when
+	## reading from trace files.
+	##
+	## info: A record holding meta-information about the log file to be
+	##       postprocessed.
+	##
+	## Returns: True if sftp system command was initiated or
+	##          if no destination was configured for the log as described
+	##          by *info*.
+	global sftp_postprocessor: function(info: Log::RotationInfo): bool;
+
+	## A container that describes the remote destination for the SFTP command,
+	## comprised of the username, host, and path at which to upload the file.
+	type SFTPDestination: record {
+		## The remote user to log in as.  A trust mechanism should be
+		## pre-established.
+		user: string;
+		## The remote host to which to transfer logs.
+		host: string;
+		## The path/directory on the remote host to send logs.
+		path: string;
+	};
+
+	## A table indexed by a particular log writer and filter path, that yields
+	## a set remote destinations.  The :bro:id:`Log::sftp_postprocessor`
+	## function queries this table upon log rotation and performs a secure
+	## transfer of the rotated-log to each destination in the set.  This
+	## table can be modified at run-time.
+	global sftp_destinations: table[Writer, string] of set[SFTPDestination];
+
+	## Default naming format for timestamps embedded into log filenames
+	## that use the SFTP rotator.
+	const sftp_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;
+}
+
+function sftp_postprocessor(info: Log::RotationInfo): bool
+	{
+	if ( reading_traces() || [info$writer, info$path] !in sftp_destinations )
+		return T;
+
+	local command = "";
+	for ( d in sftp_destinations[info$writer, info$path] )
+		{
+		local dst = fmt("%s/%s.%s.log", d$path, info$path,
+		                strftime(Log::sftp_rotation_date_format, info$open));
+		command += fmt("echo put %s %s | sftp -b - %s@%s;", info$fname, dst,
+		               d$user, d$host);
+		}
+
+	command += fmt("/bin/rm %s", info$fname);
+	system(command);
+	return T;
+	}

scripts/base/frameworks/logging/writers/ascii.bro

@@ -1,4 +1,16 @@
-##! Interface for the ascii log writer.
+##! Interface for the ASCII log writer.  Redefinable options are available
+##! to tweak the output format of ASCII logs.
+##!
+##! The ASCII writer supports currently one writer-specific filter option via
+##! ``config``: setting ``tsv`` to the string ``T`` turns the output into into
+##! "tab-separated-value" mode where only a single header row with the column names
+##! is printed out as meta information, with no "# fields" prepended; no other meta
+##! data gets included in that mode.   
+##! 
+##! Example filter using this::
+##! 
+##!    local my_filter: Log::Filter = [$name = "my-filter", $writer = Log::WRITER_ASCII, $config = table(["tsv"] = "T")]; 
+##!
 
 module LogAscii;
 
@@ -7,24 +19,26 @@ export {
 	## into files. This is primarily for debugging purposes.
 	const output_to_stdout = F &redef;
 
-	## If true, include a header line with column names.
+	## If true, include lines with log meta information such as column names with
-	const include_header = T &redef;
+	## types, the values of ASCII logging options that in use, and the time when the
+	## file was opened and closes (the latter at the end). 
+	const include_meta = T &redef;
 
-	## Prefix for the header line if included.
+	## Prefix for lines with meta information.
-	const header_prefix = "#" &redef;
+	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
-	const separator = "\t" &redef;
+	const separator = Log::separator &redef;
 
 	## Separator between set elements.
-	const set_separator = "," &redef;
+	const set_separator = Log::set_separator &redef;
 
 	## String to use for empty fields. This should be different from
         ## *unset_field* to make the output non-ambigious. 
-	const empty_field = "(empty)" &redef;
+	const empty_field = Log::empty_field &redef;
 
 	## String to use for an unset &optional field.
-	const unset_field = "-" &redef;
+	const unset_field = Log::unset_field &redef;
 }
 
 # Default function to postprocess a rotated ASCII log file. It moves the rotated

scripts/base/frameworks/logging/writers/dataseries.bro

@@ -0,0 +1,60 @@
+##! Interface for the DataSeries log writer.
+
+module LogDataSeries;
+
+export {
+	## Compression to use with the DS output file.  Options are:
+	##
+	## 'none' -- No compression.
+	## 'lzf'  -- LZF compression.   Very quick, but leads to larger output files.
+	## 'lzo'  -- LZO compression.   Very fast decompression times.
+	## 'gz'   -- GZIP compression.  Slower than LZF, but also produces smaller output.
+	## 'bz2'  -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
+	const compression = "gz" &redef;
+
+	## The extent buffer size.
+	## Larger values here lead to better compression and more efficient writes, but
+	## also increase the lag between the time events are received and the time they
+	## are actually written to disk.
+	const extent_size = 65536 &redef;
+
+	## Should we dump the XML schema we use for this DS file to disk?
+	## If yes, the XML schema shares the name of the logfile, but has
+	## an XML ending.
+	const dump_schema = F &redef;
+
+	## How many threads should DataSeries spawn to perform compression?
+	## Note that this dictates the number of threads per log stream.  If
+	## you're using a lot of streams, you may want to keep this number
+	## relatively small.
+	##
+	## Default value is 1, which will spawn one thread / stream.
+	##
+	## Maximum is 128, minimum is 1.
+	const num_threads = 1 &redef;
+
+	## Should time be stored as an integer or a double?
+	## Storing time as a double leads to possible precision issues and
+	## can (significantly) increase the size of the resulting DS log.
+	## That said, timestamps stored in double form are consistent
+	## with the rest of Bro, including the standard ASCII log. Hence, we
+	## use them by default.
+	const use_integer_for_time = F &redef;
+}
+
+# Default function to postprocess a rotated DataSeries log file. It moves the
+# rotated file to a new name that includes a timestamp with the opening time, and
+# then runs the writer's default postprocessor command on it.
+function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
+	{
+	# Move file to name including both opening and closing time.
+	local dst = fmt("%s.%s.ds", info$path,
+			strftime(Log::default_rotation_date_format, info$open));
+
+	system(fmt("/bin/mv %s %s", info$fname, dst));
+
+	# Run default postprocessor.
+	return Log::run_rotation_postprocessor_cmd(info, dst);
+	}
+
+redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };

scripts/base/frameworks/logging/writers/elasticsearch.bro

@@ -0,0 +1,48 @@
+##! Log writer for sending logs to an ElasticSearch server.
+##!
+##! Note: This module is in testing and is not yet considered stable!
+##!
+##! There is one known memory issue.  If your elasticsearch server is
+##! running slowly and taking too long to return from bulk insert
+##! requests, the message queue to the writer thread will continue
+##! growing larger and larger giving the appearance of a memory leak.
+
+module LogElasticSearch;
+
+export {
+	## Name of the ES cluster
+	const cluster_name = "elasticsearch" &redef;
+
+	## ES Server
+	const server_host = "127.0.0.1" &redef;
+
+	## ES Port
+	const server_port = 9200 &redef;
+
+	## Name of the ES index
+	const index_prefix = "bro" &redef;
+
+	## The ES type prefix comes before the name of the related log.
+	## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
+	const type_prefix = "" &redef;
+
+	## The time before an ElasticSearch transfer will timeout. Note that
+	## the fractional part of the timeout will be ignored. In particular, time
+	## specifications less than a second result in a timeout value of 0, which
+	## means "no timeout."
+	const transfer_timeout = 2secs;
+
+	## The batch size is the number of messages that will be queued up before
+	## they are sent to be bulk indexed.
+	const max_batch_size = 1000 &redef;
+
+	## The maximum amount of wall-clock time that is allowed to pass without
+	## finishing a bulk log send.  This represents the maximum delay you
+	## would like to have with your logs before they are sent to ElasticSearch.
+	const max_batch_interval = 1min &redef;
+
+	## The maximum byte size for a buffered JSON string to send to the bulk
+	## insert API.
+	const max_byte_size = 1024 * 1024 &redef;
+}
+

scripts/base/frameworks/logging/writers/none.bro

@@ -0,0 +1,17 @@
+##! Interface for the None log writer. Thiis writer is mainly for debugging.
+
+module LogNone;
+
+export {
+	## If true, output debugging output that can be useful for unit
+        ## testing the logging framework.
+	const debug = F &redef;
+}
+
+function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
+	{
+	return T;
+	}
+
+redef Log::default_rotation_postprocessors += { [Log::WRITER_NONE] = default_rotation_postprocessor_func };
+

scripts/base/frameworks/logging/writers/sqlite.bro

@@ -0,0 +1,17 @@
+##! Interface for the SQLite log writer.  Redefinable options are available
+##! to tweak the output format of the SQLite reader.
+
+module LogSQLite;
+
+export {
+	## Separator between set elements.
+	const set_separator = Log::set_separator &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = Log::unset_field &redef;
+
+	## String to use for empty fields. This should be different from
+        ## *unset_field* to make the output non-ambigious.
+	const empty_field = Log::empty_field &redef;
+}
+

scripts/base/frameworks/metrics/cluster.bro

@@ -1,264 +0,0 @@
-##! This implements transparent cluster support for the metrics framework.
-##! Do not load this file directly.  It's only meant to be loaded automatically
-##! and will be depending on if the cluster framework has been enabled.
-##! The goal of this script is to make metric calculation completely and
-##! transparently automated when running on a cluster.
-##! 
-##! Events defined here are not exported deliberately because they are meant
-##! to be an internal implementation detail.
-
-@load base/frameworks/cluster
-@load ./main
-
-module Metrics;
-
-export {
-	## This value allows a user to decide how large of result groups the 
-	## workers should transmit values.
-	const cluster_send_in_groups_of = 50 &redef;
-	
-	## This is the percent of the full threshold value that needs to be met 
-	## on a single worker for that worker to send the value to its manager in
-	## order for it to request a global view for that value.  There is no
-	## requirement that the manager requests a global view for the index
-	## since it may opt not to if it requested a global view for the index
-	## recently.
-	const cluster_request_global_view_percent = 0.1 &redef;
-	
-	## This event is sent by the manager in a cluster to initiate the 
-	## collection of metrics values for a filter.
-	global cluster_filter_request: event(uid: string, id: ID, filter_name: string);
-
-	## This event is sent by nodes that are collecting metrics after receiving
-	## a request for the metric filter from the manager.
-	global cluster_filter_response: event(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool);
-
-	## This event is sent by the manager in a cluster to initiate the
-	## collection of a single index value from a filter.  It's typically
-	## used to get intermediate updates before the break interval triggers
-	## to speed detection of a value crossing a threshold.
-	global cluster_index_request: event(uid: string, id: ID, filter_name: string, index: Index);
-
-	## This event is sent by nodes in response to a 
-	## :bro:id:`cluster_index_request` event.
-	global cluster_index_response: event(uid: string, id: ID, filter_name: string, index: Index, val: count);
-
-	## This is sent by workers to indicate that they crossed the percent of the 
-	## current threshold by the percentage defined globally in 
-	## :bro:id:`cluster_request_global_view_percent`
-	global cluster_index_intermediate_response: event(id: Metrics::ID, filter_name: string, index: Metrics::Index, val: count);
-
-	## This event is scheduled internally on workers to send result chunks.
-	global send_data: event(uid: string, id: ID, filter_name: string, data: MetricTable);
-	
-}
-
-
-# This is maintained by managers so they can know what data they requested and
-# when they requested it.
-global requested_results: table[string] of time = table() &create_expire=5mins;
-
-# TODO: The next 4 variables make the assumption that a value never 
-#       takes longer than 5 minutes to transmit from workers to manager.  This needs to 
-#       be tunable or self-tuning.  These should also be restructured to be
-#       maintained within a single variable.
-
-# This variable is maintained by manager nodes as they collect and aggregate 
-# results.
-global filter_results: table[string, ID, string] of MetricTable &create_expire=5mins;
-
-# This variable is maintained by manager nodes to track how many "dones" they
-# collected per collection unique id.  Once the number of results for a uid 
-# matches the number of peer nodes that results should be coming from, the 
-# result is written out and deleted from here.
-# TODO: add an &expire_func in case not all results are received.
-global done_with: table[string] of count &create_expire=5mins &default=0;
-
-# This variable is maintained by managers to track intermediate responses as 
-# they are getting a global view for a certain index.
-global index_requests: table[string, ID, string, Index] of count &create_expire=5mins &default=0;
-
-# This variable is maintained by all hosts for different purposes. Non-managers
-# maintain it to know what indexes they have recently sent as intermediate
-# updates so they don't overwhelm their manager. Managers maintain it so they
-# don't overwhelm workers with intermediate index requests. The count that is
-# yielded is the number of times the percentage threshold has been crossed and
-# an intermediate result has been received. The manager may optionally request
-# the index again before data expires from here if too many workers are crossing
-# the percentage threshold (not implemented yet!).
-global recent_global_view_indexes: table[ID, string, Index] of count &create_expire=5mins &default=0;
-
-# Add events to the cluster framework to make this work.
-redef Cluster::manager2worker_events += /Metrics::cluster_(filter_request|index_request)/;
-redef Cluster::worker2manager_events += /Metrics::cluster_(filter_response|index_response|index_intermediate_response)/;
-
-@if ( Cluster::local_node_type() != Cluster::MANAGER )
-# This is done on all non-manager node types in the event that a metric is 
-# being collected somewhere other than a worker.
-function data_added(filter: Filter, index: Index, val: count)
-	{
-	# If an intermediate update for this value was sent recently, don't send
-	# it again.
-	if ( [filter$id, filter$name, index] in recent_global_view_indexes )
-		return;
-		
-	# If val is 5 and global view % is 0.1 (10%), pct_val will be 50.  If that
-	# crosses the full threshold then it's a candidate to send as an 
-	# intermediate update.
-	local pct_val = double_to_count(val / cluster_request_global_view_percent);
-	
-	if ( check_notice(filter, index, pct_val) ) 
-		{
-		# kick off intermediate update
-		event Metrics::cluster_index_intermediate_response(filter$id, filter$name, index, val);
-		
-		++recent_global_view_indexes[filter$id, filter$name, index];
-		}
-	}
-
-event Metrics::send_data(uid: string, id: ID, filter_name: string, data: MetricTable)
-	{
-	#print fmt("WORKER %s: sending data for uid %s...", Cluster::node, uid);
-	
-	local local_data: MetricTable;
-	local num_added = 0;
-	for ( index in data )
-		{
-		local_data[index] = data[index];
-		delete data[index];
-		
-		# Only send cluster_send_in_groups_of at a time.  Queue another
-		# event to send the next group.
-		if ( cluster_send_in_groups_of == ++num_added )
-			break;
-		}
-	
-	local done = F;
-	# If data is empty, this metric is done.
-	if ( |data| == 0 )
-		done = T;
-	
-	event Metrics::cluster_filter_response(uid, id, filter_name, local_data, done);
-	if ( ! done )
-		event Metrics::send_data(uid, id, filter_name, data);
-	}
-
-event Metrics::cluster_filter_request(uid: string, id: ID, filter_name: string)
-	{
-	#print fmt("WORKER %s: received the cluster_filter_request event.", Cluster::node);
-	
-	# Initiate sending all of the data for the requested filter.
-	event Metrics::send_data(uid, id, filter_name, store[id, filter_name]);
-	
-	# Lookup the actual filter and reset it, the reference to the data
-	# currently stored will be maintained interally by the send_data event.
-	reset(filter_store[id, filter_name]);
-	}
-	
-event Metrics::cluster_index_request(uid: string, id: ID, filter_name: string, index: Index)
-	{
-	local val=0;
-	if ( index in store[id, filter_name] )
-		val = store[id, filter_name][index];
-	
-	# fmt("WORKER %s: received the cluster_index_request event for %s=%d.", Cluster::node, index2str(index), val);
-	event Metrics::cluster_index_response(uid, id, filter_name, index, val);
-	}
-
-@endif
-
-
-@if ( Cluster::local_node_type() == Cluster::MANAGER )
-
-# Manager's handle logging.
-event Metrics::log_it(filter: Filter)
-	{
-	#print fmt("%.6f MANAGER: breaking %s filter for %s metric", network_time(), filter$name, filter$id);
-	
-	local uid = unique_id("");
-	
-	# Set some tracking variables.
-	requested_results[uid] = network_time();
-	filter_results[uid, filter$id, filter$name] = table();
-	
-	# Request data from peers.
-	event Metrics::cluster_filter_request(uid, filter$id, filter$name);
-	# Schedule the log_it event for the next break period.
-	schedule filter$break_interval { Metrics::log_it(filter) };
-	}
-
-# This is unlikely to be called often, but it's here in case there are metrics
-# being collected by managers.
-function data_added(filter: Filter, index: Index, val: count)
-	{
-	if ( check_notice(filter, index, val) )
-		do_notice(filter, index, val);
-	}
-	
-event Metrics::cluster_index_response(uid: string, id: ID, filter_name: string, index: Index, val: count)
-	{
-	#print fmt("%0.6f MANAGER: receiving index data from %s", network_time(), get_event_peer()$descr);
-
-	if ( [uid, id, filter_name, index] !in index_requests )
-		index_requests[uid, id, filter_name, index] = 0;
-	
-	index_requests[uid, id, filter_name, index] += val;
-	local ir = index_requests[uid, id, filter_name, index];
-	
-	++done_with[uid];
-	if ( Cluster::worker_count == done_with[uid] )
-		{
-		if ( check_notice(filter_store[id, filter_name], index, ir) )
-			do_notice(filter_store[id, filter_name], index, ir);
-		delete done_with[uid];
-		delete index_requests[uid, id, filter_name, index];
-		}
-	}
-
-# Managers handle intermediate updates here.
-event Metrics::cluster_index_intermediate_response(id: ID, filter_name: string, index: Index, val: count)
-	{
-	#print fmt("MANAGER: receiving intermediate index data from %s", get_event_peer()$descr);
-	#print fmt("MANAGER: requesting index data for %s", index2str(index));
-	
-	local uid = unique_id("");
-	event Metrics::cluster_index_request(uid, id, filter_name, index);
-	++recent_global_view_indexes[id, filter_name, index];
-	}
-
-event Metrics::cluster_filter_response(uid: string, id: ID, filter_name: string, data: MetricTable, done: bool)
-	{
-	#print fmt("MANAGER: receiving results from %s", get_event_peer()$descr);
-	
-	local local_data = filter_results[uid, id, filter_name];
-	for ( index in data )
-		{
-		if ( index !in local_data )
-			local_data[index] = 0;
-		local_data[index] += data[index];
-		}
-	
-	# Mark another worker as being "done" for this uid.
-	if ( done )
-		++done_with[uid];
-	
-	# If the data has been collected from all peers, we are done and ready to log.
-	if ( Cluster::worker_count == done_with[uid] )
-		{
-		local ts = network_time();
-		# Log the time this was initially requested if it's available.
-		if ( uid in requested_results )
-			{
-			ts = requested_results[uid];
-			delete requested_results[uid];
-			}
-		
-		write_log(ts, filter_store[id, filter_name], local_data);
-		
-		# Clean up
-		delete filter_results[uid, id, filter_name];
-		delete done_with[uid];
-		}
-	}
-
-@endif

scripts/base/frameworks/metrics/main.bro

@@ -1,274 +0,0 @@
-##! This is the implementation of the metrics framework.
-
-@load base/frameworks/notice
-
-module Metrics;
-
-export {
-	redef enum Log::ID += { LOG };
-	
-	type ID: enum {
-		NOTHING,
-	};
-	
-	## The default interval used for "breaking" metrics and writing the 
-	## current value to the logging stream.
-	const default_break_interval = 15mins &redef;
-	
-	## This is the interval for how often notices will happen after they have
-	## already fired.
-	const renotice_interval = 1hr &redef;
-	
-	type Index: record {
-		## Host is the value to which this metric applies.
-		host:         addr &optional;
-		
-		## A non-address related metric or a sub-key for an address based metric.
-		## An example might be successful SSH connections by client IP address
-		## where the client string would be the index value.
-		## Another example might be number of HTTP requests to a particular
-		## value in a Host header.  This is an example of a non-host based
-		## metric since multiple IP addresses could respond for the same Host
-		## header value.
-		str:        string &optional;
-		
-		## The CIDR block that this metric applies to.  This is typically
-		## only used internally for host based aggregation.
-		network:      subnet &optional;
-	} &log;
-	
-	type Info: record {
-		ts:           time   &log;
-		metric_id:    ID     &log;
-		filter_name:  string &log;
-		index:        Index  &log;
-		value:        count  &log;
-	};
-	
-	# TODO: configure a metrics filter logging stream to log the current
-	#       metrics configuration in case someone is looking through
-	#       old logs and the configuration has changed since then.
-	type Filter: record {
-		## The :bro:type:`Metrics::ID` that this filter applies to.
-		id:                ID                      &optional;
-		## The name for this filter so that multiple filters can be
-		## applied to a single metrics to get a different view of the same
-		## metric data being collected (different aggregation, break, etc).
-		name:              string                  &default="default";
-		## A predicate so that you can decide per index if you would like
-		## to accept the data being inserted.
-		pred:              function(index: Index): bool &optional;
-		## Global mask by which you'd like to aggregate traffic.
-		aggregation_mask:  count                   &optional;
-		## This is essentially a mapping table between addresses and subnets.
-		aggregation_table: table[subnet] of subnet &optional;
-		## The interval at which the metric should be "broken" and written
-		## to the logging stream.  The counters are also reset to zero at 
-		## this time so any threshold based detection needs to be set to a 
-		## number that should be expected to happen within this period.
-		break_interval:    interval                &default=default_break_interval;
-		## This determines if the result of this filter is sent to the metrics
-		## logging stream.  One use for the logging framework is as an internal
-		## thresholding and statistics gathering utility that is meant to
-		## never log but rather to generate notices and derive data.
-		log:               bool                    &default=T;
-		## If this and a $notice_threshold value are set, this notice type
-		## will be generated by the metrics framework.
-		note:              Notice::Type            &optional;
-		## A straight threshold for generating a notice.
-		notice_threshold:  count                   &optional;
-		## A series of thresholds at which to generate notices.
-		notice_thresholds: vector of count         &optional;
-		## How often this notice should be raised for this metric index.  It 
-		## will be generated everytime it crosses a threshold, but if the 
-		## $break_interval is set to 5mins and this is set to 1hr the notice
-		## only be generated once per hour even if something crosses the
-		## threshold in every break interval.
-		notice_freq:       interval                &optional;
-	};
-	
-	global add_filter: function(id: ID, filter: Filter);
-	global add_data: function(id: ID, index: Index, increment: count);
-	global index2str: function(index: Index): string;
-	
-	# This is the event that is used to "finish" metrics and adapt the metrics
-	# framework for clustered or non-clustered usage.
-	global log_it: event(filter: Filter);
-	
-	global log_metrics: event(rec: Info);
-}
-
-redef record Notice::Info += {
-	metric_index: Index &log &optional;
-};
-
-global metric_filters: table[ID] of vector of Filter = table();
-global filter_store: table[ID, string] of Filter = table();
-
-type MetricTable: table[Index] of count &default=0;
-# This is indexed by metric ID and stream filter name.
-global store: table[ID, string] of MetricTable = table() &default=table();
-
-# This function checks if a threshold has been crossed and generates a 
-# notice if it has.  It is also used as a method to implement 
-# mid-break-interval threshold crossing detection for cluster deployments.
-global check_notice: function(filter: Filter, index: Index, val: count): bool;
-
-# This is hook for watching thresholds being crossed.  It is called whenever
-# index values are updated and the new val is given as the `val` argument.
-global data_added: function(filter: Filter, index: Index, val: count);
-
-# This stores the current threshold index for filters using the
-# $notice_threshold and $notice_thresholds elements.
-global thresholds: table[ID, string, Index] of count = {} &create_expire=renotice_interval &default=0;
-
-event bro_init() &priority=5
-	{
-	Log::create_stream(Metrics::LOG, [$columns=Info, $ev=log_metrics]);
-	}
-
-function index2str(index: Index): string
-	{
-	local out = "";
-	if ( index?$host )
-		out = fmt("%shost=%s", out, index$host);
-	if ( index?$network )
-		out = fmt("%s%snetwork=%s", out, |out|==0 ? "" : ", ", index$network);
-	if ( index?$str )
-		out = fmt("%s%sstr=%s", out, |out|==0 ? "" : ", ", index$str);
-	return fmt("metric_index(%s)", out);
-	}
-	
-function write_log(ts: time, filter: Filter, data: MetricTable)
-	{
-	for ( index in data )
-		{
-		local val = data[index];
-		local m: Info = [$ts=ts,
-		                 $metric_id=filter$id,
-		                 $filter_name=filter$name,
-		                 $index=index,
-		                 $value=val];
-		
-		if ( filter$log )
-			Log::write(Metrics::LOG, m);
-		}
-	}
-
-
-function reset(filter: Filter)
-	{
-	store[filter$id, filter$name] = table();
-	}
-
-function add_filter(id: ID, filter: Filter)
-	{
-	if ( filter?$aggregation_table && filter?$aggregation_mask )
-		{
-		print "INVALID Metric filter: Defined $aggregation_table and $aggregation_mask.";
-		return;
-		}
-	if ( [id, filter$name] in store )
-		{
-		print fmt("INVALID Metric filter: Filter with name \"%s\" already exists.", filter$name);
-		return;
-		}
-	if ( filter?$notice_threshold && filter?$notice_thresholds )
-		{
-		print "INVALID Metric filter: Defined both $notice_threshold and $notice_thresholds";
-		return;
-		}
-	
-	if ( ! filter?$id )
-		filter$id = id;
-	
-	if ( id !in metric_filters )
-		metric_filters[id] = vector();
-	metric_filters[id][|metric_filters[id]|] = filter;
-
-	filter_store[id, filter$name] = filter;
-	store[id, filter$name] = table();
-	
-	schedule filter$break_interval { Metrics::log_it(filter) };
-	}
-	
-function add_data(id: ID, index: Index, increment: count)
-	{
-	if ( id !in metric_filters )
-		return;
-	
-	local filters = metric_filters[id];
-	
-	# Try to add the data to all of the defined filters for the metric.
-	for ( filter_id in filters )
-		{
-		local filter = filters[filter_id];
-		
-		# If this filter has a predicate, run the predicate and skip this
-		# index if the predicate return false.
-		if ( filter?$pred && ! filter$pred(index) )
-			next;
-		
-		if ( index?$host )
-			{
-			if ( filter?$aggregation_mask )
-				{
-				index$network = mask_addr(index$host, filter$aggregation_mask);
-				delete index$host;
-				}
-			else if ( filter?$aggregation_table )
-				{
-				# Don't add the data if the aggregation table doesn't include 
-				# the given host address.
-				if ( index$host !in filter$aggregation_table )
-					return;
-				index$network = filter$aggregation_table[index$host];
-				delete index$host;
-				}
-			}
-		
-		local metric_tbl = store[id, filter$name];
-		if ( index !in metric_tbl )
-			metric_tbl[index] = 0;
-		metric_tbl[index] += increment;
-		
-		data_added(filter, index, metric_tbl[index]);
-		}
-	}
-
-function check_notice(filter: Filter, index: Index, val: count): bool
-	{
-	if ( (filter?$notice_threshold &&
-	      [filter$id, filter$name, index] !in thresholds &&
-	      val >= filter$notice_threshold) ||
-	     (filter?$notice_thresholds &&
-	      |filter$notice_thresholds| <= thresholds[filter$id, filter$name, index] &&
-	      val >= filter$notice_thresholds[thresholds[filter$id, filter$name, index]]) )
-		return T;
-	else
-		return F;
-	}
-		
-function do_notice(filter: Filter, index: Index, val: count)
-	{
-	# We include $peer_descr here because the a manager count have actually 
-	# generated the notice even though the current remote peer for the event 
-	# calling this could be a worker if this is running as a cluster.
-	local n: Notice::Info = [$note=filter$note, 
-	                         $n=val, 
-	                         $metric_index=index, 
-	                         $peer_descr=peer_description];
-	n$msg = fmt("Threshold crossed by %s %d/%d", index2str(index), val, filter$notice_threshold);
-	if ( index?$str )
-		n$sub = index$str;
-	if ( index?$host )
-		n$src = index$host;
-	# TODO: not sure where to put the network yet.
-	
-	NOTICE(n);
-	
-	# This just needs set to some value so that it doesn't refire the 
-	# notice until it expires from the table or it crosses the next 
-	# threshold in the case of vectors of thresholds.
-	++thresholds[filter$id, filter$name, index];
-	}

scripts/base/frameworks/metrics/non-cluster.bro

@@ -1,21 +0,0 @@
-@load ./main
-
-module Metrics;
-
-event Metrics::log_it(filter: Filter)
-	{
-	local id = filter$id;
-	local name = filter$name;
-	
-	write_log(network_time(), filter, store[id, name]);
-	reset(filter);
-	
-	schedule filter$break_interval { Metrics::log_it(filter) };
-	}
-	
-	
-function data_added(filter: Filter, index: Index, val: count)
-	{
-	if ( check_notice(filter, index, val) )
-		do_notice(filter, index, val);
-	}

scripts/base/frameworks/notice/__load__.bro

@@ -17,6 +17,8 @@
 
 @if ( Cluster::is_enabled() )
 @load ./cluster
+@else
+@load ./non-cluster
 @endif
 
 # Load here so that it can check whether clustering is enabled.

scripts/base/frameworks/notice/actions/add-geodata.bro

@@ -27,18 +27,17 @@ export {
 	## Notice types which should have the "remote" location looked up.
 	## If GeoIP support is not built in, this does nothing.
 	const lookup_location_types: set[Notice::Type] = {} &redef;
-	
-	## Add a helper to the notice policy for looking up GeoIP data.
-	redef Notice::policy += {
-		[$pred(n: Notice::Info) = { return (n$note in Notice::lookup_location_types); },
-		 $action = ACTION_ADD_GEODATA,
-		 $priority = 10],
-	};
 }
 
+hook policy(n: Notice::Info) &priority=10
+	{
+	if ( n$note in Notice::lookup_location_types )
+		add n$actions[ACTION_ADD_GEODATA];
+	}
+
 # This is handled at a high priority in case other notice handlers 
 # want to use the data.
-event notice(n: Notice::Info) &priority=10
+hook notice(n: Notice::Info) &priority=10
 	{
 	if ( ACTION_ADD_GEODATA in n$actions &&
 	     |Site::local_nets| > 0 &&

scripts/base/frameworks/notice/actions/drop.bro

@@ -17,20 +17,13 @@ export {
 	};
 }
 
-# This is a little awkward because we want to inject drop along with the
+hook notice(n: Notice::Info)
-# synchronous functions.
-event bro_init()
 	{
-	local drop_func = function(n: Notice::Info)
+	if ( ACTION_DROP in n$actions )
 		{
-		if ( ACTION_DROP in n$actions )
+		#local drop = React::drop_address(n$src, "");
-			{
+		#local addl = drop?$sub ? fmt(" %s", drop$sub) : "";
-			#local drop = React::drop_address(n$src, "");
+		#n$dropped = drop$note != Drop::AddressDropIgnored;
-			#local addl = drop?$sub ? fmt(" %s", drop$sub) : "";
+		#n$msg += fmt(" [%s%s]", drop$note, addl);
-			#n$dropped = drop$note != Drop::AddressDropIgnored;
+		}
-			#n$msg += fmt(" [%s%s]", drop$note, addl);
-			}
-		};
-
-	add Notice::sync_functions[drop_func];
 	}