Merging master.

CMakeLists.txt

@@ -88,25 +88,60 @@ if (LIBGEOIP_FOUND)
     list(APPEND OPTLIBS ${LibGeoIP_LIBRARY})
 endif ()
 
-set(USE_PERFTOOLS false)
+set(HAVE_PERFTOOLS false)
 set(USE_PERFTOOLS_DEBUG false)
+set(USE_PERFTOOLS_TCMALLOC false)
 
-find_package(GooglePerftools)
+if (NOT DISABLE_PERFTOOLS)
+   find_package(GooglePerftools)
+endif ()
 
 if (GOOGLEPERFTOOLS_FOUND)
-    include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
+    set(HAVE_PERFTOOLS true)
-    set(USE_PERFTOOLS true)
+    # Non-Linux systems may not be well-supported by gperftools, so
+    # require explicit request from user to enable it in that case.
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Linux" OR ENABLE_PERFTOOLS)
+        set(USE_PERFTOOLS_TCMALLOC true)
 
         if (ENABLE_PERFTOOLS_DEBUG)
             # Enable heap debugging with perftools.
             set(USE_PERFTOOLS_DEBUG true)
+            include_directories(BEFORE ${GooglePerftools_INCLUDE_DIR})
             list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES_DEBUG})
         else ()
             # Link in tcmalloc for better performance.
             list(APPEND OPTLIBS ${GooglePerftools_LIBRARIES})
         endif ()
+    endif ()
 endif ()
 
+set(USE_DATASERIES false)
+find_package(Lintel)
+find_package(DataSeries)
+find_package(LibXML2)
+
+if (NOT DISABLE_DATASERIES AND
+    LINTEL_FOUND AND DATASERIES_FOUND AND LIBXML2_FOUND)
+    set(USE_DATASERIES true)
+	include_directories(BEFORE ${Lintel_INCLUDE_DIR})
+	include_directories(BEFORE ${DataSeries_INCLUDE_DIR})
+	include_directories(BEFORE ${LibXML2_INCLUDE_DIR})
+	list(APPEND OPTLIBS ${Lintel_LIBRARIES})
+	list(APPEND OPTLIBS ${DataSeries_LIBRARIES})
+	list(APPEND OPTLIBS ${LibXML2_LIBRARIES})
+endif()
+
+set(USE_ELASTICSEARCH false)
+set(USE_CURL false)
+find_package(LibCURL)
+
+if (NOT DISABLE_ELASTICSEARCH AND LIBCURL_FOUND)
+  set(USE_ELASTICSEARCH true)
+  set(USE_CURL true)
+  include_directories(BEFORE ${LibCURL_INCLUDE_DIR})
+  list(APPEND OPTLIBS ${LibCURL_LIBRARIES})
+endif()
+
 if (ENABLE_PERFTOOLS_DEBUG)
     # Just a no op to prevent CMake from complaining about manually-specified
     # ENABLE_PERFTOOLS_DEBUG not being used if google perftools weren't found
@@ -196,8 +231,13 @@ message(
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"
     "\n"
     "\nGeoIP:             ${USE_GEOIP}"
-    "\nGoogle perftools:  ${USE_PERFTOOLS}"
+    "\ngperftools found:  ${HAVE_PERFTOOLS}"
+    "\n        tcmalloc:  ${USE_PERFTOOLS_TCMALLOC}"
     "\n       debugging:  ${USE_PERFTOOLS_DEBUG}"
+    "\ncURL:              ${USE_CURL}"
+    "\n"
+    "\nDataSeries:        ${USE_DATASERIES}"
+    "\nElasticSearch:     ${USE_ELASTICSEARCH}"
     "\n"
     "\n================================================================\n"
 )

DocSourcesList.cmake

@@ -0,0 +1,144 @@
+# DO NOT EDIT
+# This file is auto-generated from the genDocSourcesList.sh script.
+#
+# This is a list of Bro script sources for which to generate reST documentation.
+# It will be included inline in the CMakeLists.txt found in the same directory
+# in order to create Makefile targets that define how to generate reST from
+# a given Bro script.
+#
+# Note: any path prefix of the script (2nd argument of rest_target macro)
+# will be used to derive what path under scripts/ the generated documentation
+# will be placed.
+
+set(psd ${PROJECT_SOURCE_DIR}/scripts)
+
+rest_target(${CMAKE_CURRENT_SOURCE_DIR} example.bro internal)
+rest_target(${psd} base/init-default.bro internal)
+rest_target(${psd} base/init-bare.bro internal)
+
+rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/types.bif.bro)
+rest_target(${psd} base/frameworks/cluster/main.bro)
+rest_target(${psd} base/frameworks/cluster/nodes/manager.bro)
+rest_target(${psd} base/frameworks/cluster/nodes/proxy.bro)
+rest_target(${psd} base/frameworks/cluster/nodes/worker.bro)
+rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
+rest_target(${psd} base/frameworks/communication/main.bro)
+rest_target(${psd} base/frameworks/control/main.bro)
+rest_target(${psd} base/frameworks/dpd/main.bro)
+rest_target(${psd} base/frameworks/intel/main.bro)
+rest_target(${psd} base/frameworks/logging/main.bro)
+rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
+rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
+rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
+rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
+rest_target(${psd} base/frameworks/metrics/cluster.bro)
+rest_target(${psd} base/frameworks/metrics/main.bro)
+rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
+rest_target(${psd} base/frameworks/notice/actions/add-geodata.bro)
+rest_target(${psd} base/frameworks/notice/actions/drop.bro)
+rest_target(${psd} base/frameworks/notice/actions/email_admin.bro)
+rest_target(${psd} base/frameworks/notice/actions/page.bro)
+rest_target(${psd} base/frameworks/notice/actions/pp-alarms.bro)
+rest_target(${psd} base/frameworks/notice/cluster.bro)
+rest_target(${psd} base/frameworks/notice/extend-email/hostnames.bro)
+rest_target(${psd} base/frameworks/notice/main.bro)
+rest_target(${psd} base/frameworks/notice/weird.bro)
+rest_target(${psd} base/frameworks/packet-filter/main.bro)
+rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
+rest_target(${psd} base/frameworks/reporter/main.bro)
+rest_target(${psd} base/frameworks/signatures/main.bro)
+rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/protocols/conn/contents.bro)
+rest_target(${psd} base/protocols/conn/inactivity.bro)
+rest_target(${psd} base/protocols/conn/main.bro)
+rest_target(${psd} base/protocols/dns/consts.bro)
+rest_target(${psd} base/protocols/dns/main.bro)
+rest_target(${psd} base/protocols/ftp/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/main.bro)
+rest_target(${psd} base/protocols/ftp/utils-commands.bro)
+rest_target(${psd} base/protocols/http/file-extract.bro)
+rest_target(${psd} base/protocols/http/file-hash.bro)
+rest_target(${psd} base/protocols/http/file-ident.bro)
+rest_target(${psd} base/protocols/http/main.bro)
+rest_target(${psd} base/protocols/http/utils.bro)
+rest_target(${psd} base/protocols/irc/dcc-send.bro)
+rest_target(${psd} base/protocols/irc/main.bro)
+rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
+rest_target(${psd} base/protocols/smtp/entities.bro)
+rest_target(${psd} base/protocols/smtp/main.bro)
+rest_target(${psd} base/protocols/ssh/main.bro)
+rest_target(${psd} base/protocols/ssl/consts.bro)
+rest_target(${psd} base/protocols/ssl/main.bro)
+rest_target(${psd} base/protocols/ssl/mozilla-ca-list.bro)
+rest_target(${psd} base/protocols/syslog/consts.bro)
+rest_target(${psd} base/protocols/syslog/main.bro)
+rest_target(${psd} base/utils/addrs.bro)
+rest_target(${psd} base/utils/conn-ids.bro)
+rest_target(${psd} base/utils/directions-and-hosts.bro)
+rest_target(${psd} base/utils/files.bro)
+rest_target(${psd} base/utils/numbers.bro)
+rest_target(${psd} base/utils/paths.bro)
+rest_target(${psd} base/utils/patterns.bro)
+rest_target(${psd} base/utils/site.bro)
+rest_target(${psd} base/utils/strings.bro)
+rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} policy/frameworks/communication/listen.bro)
+rest_target(${psd} policy/frameworks/control/controllee.bro)
+rest_target(${psd} policy/frameworks/control/controller.bro)
+rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
+rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
+rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
+rest_target(${psd} policy/frameworks/metrics/http-example.bro)
+rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
+rest_target(${psd} policy/frameworks/software/version-changes.bro)
+rest_target(${psd} policy/frameworks/software/vulnerable.bro)
+rest_target(${psd} policy/integration/barnyard2/main.bro)
+rest_target(${psd} policy/integration/barnyard2/types.bro)
+rest_target(${psd} policy/misc/analysis-groups.bro)
+rest_target(${psd} policy/misc/capture-loss.bro)
+rest_target(${psd} policy/misc/loaded-scripts.bro)
+rest_target(${psd} policy/misc/profiling.bro)
+rest_target(${psd} policy/misc/stats.bro)
+rest_target(${psd} policy/misc/trim-trace-file.bro)
+rest_target(${psd} policy/protocols/conn/known-hosts.bro)
+rest_target(${psd} policy/protocols/conn/known-services.bro)
+rest_target(${psd} policy/protocols/conn/weirds.bro)
+rest_target(${psd} policy/protocols/dns/auth-addl.bro)
+rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
+rest_target(${psd} policy/protocols/ftp/detect.bro)
+rest_target(${psd} policy/protocols/ftp/software.bro)
+rest_target(${psd} policy/protocols/http/detect-MHR.bro)
+rest_target(${psd} policy/protocols/http/detect-intel.bro)
+rest_target(${psd} policy/protocols/http/detect-sqli.bro)
+rest_target(${psd} policy/protocols/http/detect-webapps.bro)
+rest_target(${psd} policy/protocols/http/header-names.bro)
+rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
+rest_target(${psd} policy/protocols/http/software.bro)
+rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
+rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/smtp/blocklists.bro)
+rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
+rest_target(${psd} policy/protocols/smtp/software.bro)
+rest_target(${psd} policy/protocols/ssh/detect-bruteforcing.bro)
+rest_target(${psd} policy/protocols/ssh/geo-data.bro)
+rest_target(${psd} policy/protocols/ssh/interesting-hostnames.bro)
+rest_target(${psd} policy/protocols/ssh/software.bro)
+rest_target(${psd} policy/protocols/ssl/cert-hash.bro)
+rest_target(${psd} policy/protocols/ssl/expiring-certs.bro)
+rest_target(${psd} policy/protocols/ssl/extract-certs-pem.bro)
+rest_target(${psd} policy/protocols/ssl/known-certs.bro)
+rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
+rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
+rest_target(${psd} policy/tuning/defaults/warnings.bro)
+rest_target(${psd} policy/tuning/track-all-assets.bro)
+rest_target(${psd} site/local-manager.bro)
+rest_target(${psd} site/local-proxy.bro)
+rest_target(${psd} site/local-worker.bro)
+rest_target(${psd} site/local.bro)
+rest_target(${psd} test-all-policy.bro)

INSTALL

@@ -1,81 +1,303 @@
+.. _CMake: http://www.cmake.org
+.. _SWIG: http://www.swig.org
+.. _Xcode: https://developer.apple.com/xcode/
+.. _MacPorts: http://www.macports.org
+.. _Fink: http://www.finkproject.org
+.. _Homebrew: http://mxcl.github.com/homebrew
+.. _bro downloads page: http://bro-ids.org/download/index.html
+
 ==============
 Installing Bro
 ==============
 
+Bro can be downloaded in either pre-built binary package or
+source code forms.
+
 Prerequisites
 =============
 
-Bro relies on the following libraries and tools, which need to be installed
+Bro requires the following libraries and tools to be installed
 before you begin:
 
-    * CMake 2.6.3 or greater            http://www.cmake.org
+    * Libpcap                           http://www.tcpdump.org
 
-    * Libpcap (headers and libraries)   http://www.tcpdump.org
+    * OpenSSL libraries                 http://www.openssl.org
 
-    * OpenSSL (headers and libraries)   http://www.openssl.org
+    * BIND8 library
-
-    * SWIG                              http://www.swig.org
 
     * Libmagic
 
     * Libz
 
-Bro can make uses of some optional libraries if they are found at
+    * Bash  (for BroControl)
-installation time:
 
-    * LibGeoIP  For geo-locating IP addresses.
+To build Bro from source, the following additional dependencies are required:
 
-Bro also needs the following tools, but on most systems they will
+    * CMake 2.6.3 or greater            http://www.cmake.org
-already come preinstalled:
+
+    * SWIG                              http://www.swig.org
 
-    * Bash  (For Bro Control).
-    * BIND8 (headers and libraries)
     * Bison (GNU Parser Generator)
+
     * Flex  (Fast Lexical Analyzer)
-    * Perl  (Used only during the Bro build process)
+
+    * Libpcap headers                   http://www.tcpdump.org
+
+    * OpenSSL headers                   http://www.openssl.org
+
+    * libmagic headers
+
+    * zlib headers
+
+    * Perl
+
+Bro can make use of some optional libraries and tools if they are found at
+build time:
+
+    * LibGeoIP (for geo-locating IP addresses)
+
+    * gperftools (tcmalloc is used to improve memory and CPU usage)
+
+    * sendmail (for BroControl)
+
+    * ipsumdump (for trace-summary)     http://www.cs.ucla.edu/~kohler/ipsumdump
+
+    * Ruby executable, library, and headers (for Broccoli Ruby bindings)
 
 
-Installation
+Installing From Pre-Built Binary Release Packages
-============
+=================================================
 
-To build and install into ``/usr/local/bro``::
+See the `bro downloads page`_ for currently supported/targeted platforms.
+
+* RPM
+
+  .. console::
+
+      sudo yum localinstall Bro-*.rpm
+
+* DEB
+
+  .. console::
+
+      sudo gdebi Bro-*.deb
+
+* MacOS Disk Image with Installer
+
+  Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer.
+  Everything installed by the package will go into ``/opt/bro``.
+
+The primary install prefix for binary packages is ``/opt/bro``.
+Non-MacOS packages that include BroControl also put variable/runtime
+data (e.g. Bro logs) in ``/var/opt/bro``.
+
+
+Installing From Source
+======================
+
+Required Dependencies
+~~~~~~~~~~~~~~~~~~~~~
+
+The following dependencies are required to build Bro:
+
+* RPM/RedHat-based Linux:
+
+  .. console::
+
+     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+
+* DEB/Debian-based Linux:
+
+  .. console::
+
+     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+
+* FreeBSD
+
+  Most required dependencies should come with a minimal FreeBSD install
+  except for the following.
+
+  .. console::
+
+      sudo pkg_add -r bash cmake swig bison python
+
+  Note that ``bash`` needs to be in ``PATH``, which by default it is
+  not. The FreeBSD package installs the binary into
+  ``/usr/local/bin``.
+
+* Mac OS X
+
+  Compiling source code on Macs requires first downloading Xcode_,
+  then going through its "Preferences..." -> "Downloads" menus to
+  install the "Command Line Tools" component.
+
+  Lion (10.7) and Mountain Lion (10.8) come with all required
+  dependencies except for CMake_, SWIG_, and ``libmagic``.
+
+  Distributions of these dependencies can likely be obtained from your
+  preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
+  or Homebrew_).
+
+  Specifically for MacPorts, the ``swig``, ``swig-ruby``, ``swig-python``
+  and ``file`` packages provide the required dependencies.
+
+Optional Dependencies
+~~~~~~~~~~~~~~~~~~~~~
+
+Bro can use libGeoIP for geo-locating IP addresses, and sendmail for
+sending emails.
+
+* RedHat Enterprise Linux:
+
+  .. console::
+
+      sudo yum install geoip-devel sendmail
+
+* CentOS Linux:
+
+  .. console::
+  
+      sudo yum install GeoIP-devel sendmail
+
+* DEB/Debian-based Linux:
+
+  .. console::
+
+      sudo apt-get install libgeoip-dev sendmail
+
+* Ports-based FreeBSD
+
+  .. console::
+
+      sudo pkg_add -r GeoIP
+
+  sendmail is typically already available.
+
+* Mac OS X
+
+  Vanilla OS X installations don't ship with libGeoIP, but
+  if installed from your preferred package management system (e.g. MacPorts,
+  Fink, or Homebrew), they should be automatically detected and Bro will
+  compile against them.
+
+Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`.
+
+Compiling Bro Source Code
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Bro releases are bundled into source packages for convenience and
+available from the `bro downloads page`_.
+
+Alternatively, the latest Bro development version can be obtained through git
+repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_.  See
+our `git development documentation
+<http://bro-ids.org/development/process.html>`_ for comprehensive
+information on Bro's use of git revision control, but the short story
+for downloading the full source code experience for Bro via git is:
+
+.. console::
+
+    git clone --recursive git://git.bro-ids.org/bro
+
+.. note:: If you choose to clone the ``bro`` repository non-recursively for
+   a "minimal Bro experience", be aware that compiling it depends on
+   BinPAC, which has its own ``binpac`` repository.  Either install it
+   first or initialize/update the cloned ``bro`` repository's
+   ``aux/binpac`` submodule.
+
+The typical way to build and install from source is (for more options,
+run ``./configure --help``):
+
+.. console::
 
     ./configure
     make
     make install
 
-This will first build Bro into a directory inside the distribution
+The default installation path is ``/usr/local/bro``, which would typically
-called ``build/``, using default build options. It then installs all
+require root privileges when doing the ``make install``.  A different 
-required files into ``/usr/local/bro``, including the Bro binary in
+installation path can be chosen by specifying the ``--prefix`` option.
-``/usr/local/bro/bin/bro``.
+Note that ``/usr`` and ``/opt/bro`` are the
-
+standard prefixes for binary Bro packages to be installed, so those are
-You can specify a different installation directory with::
+typically not good choices unless you are creating such a package.
-
-    ./configure --prefix=<dir>
-
-Note that ``/usr`` and ``/opt/bro`` are the standard prefixes for
-binary Bro packages to be installed, so those are typically not good
-choices unless you are creating such a package.
-
-Run ``./configure --help`` for more options.
 
 Depending on the Bro package you downloaded, there may be auxiliary
-tools and libraries available in the ``aux/`` directory. All of them
+tools and libraries available in the ``aux/`` directory. Some of them
-except for ``aux/bro-aux`` will also be built and installed by doing
+will be automatically built and installed along with Bro. There are
-``make install``.  To install the programs that come in the
-``aux/bro-aux`` directory, use ``make install-aux``. There are
 ``--disable-*`` options that can be given to the configure script to
-turn off unwanted auxiliary projects.
+turn off unwanted auxiliary projects that would otherwise be installed
+automatically.  Finally, use ``make install-aux`` to install some of
+the other programs that are in the ``aux/bro-aux`` directory.
 
-OpenBSD users, please see our `FAQ
+OpenBSD users, please see our FAQ at
-<http://www.bro-ids.org/documentation/faq.html>` if you are having
+http://www.bro-ids.org/documentation/faq.html if you are having
 problems installing Bro.
 
+
+Upgrading From a Previous Version of Bro
+========================================
+
+If you're doing an upgrade install (rather than a fresh install),
+there's two suggested approaches: either install Bro using the same
+installation prefix directory as before, or pick a new prefix and copy
+local customizations over.
+
+Re-Use Previous Install Prefix
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you choose to configure and install Bro with the same prefix
+directory as before, local customization and configuration to files in
+``$prefix/share/bro/site`` and ``$prefix/etc`` won't be overwritten
+(``$prefix`` indicating the root of where Bro was installed). Also, logs
+generated at run-time won't be touched by the upgrade. (But making
+a backup of local changes before upgrading is still recommended.)
+
+After upgrading, remember to check ``$prefix/share/bro/site`` and
+``$prefix/etc`` for ``.example`` files, which indicate the
+distribution's version of the file differs from the local one, which may
+include local changes.  Review the differences, and make adjustments
+as necessary (for differences that aren't the result of a local change,
+use the new version's).
+
+Pick a New Install prefix
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you want to install the newer version in a different prefix
+directory than before, you can just copy local customization and
+configuration files from ``$prefix/share/bro/site`` and ``$prefix/etc``
+to the new location (``$prefix`` indicating the root of where Bro was
+originally installed).  Make sure to review the files for difference
+before copying and make adjustments as necessary (for differences that
+aren't the result of a local change, use the new version's).  Of
+particular note, the copied version of ``$prefix/etc/broctl.cfg`` is
+likely to need changes to the ``SpoolDir`` and ``LogDir`` settings.
+
+
+Configure the Run-Time Environment
+==================================
+
+Just remember that you may need to adjust your ``PATH`` environment variable
+according to the platform/shell/package you're using.  For example:
+
+Bourne-Shell Syntax:
+
+.. console::
+
+   export PATH=/usr/local/bro/bin:$PATH
+
+C-Shell Syntax:
+
+.. console::
+
+   setenv PATH /usr/local/bro/bin:$PATH
+
+Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
+
 Running Bro
 ===========
 
 Bro is a complex program and it takes a bit of time to get familiar
-with it.  A good place for newcomers to start is the Quickstart Guide
+with it.  A good place for newcomers to start is the Quick Start Guide
 at http://www.bro-ids.org/documentation/quickstart.html.
 
 For developers that wish to run Bro directly from the ``build/``

NEWS

@@ -3,22 +3,152 @@ Release Notes
 =============
 
 This document summarizes the most important changes in the current Bro
-release. For a complete list of changes, see the ``CHANGES`` file.
+release. For a complete list of changes, see the ``CHANGES`` file
+(note that submodules, such as BroControl and Broccoli, come with
+their own CHANGES.)
 
+Bro 2.2
+-------
+
+New Functionality
+~~~~~~~~~~~~~~~~~
+
+- GridFTP support. TODO: Extend.
+
+- ssl.log now also records the subject client and issuer certificates.
+
+Changed Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+- We removed the following, already deprecated, functionality:
+
+    * Scripting language:
+        - &disable_print_hook attribute.
+
+    * BiF functions:
+        - parse_dotted_addr(), dump_config(),
+          make_connection_persistent(), generate_idmef(),
+          split_complete()
+
+- Removed a now unused argument from "do_split" helper function.
+
+- "this" is no longer a reserved keyword.
+
+- The Input Framework's update_finished event has been renamed to
+  end_of_data. It will now not only fire after table-reads have been
+  completed, but also after the last event of a whole-file-read (or
+  whole-db-read, etc.).
+
+- Renamed the option defining the frequency of alarm summary mails to
+  'Logging::default_alarm_mail_interval'. When using BroControl, the
+  value can now be set with the new broctl.cfg option
+  "MailAlarmsInterval".
 
 Bro 2.1
 -------
 
-- Dependencies:
+New Functionality
+~~~~~~~~~~~~~~~~~
+
+- Bro now comes with extensive IPv6 support. Past versions offered
+  only basic IPv6 functionality that was rarely used in practice as it
+  had to be enabled explicitly. IPv6 support is now fully integrated
+  into all parts of Bro including protocol analysis and the scripting
+  language. It's on by default and no longer requires any special
+  configuration.
+
+  Some of the most significant enhancements include support for IPv6
+  fragment reassembly, support for following IPv6 extension header
+  chains, and support for tunnel decapsulation (6to4 and Teredo). The
+  DNS analyzer now handles AAAA records properly, and DNS lookups that
+  Bro itself performs now include AAAA queries, so that, for example,
+  the result returned by script-level lookups is a set that can
+  contain both IPv4 and IPv6 addresses. Support for the most common
+  ICMPv6 message types has been added. Also, the FTP EPSV and EPRT
+  commands are now handled properly. Internally, the way IP addresses
+  are stored has been improved, so Bro can handle both IPv4
+  and IPv6 by default without any special configuration.
+
+  In addition to Bro itself, the other Bro components have also been
+  made IPv6-aware by default. In particular, significant changes were
+  made to trace-summary, PySubnetTree, and Broccoli to support IPv6.
+
+- Bro now decapsulates tunnels via its new tunnel framework located in
+  scripts/base/frameworks/tunnels. It currently supports Teredo,
+  AYIYA, IP-in-IP (both IPv4 and IPv6), and SOCKS. For all these, it
+  logs the outer tunnel connections in both conn.log and tunnel.log,
+  and then proceeds to analyze the inner payload as if it were not
+  tunneled, including also logging that session in conn.log. For
+  SOCKS, it generates a new socks.log in addition with more
+  information.
+
+- Bro now features a flexible input framework that allows users to
+  integrate external information in real-time into Bro while it's
+  processing network traffic. The most direct use-case at the moment
+  is reading data from ASCII files into Bro tables, with updates
+  picked up automatically when the file changes during runtime. See
+  doc/input.rst for more information.
+
+  Internally, the input framework is structured around the notion of
+  "reader plugins" that make it easy to interface to different data
+  sources. We will add more in the future.
+
+- BroControl now has built-in support for host-based load-balancing
+  when using either PF_RING, Myricom cards, or individual interfaces.
+  Instead of adding a separate worker entry in node.cfg for each Bro
+  worker process on each worker host, it is now possible to just
+  specify the number of worker processes on each host and BroControl
+  configures everything correctly (including any neccessary enviroment
+  variables for the balancers).
+
+  This change adds three new keywords to the node.cfg file (to be used
+  with worker entries): lb_procs (specifies number of workers on a
+  host), lb_method (specifies what type of load balancing to use:
+  pf_ring, myricom, or interfaces), and lb_interfaces (used only with
+  "lb_method=interfaces" to specify which interfaces to load-balance
+  on).
+
+- Bro's default ASCII log format is not exactly the most efficient way
+  for storing and searching large volumes of data. An alternatives,
+  Bro now comes with experimental support for two alternative output
+  formats:
+
+    * DataSeries: an efficient binary format for recording structured
+      bulk data. DataSeries is developed and maintained at HP Labs.
+      See doc/logging-dataseries for more information.
+
+    * ElasticSearch: a distributed RESTful, storage engine and search
+      engine built on top of Apache Lucene. It scales very well, both
+      for distributed indexing and distributed searching. See
+      doc/logging-elasticsearch.rst for more information.
+
+  Note that at this point, we consider Bro's support for these two
+  formats as prototypes for collecting experience with alternative
+  outputs. We do not yet recommend them for production (but welcome
+  feedback!)
+
+
+Changed Functionality
+~~~~~~~~~~~~~~~~~~~~~
+
+The following summarizes the most important differences in existing
+functionality. Note that this list is not complete, see CHANGES for
+the full set.
+
+- Changes in dependencies:
 
     * Bro now requires CMake >= 2.6.3.
 
-    * Bro now links in tcmalloc (part of Google perftools) if found at
+    * On Linux, Bro now links in tcmalloc (part of Google perftools)
-      configure time. Doing so can significantly improve memory and
+      if found at configure time. Doing so can significantly improve
-      CPU use.
+      memory and CPU use.
 
-- Bro now supports IPv6 out of the box; the configure switch
+      On the other platforms, the new configure option
-  --enable-brov6 is gone.
+      --enable-perftools can be used to enable linking to tcmalloc.
+      (Note that perftools's support for non-Linux platforms may be
+      less reliable).
+
+- The configure switch --enable-brov6 is gone.
 
 - DNS name lookups performed by Bro now also query AAAA records. The
   results of the A and AAAA queries for a given hostname are combined
@@ -35,20 +165,44 @@ Bro 2.1
 - The syntax for IPv6 literals changed from "2607:f8b0:4009:802::1012"
   to "[2607:f8b0:4009:802::1012]".
 
-- Bro now spawn threads for doing its logging. From a user's
+- Bro now spawns threads for doing its logging. From a user's
   perspective not much should change, except that the OS may now show
   a bunch of Bro threads.
 
 - We renamed the configure option --enable-perftools to
-  --enable-perftool-debug to indicate that the switch is only relevant
+  --enable-perftools-debug to indicate that the switch is only relevant
   for debugging the heap.
 
 - Bro's ICMP analyzer now handles both IPv4 and IPv6 messages with a
   joint set of events.  The `icmp_conn` record got a new boolean field
   'v6' that indicates whether the ICMP message is v4 or v6.
 
+- Log postprocessor scripts get an additional argument indicating the
+  type of the log writer in use (e.g., "ascii").
+
+- BroControl's make-archive-name script also receives the writer
+  type, but as its 2nd(!) argument. If you're using a custom version
+  of that script, you need to adapt it. See the shipped version for
+  details.
+
+- Signature files can now be loaded via the new "@load-sigs"
+  directive. In contrast to the existing (and still supported)
+  signature_files constant, this can be used to load signatures
+  relative to the current script (e.g., "@load-sigs ./foo.sig").
+
+- The options "tunnel_port" and "parse_udp_tunnels" have been removed.
+  Bro now supports decapsulating tunnels directly for protocols it
+  understands.
+
+- ASCII logs now record the time when they were opened/closed at the
+  beginning and end of the file, respectively (wall clock). The
+  options LogAscii::header_prefix and LogAscii::include_header have
+  been renamed to LogAscii::meta_prefix and LogAscii::include_meta,
+  respectively.
+
+- The ASCII writers "header_*" options have been renamed to "meta_*"
+  (because there's now also a footer).
 
-TODO: Extend.
 
 Bro 2.0
 -------

VERSION

@@ -1 +1 @@
-2.0-306
+2.1-167

aux/binpac

@@ -1 +1 @@
-Subproject commit 71c37019bc371eb7863fb6aa47a7daa4540f4f1f
+Subproject commit 2fd9086c9dc0e76f6ff1ae04a60cbbce60507aab

aux/bro-aux

@@ -1 +1 @@
-Subproject commit d885987e7968669e34504b0403ac89bd13928e9a
+Subproject commit bea556198b69d30d64c0cf1b594e6de71176df6f

aux/broccoli

@@ -1 +1 @@
-Subproject commit 55f368b0ad283b2e7d68ef72922b5d9683e2a880
+Subproject commit 907210ce1470724fb386f939cc1b10a4caa2ae39

aux/broctl

@@ -1 +1 @@
-Subproject commit ff35c3c144885902c898bf8b47e351c7b8d55e10
+Subproject commit 834131cd0ec0f63cce9de818726fe6167dedbf34

aux/btest

@@ -1 +1 @@
-Subproject commit 045a02749b20b3c5896497959e6fda02d060508f
+Subproject commit d83e10c5f76cbfdf81c843575351fbc7b544fc93

cmake

@@ -1 +1 @@
-Subproject commit 49278736c1404cb8c077272b80312c947e68bf52
+Subproject commit 14537f56d66b18ab9d5024f798caf4d1f356fc67

config.h.in

@@ -114,6 +114,15 @@
 /* Analyze Mobile IPv6 traffic */
 #cmakedefine ENABLE_MOBILE_IPV6
 
+/* Use libCurl. */
+#cmakedefine USE_CURL
+
+/* Use the DataSeries writer. */
+#cmakedefine USE_DATASERIES
+
+/* Use the ElasticSearch writer. */
+#cmakedefine USE_ELASTICSEARCH
+
 /* Version number of package */
 #define VERSION "@VERSION@"
 
@@ -162,6 +171,10 @@
 #ifndef HAVE_IPPROTO_IPV6
 #define IPPROTO_IPV6 41
 #endif
+#cmakedefine HAVE_IPPROTO_IPV4
+#ifndef HAVE_IPPROTO_IPV4
+#define IPPROTO_IPV4 4
+#endif
 #cmakedefine HAVE_IPPROTO_ROUTING
 #ifndef HAVE_IPPROTO_ROUTING
 #define IPPROTO_ROUTING 43

configure

@@ -1,7 +1,7 @@
 #!/bin/sh
 # Convenience wrapper for easily viewing/setting options that
 # the project's CMake scripts will recognize
-
+set -e
 command="$0 $*"
 
 # check for `cmake` command
@@ -29,12 +29,17 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
   Optional Features:
     --enable-debug         compile in debugging mode
     --enable-mobile-ipv6   analyze mobile IPv6 features defined by RFC 6275
+    --enable-perftools     force use of Google perftools on non-Linux systems
+                           (automatically on when perftools is present on Linux)
     --enable-perftools-debug use Google's perftools for debugging
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
-    --disable-auxtools     don't build or install auxilliary tools
+    --disable-auxtools     don't build or install auxiliary tools
+    --disable-perftools    don't try to build with Google Perftools
     --disable-python       don't try to build python bindings for broccoli
     --disable-ruby         don't try to build ruby bindings for broccoli
+    --disable-dataseries   don't use the optional DataSeries log writer
+    --disable-elasticsearch don't use the optional ElasticSearch log writer
 
   Required Packages in Non-Standard Locations:
     --with-openssl=PATH    path to OpenSSL install root
@@ -56,6 +61,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
+    --with-dataseries=PATH path to DataSeries and Lintel libraries
+    --with-xml2=PATH       path to libxml2 installation (for DataSeries)
+    --with-curl=PATH       path to libcurl install root (for ElasticSearch)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -95,6 +103,7 @@ append_cache_entry PY_MOD_INSTALL_DIR   PATH   $prefix/lib/broctl
 append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $prefix/share/bro
 append_cache_entry BRO_ETC_INSTALL_DIR  PATH   $prefix/etc
 append_cache_entry ENABLE_DEBUG         BOOL   false
+append_cache_entry ENABLE_PERFTOOLS     BOOL   false
 append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL false
 append_cache_entry BinPAC_SKIP_INSTALL  BOOL   true
 append_cache_entry BUILD_SHARED_LIBS    BOOL   true
@@ -103,6 +112,7 @@ append_cache_entry INSTALL_BROCCOLI     BOOL   true
 append_cache_entry INSTALL_BROCTL       BOOL   true
 append_cache_entry CPACK_SOURCE_IGNORE_FILES STRING
 append_cache_entry ENABLE_MOBILE_IPV6   BOOL   false
+append_cache_entry DISABLE_PERFTOOLS    BOOL   false
 
 # parse arguments
 while [ $# -ne 0 ]; do
@@ -142,7 +152,11 @@ while [ $# -ne 0 ]; do
         --enable-mobile-ipv6)
             append_cache_entry ENABLE_MOBILE_IPV6         BOOL   true
             ;;
+        --enable-perftools)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
+            ;;
         --enable-perftools-debug)
+            append_cache_entry ENABLE_PERFTOOLS     BOOL   true
             append_cache_entry ENABLE_PERFTOOLS_DEBUG BOOL   true
             ;;
         --disable-broccoli)
@@ -154,12 +168,21 @@ while [ $# -ne 0 ]; do
         --disable-auxtools)
             append_cache_entry INSTALL_AUX_TOOLS    BOOL   false
             ;;
+        --disable-perftools)
+            append_cache_entry DISABLE_PERFTOOLS    BOOL   true
+            ;;
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS     BOOL   true
             ;;
         --disable-ruby)
             append_cache_entry DISABLE_RUBY_BINDINGS     BOOL   true
             ;;
+        --disable-dataseries)
+            append_cache_entry DISABLE_DATASERIES     BOOL   true
+            ;;
+        --disable-elasticsearch)
+            append_cache_entry DISABLE_ELASTICSEARCH     BOOL   true
+            ;;
         --with-openssl=*)
             append_cache_entry OpenSSL_ROOT_DIR PATH $optarg
             ;;
@@ -213,6 +236,16 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-dataseries=*)
+            append_cache_entry DataSeries_ROOT_DIR PATH $optarg
+            append_cache_entry Lintel_ROOT_DIR PATH $optarg
+            ;;
+        --with-xml2=*)
+            append_cache_entry LibXML2_ROOT_DIR PATH $optarg
+            ;;
+        --with-curl=*)
+            append_cache_entry LibCURL_ROOT_DIR PATH $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

doc/ext/bro_lexer/bro.py

@@ -29,7 +29,7 @@ class BroLexer(RegexLexer):
              r'|vector)\b', Keyword.Type),
             (r'(T|F)\b', Keyword.Constant),
             (r'(&)((?:add|delete|expire)_func|attr|(create|read|write)_expire'
-             r'|default|disable_print_hook|raw_output|encrypt|group|log'
+             r'|default|raw_output|encrypt|group|log'
              r'|mergeable|optional|persistent|priority|redef'
              r'|rotate_(?:interval|size)|synchronized)\b', bygroups(Punctuation,
                  Keyword)),

doc/faq.rst

@@ -46,7 +46,7 @@ directions:
 http://securityonion.blogspot.com/2011/10/when-is-full-packet-capture-not-full.html
 
 What does an error message like ``internal error: NB-DNS error`` mean?
----------------------------------------------------------------------------------------------------------------------------------
+----------------------------------------------------------------------
 
 That often means that DNS is not set up correctly on the system
 running Bro. Try verifying from the command line that DNS lookups
@@ -65,6 +65,15 @@ Generally, please note that we do not regularly test OpenBSD builds.
 We appreciate any patches that improve Bro's support for this
 platform.
 
+How do BroControl options affect Bro script variables?
+------------------------------------------------------
+
+Some (but not all) BroControl options override a corresponding Bro script variable.
+For example, setting the BroControl option "LogRotationInterval" will override
+the value of the Bro script variable "Log::default_rotation_interval".
+See the :doc:`BroControl Documentation <components/broctl/README>` to find out
+which BroControl options override Bro script variables, and for more discussion
+on site-specific customization.
 
 Usage
 =====

doc/index.rst

@@ -11,8 +11,8 @@ Guides
    :maxdepth: 1
 
    INSTALL
-   quickstart
    upgrade
+   quickstart
    faq
    reporting-problems
 
@@ -24,6 +24,7 @@ Frameworks
 
    notice
    logging
+   input
    cluster
    signatures
 

doc/input.rst

@@ -0,0 +1,407 @@
+==============================================
+Loading Data into Bro with the Input Framework
+==============================================
+
+.. rst-class:: opening
+
+   Bro now features a flexible input framework that allows users
+   to import data into Bro. Data is either read into Bro tables or
+   converted to events which can then be handled by scripts.
+   This document gives an overview of how to use the input framework
+   with some examples. For more complex scenarios it is
+   worthwhile to take a look at the unit tests in
+   ``testing/btest/scripts/base/frameworks/input/``.
+
+.. contents::
+
+Reading Data into Tables
+========================
+
+Probably the most interesting use-case of the input framework is to
+read data into a Bro table.
+
+By default, the input framework reads the data in the same format
+as it is written by the logging framework in Bro - a tab-separated
+ASCII file.
+
+We will show the ways to read files into Bro with a simple example.
+For this example we assume that we want to import data from a blacklist
+that contains server IP addresses as well as the timestamp and the reason
+for the block.
+
+An example input file could look like this:
+
+::
+
+        #fields ip timestamp reason
+        192.168.17.1 1333252748 Malware host
+        192.168.27.2 1330235733 Botnet server
+        192.168.250.3 1333145108 Virus detected
+
+To read a file into a Bro table, two record types have to be defined.
+One contains the types and names of the columns that should constitute the
+table keys and the second contains the types and names of the columns that
+should constitute the table values.
+
+In our case, we want to be able to lookup IPs. Hence, our key record
+only contains the server IP. All other elements should be stored as
+the table content.
+
+The two records are defined as:
+
+.. code:: bro
+
+        type Idx: record {
+                ip: addr;
+        };
+
+        type Val: record {
+                timestamp: time;
+                reason: string;
+        };
+
+Note that the names of the fields in the record definitions have to correspond
+to the column names listed in the '#fields' line of the log file, in this
+case 'ip', 'timestamp', and 'reason'.
+
+The log file is read into the table with a simple call of the ``add_table``
+function:
+
+.. code:: bro
+
+        global blacklist: table[addr] of Val = table();
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist]);
+        Input::remove("blacklist");
+
+With these three lines we first create an empty table that should contain the
+blacklist data and then instruct the input framework to open an input stream
+named ``blacklist`` to read the data into the table. The third line removes the
+input stream again, because we do not need it any more after the data has been
+read.
+
+Because some data files can - potentially - be rather big, the input framework
+works asynchronously. A new thread is created for each new input stream.
+This thread opens the input data file, converts the data into a Bro format and
+sends it back to the main Bro thread.
+
+Because of this, the data is not immediately accessible. Depending on the
+size of the data source it might take from a few milliseconds up to a few
+seconds until all data is present in the table. Please note that this means
+that when Bro is running without an input source or on very short captured
+files, it might terminate before the data is present in the system (because
+Bro already handled all packets before the import thread finished).
+
+Subsequent calls to an input source are queued until the previous action has
+been completed. Because of this, it is, for example, possible to call
+``add_table`` and ``remove`` in two subsequent lines: the ``remove`` action
+will remain queued until the first read has been completed.
+
+Once the input framework finishes reading from a data source, it fires
+the ``end_of_data`` event. Once this event has been received all data
+from the input file is available in the table.
+
+.. code:: bro
+
+        event Input::end_of_data(name: string, source: string) {
+                # now all data is in the table
+                print blacklist;
+        }
+
+The table can also already be used while the data is still being read - it
+just might not contain all lines in the input file when the event has not
+yet fired. After it has been populated it can be used like any other Bro
+table and blacklist entries can easily be tested:
+
+.. code:: bro
+
+        if ( 192.168.18.12 in blacklist )
+                # take action
+
+
+Re-reading and streaming data
+-----------------------------
+
+For many data sources, like for many blacklists, the source data is continually
+changing. For these cases, the Bro input framework supports several ways to
+deal with changing data files.
+
+The first, very basic method is an explicit refresh of an input stream. When
+an input stream is open, the function ``force_update`` can be called. This
+will trigger a complete refresh of the table; any changed elements from the
+file will be updated.  After the update is finished the ``end_of_data``
+event will be raised.
+
+In our example the call would look like:
+
+.. code:: bro
+
+        Input::force_update("blacklist");
+
+The input framework also supports two automatic refresh modes. The first mode
+continually checks if a file has been changed. If the file has been changed, it
+is re-read and the data in the Bro table is updated to reflect the current
+state.  Each time a change has been detected and all the new data has been
+read into the table, the ``end_of_data`` event is raised.
+
+The second mode is a streaming mode. This mode assumes that the source data
+file is an append-only file to which new data is continually appended. Bro
+continually checks for new data at the end of the file and will add the new
+data to the table.  If newer lines in the file have the same index as previous
+lines, they will overwrite the values in the output table.  Because of the
+nature of streaming reads (data is continually added to the table),
+the ``end_of_data`` event is never raised when using streaming reads.
+
+The reading mode can be selected by setting the ``mode`` option of the
+add_table call.  Valid values are ``MANUAL`` (the default), ``REREAD``
+and ``STREAM``.
+
+Hence, when adding ``$mode=Input::REREAD`` to the previous example, the
+blacklist table will always reflect the state of the blacklist input file.
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD]);
+
+Receiving change events
+-----------------------
+
+When re-reading files, it might be interesting to know exactly which lines in
+the source files have changed.
+
+For this reason, the input framework can raise an event each time when a data
+item is added to, removed from or changed in a table.
+
+The event definition looks like this:
+
+.. code:: bro
+
+        event entry(description: Input::TableDescription, tpe: Input::Event, left: Idx, right: Val) {
+                # act on values
+        }
+
+The event has to be specified in ``$ev`` in the ``add_table`` call:
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD, $ev=entry]);
+
+The ``description`` field of the event contains the arguments that were
+originally supplied to the add_table call.  Hence, the name of the stream can,
+for example, be accessed with ``description$name``. ``tpe`` is an enum
+containing the type of the change that occurred.
+
+If a line that was not previously present in the table has been added,
+then ``tpe`` will contain ``Input::EVENT_NEW``. In this case ``left`` contains
+the index of the added table entry and ``right`` contains the values of the
+added entry.
+
+If a table entry that already was present is altered during the re-reading or
+streaming read of a file, ``tpe`` will contain ``Input::EVENT_CHANGED``. In
+this case ``left`` contains the index of the changed table entry and ``right``
+contains the values of the entry before the change. The reason for this is
+that the table already has been updated when the event is raised. The current
+value in the table can be ascertained by looking up the current table value.
+Hence it is possible to compare the new and the old values of the table.
+
+If a table element is removed because it was no longer present during a
+re-read, then ``tpe`` will contain ``Input::REMOVED``.  In this case ``left``
+contains the index and ``right`` the values of the removed element.
+
+
+Filtering data during import
+----------------------------
+
+The input framework also allows a user to filter the data during the import.
+To this end, predicate functions are used. A predicate function is called
+before a new element is added/changed/removed from a table. The predicate
+can either accept or veto the change by returning true for an accepted
+change and false for a rejected change. Furthermore, it can alter the data
+before it is written to the table.
+
+The following example filter will reject to add entries to the table when
+they were generated over a month ago. It will accept all changes and all
+removals of values that are already present in the table.
+
+.. code:: bro
+
+        Input::add_table([$source="blacklist.file", $name="blacklist", $idx=Idx, $val=Val, $destination=blacklist, $mode=Input::REREAD,
+                        $pred(typ: Input::Event, left: Idx, right: Val) = {
+                                if ( typ != Input::EVENT_NEW ) {
+                                        return T;
+                                }
+                                return ( ( current_time() - right$timestamp ) < (30 day) );
+                        }]);
+
+To change elements while they are being imported, the predicate function can
+manipulate ``left`` and ``right``. Note that predicate functions are called
+before the change is committed to the table. Hence, when a table element is
+changed (``tpe`` is ``INPUT::EVENT_CHANGED``), ``left`` and ``right``
+contain the new values, but the destination (``blacklist`` in our example)
+still contains the old values. This allows predicate functions to examine
+the changes between the old and the new version before deciding if they
+should be allowed.
+
+Different readers
+-----------------
+
+The input framework supports different kinds of readers for different kinds
+of source data files. At the moment, the default reader reads ASCII files
+formatted in the Bro log file format (tab-separated values). At the moment,
+Bro comes with two other readers. The ``RAW`` reader reads a file that is
+split by a specified record separator (usually newline). The contents are
+returned line-by-line as strings; it can, for example, be used to read
+configuration files and the like and is probably
+only useful in the event mode and not for reading data to tables.
+
+Another included reader is the ``BENCHMARK`` reader, which is being used
+to optimize the speed of the input framework. It can generate arbitrary
+amounts of semi-random data in all Bro data types supported by the input
+framework.
+
+In the future, the input framework will get support for new data sources
+like, for example, different databases.
+
+Add_table options
+-----------------
+
+This section lists all possible options that can be used for the add_table
+function and gives a short explanation of their use. Most of the options
+already have been discussed in the previous sections.
+
+The possible fields that can be set for a table stream are:
+
+        ``source``
+                A mandatory string identifying the source of the data.
+                For the ASCII reader this is the filename.
+
+        ``name``
+                A mandatory name for the filter that can later be used
+                to manipulate it further.
+
+        ``idx``
+                Record type that defines the index of the table.
+
+        ``val``
+                Record type that defines the values of the table.
+
+        ``reader``
+                The reader used for this stream. Default is ``READER_ASCII``.
+
+        ``mode``
+                The mode in which the stream is opened. Possible values are
+                ``MANUAL``, ``REREAD`` and ``STREAM``.  Default is ``MANUAL``.
+                ``MANUAL`` means that the file is not updated after it has
+                been read. Changes to the file will not be reflected in the
+                data Bro knows.  ``REREAD`` means that the whole file is read
+                again each time a change is found. This should be used for
+                files that are mapped to a table where individual lines can
+                change.  ``STREAM`` means that the data from the file is
+                streamed. Events / table entries will be generated as new
+                data is appended to the file.
+
+        ``destination``
+                The destination table.
+
+        ``ev``
+                Optional event that is raised, when values are added to,
+                changed in, or deleted from the table.  Events are passed an
+                Input::Event description as the first argument, the index
+                record as the second argument and the values as the third
+                argument.
+
+        ``pred``
+                Optional predicate, that can prevent entries from being added
+                to the table and events from being sent.
+
+        ``want_record``
+                Boolean value, that defines if the event wants to receive the
+                fields inside of a single record value, or individually
+                (default).  This can be used if ``val`` is a record
+                containing only one type. In this case, if ``want_record`` is
+                set to false, the table will contain elements of the type
+                contained in ``val``.
+
+Reading Data to Events
+======================
+
+The second supported mode of the input framework is reading data to Bro
+events instead of reading them to a table using event streams.
+
+Event streams work very similarly to table streams that were already
+discussed in much detail. To read the blacklist of the previous example
+into an event stream, the following Bro code could be used:
+
+.. code:: bro
+
+        type Val: record {
+                ip: addr;
+                timestamp: time;
+                reason: string;
+        };
+
+        event blacklistentry(description: Input::EventDescription, tpe: Input::Event, ip: addr, timestamp: time, reason: string) {
+                # work with event data
+        }
+
+        event bro_init() {
+                Input::add_event([$source="blacklist.file", $name="blacklist", $fields=Val, $ev=blacklistentry]);
+        }
+
+
+The main difference in the declaration of the event stream is, that an event
+stream needs no separate index and value declarations -- instead, all source
+data types are provided in a single record definition.
+
+Apart from this, event streams work exactly the same as table streams and
+support most of the options that are also supported for table streams.
+
+The options that can be set when creating an event stream with
+``add_event`` are:
+
+        ``source``
+                A mandatory string identifying the source of the data.
+                For the ASCII reader this is the filename.
+
+        ``name``
+                A mandatory name for the stream that can later be used
+                to remove it.
+
+        ``fields``
+                Name of a record type containing the fields, which should be
+                retrieved from the input stream.
+
+        ``ev``
+                The event which is fired, after a line has been read from the
+                input source.  The first argument that is passed to the event
+                is an Input::Event structure, followed by the data, either
+                inside of a record (if ``want_record is set``) or as
+                individual fields.  The Input::Event structure can contain
+                information, if the received line is ``NEW``, has been
+                ``CHANGED`` or ``DELETED``. Since the ASCII reader cannot
+                track this information for event filters, the value is
+                always ``NEW`` at the moment.
+
+        ``mode``
+                The mode in which the stream is opened. Possible values are
+                ``MANUAL``, ``REREAD`` and ``STREAM``.  Default is ``MANUAL``.
+                ``MANUAL`` means that the file is not updated after it has
+                been read. Changes to the file will not be reflected in the
+                data Bro knows.  ``REREAD`` means that the whole file is read
+                again each time a change is found. This should be used for
+                files that are mapped to a table where individual lines can
+                change.  ``STREAM`` means that the data from the file is
+                streamed. Events / table entries will be generated as new
+                data is appended to the file.
+
+        ``reader``
+                The reader used for this stream. Default is ``READER_ASCII``.
+
+        ``want_record``
+                Boolean value, that defines if the event wants to receive the
+                fields inside of a single record value, or individually
+                (default). If this is set to true, the event will receive a
+                single record of the type provided in ``fields``.
+
+
+

doc/intel.rst

@@ -0,0 +1,125 @@
+Intel Framework
+===============
+
+Intro
+-----
+
+Intelligence data is critical to the process of monitoring for
+security purposes.  There is always data which will be discovered
+through the incident response process and data which is shared through
+private communities.  The goals of Bro's Intelligence Framework are to
+consume that data, make it available for matching, and provide
+infrastructure around improving performance, memory utilization, and
+generally making all of this easier.
+
+Data in the Intelligence Framework is the atomic piece of intelligence
+such as an IP address or an e-mail address along with a suite of
+metadata about it such as a freeform source field, a freeform
+descriptive field and a URL which might lead to more information about
+the specific item.  The metadata in the default scripts has been
+deliberately kept minimal so that the community can find the
+appropriate fields that need added by writing scripts which extend the
+base record using the normal record extension mechanism.
+
+Quick Start
+-----------
+
+Load the package of scripts that sends data into the Intelligence
+Framework to be checked by loading this script in local.bro::
+
+	@load policy/frameworks/intel
+
+(TODO: find some good mechanism for getting setup with good data
+quickly)
+
+Refer to the "Loading Intelligence" section below to see the format
+for Intelligence Framework text files, then load those text files with
+this line in local.bro::
+
+	redef Intel::read_files += { "/somewhere/yourdata.txt" };
+
+The data itself only needs to reside on the manager if running in a
+cluster.
+
+Architecture
+------------
+
+The Intelligence Framework can be thought of as containing three
+separate portions.  The first part is how intelligence is loaded,
+followed by the mechanism for indicating to the intelligence framework
+that a piece of data which needs to be checked has been seen, and
+thirdly the part where a positive match has been discovered.
+
+Loading Intelligence
+********************
+
+Intelligence data can only be loaded through plain text files using
+the Input Framework conventions.  Additionally, on clusters the
+manager is the only node that needs the intelligence data.  The
+intelligence framework has distribution mechanisms which will push
+data out to all of the nodes that need it.
+
+Here is an example of the intelligence data format.  Note that all
+whitespace separators are literal tabs and fields containing only a
+hyphen a considered to be null values.::
+
+	#fields	host	net	str	str_type	meta.source	meta.desc	meta.url
+	1.2.3.4	-	-	-	source1	Sending phishing email	http://source1.com/badhosts/1.2.3.4
+	-	31.131.248.0/21	-	-	spamhaus-drop	SBL154982	-	-
+	-	-	a.b.com	Intel::DOMAIN	source2	Name used for data exfiltration	-
+
+For more examples of built in `str_type` values, please refer to the
+autogenerated documentation for the intelligence framework (TODO:
+figure out how to do this link).
+
+To load the data once files are created, use the following example
+code to define files to load with your own file names of course::
+
+	redef Intel::read_files += {
+		"/somewhere/feed1.txt",
+		"/somewhere/feed2.txt",
+	};
+
+Remember, the files only need to be present on the file system of the
+manager node on cluster deployments.
+
+Seen Data
+*********
+
+When some bit of data is extracted (such as an email address in the
+"From" header in a message over SMTP), the Intelligence Framework
+needs to be informed that this data was discovered and it's presence
+should be checked within the intelligence data set.  This is
+accomplished through the Intel::seen (TODO: do a reference link)
+function.
+
+Typically users won't need to work with this function due to built in
+hook scripts that Bro ships with that will "see" data and send it into
+the intelligence framework.  A user may only need to load the entire
+package of hook scripts as a module or pick and choose specific
+scripts to load.  Keep in mind that as more data is sent into the
+intelligence framework, the CPU load consumed by Bro will increase
+depending on how many times the Intel::seen function is being called
+which is heavily traffic dependent.
+
+The full package of hook scripts that Bro ships with for sending this
+"seen" data into the intelligence framework can be loading by adding
+this line to local.bro::
+
+	@load policy/frameworks/intel
+
+Intelligence Matches
+********************
+
+Against all hopes, most networks will eventually have a hit on
+intelligence data which could indicate a possible compromise or other
+unwanted activity.  The Intelligence Framework provides an event that
+is generated whenever a match is discovered named Intel::match (TODO:
+make a link to inline docs).  Due to design restrictions placed upon
+the intelligence framework, there is no assurance as to where this
+event will be generated.  It could be generated on the worker where
+the data was seen or on the manager.  When the Intel::match event is
+handled, only the data given as event arguments to the event can be
+assured since the host where the data was seen may not be where
+Intel::match is handled.
+

doc/logging-dataseries.rst

@@ -0,0 +1,186 @@
+
+=============================
+Binary Output with DataSeries
+=============================
+
+.. rst-class:: opening
+
+   Bro's default ASCII log format is not exactly the most efficient
+   way for storing and searching large volumes of data. An an
+   alternative, Bro comes with experimental support for `DataSeries
+   <http://www.hpl.hp.com/techreports/2009/HPL-2009-323.html>`_
+   output, an efficient binary format for recording structured bulk
+   data. DataSeries is developed and maintained at HP Labs.
+
+.. contents::
+
+Installing DataSeries
+---------------------
+
+To use DataSeries, its libraries must be available at compile-time,
+along with the supporting *Lintel* package. Generally, both are
+distributed on `HP Labs' web site
+<http://tesla.hpl.hp.com/opensource/>`_. Currently, however, you need
+to use recent development versions for both packages, which you can
+download from github like this::
+
+    git clone http://github.com/dataseries/Lintel
+    git clone http://github.com/dataseries/DataSeries
+
+To build and install the two into ``<prefix>``, do::
+
+    ( cd Lintel     && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
+    ( cd DataSeries && mkdir build && cd build && cmake -DCMAKE_INSTALL_PREFIX=<prefix> .. && make && make install )
+
+Please refer to the packages' documentation for more information about
+the installation process. In particular, there's more information on
+required and optional `dependencies for Lintel
+<https://raw.github.com/dataseries/Lintel/master/doc/dependencies.txt>`_
+and `dependencies for DataSeries
+<https://raw.github.com/dataseries/DataSeries/master/doc/dependencies.txt>`_.
+For users on RedHat-style systems, you'll need the following::
+
+    yum install libxml2-devel boost-devel
+
+Compiling Bro with DataSeries Support
+-------------------------------------
+
+Once you have installed DataSeries, Bro's ``configure`` should pick it
+up automatically as long as it finds it in a standard system location.
+Alternatively, you can specify the DataSeries installation prefix
+manually with ``--with-dataseries=<prefix>``. Keep an eye on
+``configure``'s summary output, if it looks like the following, Bro
+found DataSeries and will compile in the support::
+
+    # ./configure --with-dataseries=/usr/local
+    [...]
+    ====================|  Bro Build Summary  |=====================
+    [...]
+    DataSeries:        true
+    [...]
+    ================================================================
+
+Activating DataSeries
+---------------------
+
+The direct way to use DataSeries is to switch *all* log files over to
+the binary format. To do that, just add ``redef
+Log::default_writer=Log::WRITER_DATASERIES;`` to your ``local.bro``.
+For testing, you can also just pass that on the command line::
+
+    bro -r trace.pcap Log::default_writer=Log::WRITER_DATASERIES
+
+With that, Bro will now write all its output into DataSeries files
+``*.ds``. You can inspect these using DataSeries's set of command line
+tools, which its installation process installs into ``<prefix>/bin``.
+For example, to convert a file back into an ASCII representation::
+
+    $ ds2txt conn.log
+    [... We skip a bunch of metadata here ...]
+    ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes
+    1300475167.096535 CRCC5OdDlXe 141.142.220.202 5353 224.0.0.251 5353 udp dns 0.000000 0 0 S0 F 0 D 1 73 0 0
+    1300475167.097012 o7XBsfvo3U1 fe80::217:f2ff:fed7:cf65 5353 ff02::fb 5353 udp  0.000000 0 0 S0 F 0 D 1 199 0 0
+    1300475167.099816 pXPi1kPMgxb 141.142.220.50 5353 224.0.0.251 5353 udp  0.000000 0 0 S0 F 0 D 1 179 0 0
+    1300475168.853899 R7sOc16woCj 141.142.220.118 43927 141.142.2.2 53 udp dns 0.000435 38 89 SF F 0 Dd 1 66 1 117
+    1300475168.854378 Z6dfHVmt0X7 141.142.220.118 37676 141.142.2.2 53 udp dns 0.000420 52 99 SF F 0 Dd 1 80 1 127
+    1300475168.854837 k6T92WxgNAh 141.142.220.118 40526 141.142.2.2 53 udp dns 0.000392 38 183 SF F 0 Dd 1 66 1 211
+    [...]
+
+(``--skip-all`` suppresses the metadata.)
+
+Note that the ASCII conversion is *not* equivalent to Bro's default
+output format.
+
+You can also switch only individual files over to DataSeries by adding
+code like this to your ``local.bro``:
+
+.. code:: bro
+
+    event bro_init()
+        {
+        local f = Log::get_filter(Conn::LOG, "default"); # Get default filter for connection log.
+        f$writer = Log::WRITER_DATASERIES;               # Change writer type.
+        Log::add_filter(Conn::LOG, f);                   # Replace filter with adapted version.
+        }
+
+Bro's DataSeries writer comes with a few tuning options, see
+:doc:`scripts/base/frameworks/logging/writers/dataseries`.
+
+Working with DataSeries
+=======================
+
+Here are a few examples of using DataSeries command line tools to work
+with the output files.
+
+* Printing CSV::
+
+    $ ds2txt --csv conn.log
+    ts,uid,id.orig_h,id.orig_p,id.resp_h,id.resp_p,proto,service,duration,orig_bytes,resp_bytes,conn_state,local_orig,missed_bytes,history,orig_pkts,orig_ip_bytes,resp_pkts,resp_ip_bytes
+    1258790493.773208,ZTtgbHvf4s3,192.168.1.104,137,192.168.1.255,137,udp,dns,3.748891,350,0,S0,F,0,D,7,546,0,0
+    1258790451.402091,pOY6Rw7lhUd,192.168.1.106,138,192.168.1.255,138,udp,,0.000000,0,0,S0,F,0,D,1,229,0,0
+    1258790493.787448,pn5IiEslca9,192.168.1.104,138,192.168.1.255,138,udp,,2.243339,348,0,S0,F,0,D,2,404,0,0
+    1258790615.268111,D9slyIu3hFj,192.168.1.106,137,192.168.1.255,137,udp,dns,3.764626,350,0,S0,F,0,D,7,546,0,0
+    [...]
+
+  Add ``--separator=X`` to set a different separator.
+
+* Extracting a subset of columns::
+
+    $ ds2txt --select '*' ts,id.resp_h,id.resp_p --skip-all conn.log
+    1258790493.773208 192.168.1.255 137
+    1258790451.402091 192.168.1.255 138
+    1258790493.787448 192.168.1.255 138
+    1258790615.268111 192.168.1.255 137
+    1258790615.289842 192.168.1.255 138
+    [...]
+
+* Filtering rows::
+
+    $ ds2txt --where '*' 'duration > 5 && id.resp_p > 1024' --skip-all  conn.ds
+    1258790631.532888 V8mV5WLITu5 192.168.1.105 55890 239.255.255.250 1900 udp  15.004568 798 0 S0 F 0 D 6 966 0 0
+    1258792413.439596 tMcWVWQptvd 192.168.1.105 55890 239.255.255.250 1900 udp  15.004581 798 0 S0 F 0 D 6 966 0 0
+    1258794195.346127 cQwQMRdBrKa 192.168.1.105 55890 239.255.255.250 1900 udp  15.005071 798 0 S0 F 0 D 6 966 0 0
+    1258795977.253200 i8TEjhWd2W8 192.168.1.105 55890 239.255.255.250 1900 udp  15.004824 798 0 S0 F 0 D 6 966 0 0
+    1258797759.160217 MsLsBA8Ia49 192.168.1.105 55890 239.255.255.250 1900 udp  15.005078 798 0 S0 F 0 D 6 966 0 0
+    1258799541.068452 TsOxRWJRGwf 192.168.1.105 55890 239.255.255.250 1900 udp  15.004082 798 0 S0 F 0 D 6 966 0 0
+    [...]
+
+* Calculate some statistics:
+
+    Mean/stddev/min/max over a column::
+
+        $ dsstatgroupby '*' basic duration from conn.ds
+        # Begin DSStatGroupByModule
+        # processed 2159 rows, where clause eliminated 0 rows
+        # count(*), mean(duration), stddev, min, max
+        2159, 42.7938, 1858.34, 0, 86370
+        [...]
+
+    Quantiles of total connection volume::
+
+        $ dsstatgroupby '*' quantile 'orig_bytes + resp_bytes' from conn.ds
+        [...]
+        2159 data points, mean 24616 +- 343295 [0,1.26615e+07]
+        quantiles about every 216 data points:
+        10%: 0, 124, 317, 348, 350, 350, 601, 798, 1469
+        tails: 90%: 1469, 95%: 7302, 99%: 242629, 99.5%: 1226262
+        [...]
+
+The ``man`` pages for these tools show further options, and their
+``-h`` option gives some more information (either can be a bit cryptic
+unfortunately though).
+
+Deficiencies
+------------
+
+Due to limitations of the DataSeries format, one cannot inspect its
+files before they have been fully written. In other words, when using
+DataSeries, it's currently not possible to inspect the live log
+files inside the spool directory before they are rotated to their
+final location. It seems that this could be fixed with some effort,
+and we will work with DataSeries development team on that if the
+format gains traction among Bro users.
+
+Likewise, we're considering writing custom command line tools for
+interacting with DataSeries files, making that a bit more convenient
+than what the standard utilities provide.

doc/logging-elasticsearch.rst

@@ -0,0 +1,89 @@
+
+=========================================
+Indexed Logging Output with ElasticSearch
+=========================================
+
+.. rst-class:: opening
+
+   Bro's default ASCII log format is not exactly the most efficient
+   way for searching large volumes of data. ElasticSearch
+   is a new data storage technology for dealing with tons of data.
+   It's also a search engine built on top of Apache's Lucene
+   project. It scales very well, both for distributed indexing and 
+   distributed searching.
+
+.. contents::
+
+Warning
+-------
+
+This writer plugin is still in testing and is not yet recommended for
+production use!  The approach to how logs are handled in the plugin is "fire
+and forget" at this time, there is no error handling if the server fails to
+respond successfully to the insertion request.
+
+Installing ElasticSearch
+------------------------
+
+Download the latest version from: <http://www.elasticsearch.org/download/>.
+Once extracted, start ElasticSearch with::
+
+# ./bin/elasticsearch
+
+For more detailed information, refer to the ElasticSearch installation
+documentation: http://www.elasticsearch.org/guide/reference/setup/installation.html
+
+Compiling Bro with ElasticSearch Support
+----------------------------------------
+
+First, ensure that you have libcurl installed the run configure.::
+
+    # ./configure
+    [...]
+    ====================|  Bro Build Summary  |=====================
+    [...]
+    cURL:              true
+    [...]
+    ElasticSearch:     true
+    [...]
+    ================================================================
+
+Activating ElasticSearch
+------------------------
+
+The easiest way to enable ElasticSearch output is to load the tuning/logs-to-
+elasticsearch.bro script.  If you are using BroControl, the following line in
+local.bro will enable it.
+
+.. console::
+
+    @load tuning/logs-to-elasticsearch
+
+With that, Bro will now write most of its logs into ElasticSearch in addition
+to maintaining the Ascii logs like it would do by default.  That script has
+some tunable options for choosing which logs to send to ElasticSearch, refer
+to the autogenerated script documentation for those options.
+
+There is an interface being written specifically to integrate with the data
+that Bro outputs into ElasticSearch named Brownian.  It can be found here::
+
+    https://github.com/grigorescu/Brownian
+
+Tuning
+------
+
+A common problem encountered with ElasticSearch is too many files being held
+open.  The ElasticSearch website has some suggestions on how to increase the
+open file limit.
+
+  - http://www.elasticsearch.org/tutorials/2011/04/06/too-many-open-files.html
+
+TODO
+----
+
+Lots.
+
+- Perform multicast discovery for server.
+- Better error detection.
+- Better defaults (don't index loaded-plugins, for instance).
+- 

doc/logging.rst

@@ -373,3 +373,14 @@ record, care must be given to when and how long data is stored.
 Normally data saved to the connection record will remain there for the
 duration of the connection and from a practical perspective it's not
 uncommon to need to delete that data before the end of the connection.
+
+Other Writers
+-------------
+
+Bro supports the following output formats other than ASCII:
+
+.. toctree::
+   :maxdepth: 1
+
+   logging-dataseries
+   logging-elasticsearch

doc/quickstart.rst

@@ -1,9 +1,3 @@
-.. _CMake: http://www.cmake.org
-.. _SWIG: http://www.swig.org
-.. _MacPorts: http://www.macports.org
-.. _Fink: http://www.finkproject.org
-.. _Homebrew: http://mxcl.github.com/homebrew
-.. _bro downloads page: http://bro-ids.org/download/index.html
 
 =================
 Quick Start Guide
@@ -22,178 +16,11 @@ Installation
 
 Bro works on most modern, Unix-based systems and requires no custom
 hardware.  It can be downloaded in either pre-built binary package or
-source code forms.
+source code forms.  See :doc:`Installing Bro <INSTALL>` for instructions
+on how to install Bro.
 
-Pre-Built Binary Release Packages
+.. note:: Below, ``$PREFIX`` is used to reference the Bro installation
----------------------------------
+   root directory.
-
-See the `bro downloads page`_ for currently supported/targeted platforms.
-
-* RPM
-
-  .. console::
-
-      sudo yum localinstall Bro-*.rpm
-
-* DEB
-
-  .. console::
-
-      sudo gdebi Bro-*.deb
-
-* MacOS Disk Image with Installer
-
-  Just open the ``Bro-*.dmg`` and then run the ``.pkg`` installer.
-  Everything installed by the package will go into ``/opt/bro``.
-
-The primary install prefix for binary packages is ``/opt/bro``.
-Non-MacOS packages that include BroControl also put variable/runtime
-data (e.g. Bro logs) in ``/var/opt/bro``.
-
-Building From Source
---------------------
-
-Required Dependencies
-~~~~~~~~~~~~~~~~~~~~~
-
-The following dependencies are required to build Bro:
-
-* RPM/RedHat-based Linux:
-
-  .. console::
-
-     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
-
-* DEB/Debian-based Linux:
-
-  .. console::
-
-     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
-
-* FreeBSD
-
-  Most required dependencies should come with a minimal FreeBSD install
-  except for the following.
-
-  .. console::
-
-      sudo pkg_add -r bash cmake swig bison python
-
-  Note that ``bash`` needs to be in ``PATH``, which by default it is
-  not. The FreeBSD package installs the binary into
-  ``/usr/local/bin``.
-
-* Mac OS X
-
-  Snow Leopard (10.6) comes with all required dependencies except for CMake_.
-
-  Lion (10.7) comes with all required dependencies except for CMake_ and SWIG_.
-
-  Distributions of these dependencies can be obtained from the project websites
-  linked above, but they're also likely available from your preferred Mac OS X
-  package management system (e.g. MacPorts_, Fink_, or Homebrew_).
-
-  Note that the MacPorts ``swig`` package may not include any specific
-  language support so you may need to also install ``swig-ruby`` and
-  ``swig-python``.
-
-Optional Dependencies
-~~~~~~~~~~~~~~~~~~~~~
-
-Bro can use libGeoIP for geo-locating IP addresses, and sendmail for
-sending emails.
-
-* RedHat Enterprise Linux:
-
-  .. console::
-
-      sudo yum install geoip-devel sendmail
-
-* CentOS Linux:
-
-  .. console::
-  
-      sudo yum install GeoIP-devel sendmail
-
-* DEB/Debian-based Linux:
-
-  .. console::
-
-      sudo apt-get install libgeoip-dev sendmail
-
-* Ports-based FreeBSD
-
-  .. console::
-
-      sudo pkg_add -r GeoIP
-
-  sendmail is typically already available.
-
-* Mac OS X
-
-  Vanilla OS X installations don't ship with libmagic or libGeoIP, but
-  if installed from your preferred package management system (e.g. MacPorts,
-  Fink, or Homebrew), they should be automatically detected and Bro will compile
-  against them.
-
-Additional steps may be needed to :doc:`get the right GeoIP database <geoip>`
-
-Compiling Bro Source Code
-~~~~~~~~~~~~~~~~~~~~~~~~~
-
-Bro releases are bundled into source packages for convenience and
-available from the `bro downloads page`_.
-
-The latest Bro development versions are obtainable through git
-repositories hosted at `git.bro-ids.org <http://git.bro-ids.org>`_.  See
-our `git development documentation
-<http://bro-ids.org/development/process.html>`_ for comprehensive
-information on Bro's use of git revision control, but the short story
-for downloading the full source code experience for Bro via git is:
-
-.. console::
-
-    git clone --recursive git://git.bro-ids.org/bro
-
-.. note:: If you choose to clone the ``bro`` repository non-recursively for
-   a "minimal Bro experience", be aware that compiling it depends on
-   BinPAC, which has its own ``binpac`` repository.  Either install it
-   first or initialize/update the cloned ``bro`` repository's
-   ``aux/binpac`` submodule.
-
-See the ``INSTALL`` file included with the source code for more information
-on compiling, but this is the typical way to build and install from source
-(of course, changing the value of the ``--prefix`` option to point to the
-desired root install path):
-
-.. console::
-
-    ./configure --prefix=/desired/install/path
-    make
-    make install
-
-The default installation prefix is ``/usr/local/bro``, which would typically
-require root privileges when doing the ``make install``.
-
-Configure the Run-Time Environment
-----------------------------------
-
-Just remember that you may need to adjust your ``PATH`` environment variable
-according to the platform/shell/package you're using.  For example:
-
-Bourne-Shell Syntax:
-
-.. console::
-
-   export PATH=/usr/local/bro/bin:$PATH
-
-C-Shell Syntax:
-
-.. console::
-
-   setenv PATH /usr/local/bro/bin:$PATH
-
-Or substitute ``/opt/bro/bin`` instead if you installed from a binary package.
 
 Using BroControl
 ================
@@ -202,9 +29,6 @@ BroControl is an interactive shell for easily operating/managing Bro
 installations on a single system or even across multiple systems in a
 traffic-monitoring cluster.
 
-.. note:: Below, ``$PREFIX`` is used to reference the Bro installation
-   root directory.
-
 A Minimal Starting Configuration
 --------------------------------
 

doc/scripts/DocSourcesList.cmake

@@ -19,6 +19,7 @@ rest_target(${psd} base/init-bare.bro internal)
 rest_target(${CMAKE_BINARY_DIR}/src base/bro.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/const.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/event.bif.bro)
+rest_target(${CMAKE_BINARY_DIR}/src base/input.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/logging.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/reporter.bif.bro)
 rest_target(${CMAKE_BINARY_DIR}/src base/strings.bif.bro)
@@ -31,11 +32,20 @@ rest_target(${psd} base/frameworks/cluster/setup-connections.bro)
 rest_target(${psd} base/frameworks/communication/main.bro)
 rest_target(${psd} base/frameworks/control/main.bro)
 rest_target(${psd} base/frameworks/dpd/main.bro)
+rest_target(${psd} base/frameworks/input/main.bro)
+rest_target(${psd} base/frameworks/input/readers/ascii.bro)
+rest_target(${psd} base/frameworks/input/readers/benchmark.bro)
+rest_target(${psd} base/frameworks/input/readers/raw.bro)
+rest_target(${psd} base/frameworks/intel/cluster.bro)
+rest_target(${psd} base/frameworks/intel/input.bro)
 rest_target(${psd} base/frameworks/intel/main.bro)
 rest_target(${psd} base/frameworks/logging/main.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/scp.bro)
 rest_target(${psd} base/frameworks/logging/postprocessors/sftp.bro)
 rest_target(${psd} base/frameworks/logging/writers/ascii.bro)
+rest_target(${psd} base/frameworks/logging/writers/dataseries.bro)
+rest_target(${psd} base/frameworks/logging/writers/elasticsearch.bro)
+rest_target(${psd} base/frameworks/logging/writers/none.bro)
 rest_target(${psd} base/frameworks/metrics/cluster.bro)
 rest_target(${psd} base/frameworks/metrics/main.bro)
 rest_target(${psd} base/frameworks/metrics/non-cluster.bro)
@@ -53,12 +63,16 @@ rest_target(${psd} base/frameworks/packet-filter/netstats.bro)
 rest_target(${psd} base/frameworks/reporter/main.bro)
 rest_target(${psd} base/frameworks/signatures/main.bro)
 rest_target(${psd} base/frameworks/software/main.bro)
+rest_target(${psd} base/frameworks/tunnels/main.bro)
+rest_target(${psd} base/misc/find-checksum-offloading.bro)
 rest_target(${psd} base/protocols/conn/contents.bro)
 rest_target(${psd} base/protocols/conn/inactivity.bro)
 rest_target(${psd} base/protocols/conn/main.bro)
+rest_target(${psd} base/protocols/conn/polling.bro)
 rest_target(${psd} base/protocols/dns/consts.bro)
 rest_target(${psd} base/protocols/dns/main.bro)
 rest_target(${psd} base/protocols/ftp/file-extract.bro)
+rest_target(${psd} base/protocols/ftp/gridftp.bro)
 rest_target(${psd} base/protocols/ftp/main.bro)
 rest_target(${psd} base/protocols/ftp/utils-commands.bro)
 rest_target(${psd} base/protocols/http/file-extract.bro)
@@ -68,9 +82,13 @@ rest_target(${psd} base/protocols/http/main.bro)
 rest_target(${psd} base/protocols/http/utils.bro)
 rest_target(${psd} base/protocols/irc/dcc-send.bro)
 rest_target(${psd} base/protocols/irc/main.bro)
+rest_target(${psd} base/protocols/modbus/consts.bro)
+rest_target(${psd} base/protocols/modbus/main.bro)
 rest_target(${psd} base/protocols/smtp/entities-excerpt.bro)
 rest_target(${psd} base/protocols/smtp/entities.bro)
 rest_target(${psd} base/protocols/smtp/main.bro)
+rest_target(${psd} base/protocols/socks/consts.bro)
+rest_target(${psd} base/protocols/socks/main.bro)
 rest_target(${psd} base/protocols/ssh/main.bro)
 rest_target(${psd} base/protocols/ssl/consts.bro)
 rest_target(${psd} base/protocols/ssl/main.bro)
@@ -87,11 +105,21 @@ rest_target(${psd} base/utils/patterns.bro)
 rest_target(${psd} base/utils/site.bro)
 rest_target(${psd} base/utils/strings.bro)
 rest_target(${psd} base/utils/thresholds.bro)
+rest_target(${psd} base/utils/urls.bro)
 rest_target(${psd} policy/frameworks/communication/listen.bro)
 rest_target(${psd} policy/frameworks/control/controllee.bro)
 rest_target(${psd} policy/frameworks/control/controller.bro)
 rest_target(${psd} policy/frameworks/dpd/detect-protocols.bro)
 rest_target(${psd} policy/frameworks/dpd/packet-segment-logging.bro)
+rest_target(${psd} policy/frameworks/intel/conn-established.bro)
+rest_target(${psd} policy/frameworks/intel/dns.bro)
+rest_target(${psd} policy/frameworks/intel/http-host-header.bro)
+rest_target(${psd} policy/frameworks/intel/http-url.bro)
+rest_target(${psd} policy/frameworks/intel/http-user-agents.bro)
+rest_target(${psd} policy/frameworks/intel/smtp-url-extraction.bro)
+rest_target(${psd} policy/frameworks/intel/smtp.bro)
+rest_target(${psd} policy/frameworks/intel/ssl.bro)
+rest_target(${psd} policy/frameworks/intel/where-locations.bro)
 rest_target(${psd} policy/frameworks/metrics/conn-example.bro)
 rest_target(${psd} policy/frameworks/metrics/http-example.bro)
 rest_target(${psd} policy/frameworks/metrics/ssl-example.bro)
@@ -99,6 +127,7 @@ rest_target(${psd} policy/frameworks/software/version-changes.bro)
 rest_target(${psd} policy/frameworks/software/vulnerable.bro)
 rest_target(${psd} policy/integration/barnyard2/main.bro)
 rest_target(${psd} policy/integration/barnyard2/types.bro)
+rest_target(${psd} policy/integration/collective-intel/main.bro)
 rest_target(${psd} policy/misc/analysis-groups.bro)
 rest_target(${psd} policy/misc/capture-loss.bro)
 rest_target(${psd} policy/misc/loaded-scripts.bro)
@@ -113,7 +142,6 @@ rest_target(${psd} policy/protocols/dns/detect-external-names.bro)
 rest_target(${psd} policy/protocols/ftp/detect.bro)
 rest_target(${psd} policy/protocols/ftp/software.bro)
 rest_target(${psd} policy/protocols/http/detect-MHR.bro)
-rest_target(${psd} policy/protocols/http/detect-intel.bro)
 rest_target(${psd} policy/protocols/http/detect-sqli.bro)
 rest_target(${psd} policy/protocols/http/detect-webapps.bro)
 rest_target(${psd} policy/protocols/http/header-names.bro)
@@ -121,6 +149,8 @@ rest_target(${psd} policy/protocols/http/software-browser-plugins.bro)
 rest_target(${psd} policy/protocols/http/software.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-cookies.bro)
 rest_target(${psd} policy/protocols/http/var-extraction-uri.bro)
+rest_target(${psd} policy/protocols/modbus/known-masters-slaves.bro)
+rest_target(${psd} policy/protocols/modbus/track-memmap.bro)
 rest_target(${psd} policy/protocols/smtp/blocklists.bro)
 rest_target(${psd} policy/protocols/smtp/detect-suspicious-orig.bro)
 rest_target(${psd} policy/protocols/smtp/software.bro)
@@ -135,6 +165,7 @@ rest_target(${psd} policy/protocols/ssl/known-certs.bro)
 rest_target(${psd} policy/protocols/ssl/validate-certs.bro)
 rest_target(${psd} policy/tuning/defaults/packet-fragments.bro)
 rest_target(${psd} policy/tuning/defaults/warnings.bro)
+rest_target(${psd} policy/tuning/logs-to-elasticsearch.bro)
 rest_target(${psd} policy/tuning/track-all-assets.bro)
 rest_target(${psd} site/local-manager.bro)
 rest_target(${psd} site/local-proxy.bro)

doc/scripts/builtins.rst

@@ -55,8 +55,8 @@ The Bro scripting language supports the following built-in types.
 
     A temporal type representing a relative time.  An ``interval``
     constant can be written as a numeric constant followed by a time
-    unit where the time unit is one of ``usec``, ``sec``, ``min``,
+    unit where the time unit is one of ``usec``, ``msec``, ``sec``, ``min``,
-    ``hr``, or ``day`` which respectively represent microseconds,
+    ``hr``, or ``day`` which respectively represent microseconds, milliseconds,
     seconds, minutes, hours, and days.  Whitespace between the numeric
     constant and time unit is optional.  Appending the letter "s" to the
     time unit in order to pluralize it is also optional (to no semantic
@@ -95,14 +95,14 @@ The Bro scripting language supports the following built-in types.
     and embedded.
 
     In exact matching the ``==`` equality relational operator is used
-    with one :bro:type:`string` operand and one :bro:type:`pattern`
+    with one :bro:type:`pattern` operand and one :bro:type:`string`
-    operand to check whether the full string exactly matches the
+    operand (order of operands does not matter) to check whether the full
-    pattern.  In this case, the ``^`` beginning-of-line and ``$``
+    string exactly matches the pattern.  In exact matching, the ``^``
-    end-of-line anchors are redundant since pattern is implicitly
+    beginning-of-line and ``$`` end-of-line anchors are redundant since
-    anchored to the beginning and end of the line to facilitate an exact
+    the pattern is implicitly anchored to the beginning and end of the
-    match.  For example::
+    line to facilitate an exact match.  For example::
 
-        "foo" == /foo|bar/
+        /foo|bar/ == "foo"
 
     yields true, while::
 
@@ -110,9 +110,9 @@ The Bro scripting language supports the following built-in types.
 
     yields false.  The ``!=`` operator would yield the negation of ``==``.
 
-    In embedded matching the ``in`` operator is again used with one
+    In embedded matching the ``in`` operator is used with one
-    :bro:type:`string` operand and one :bro:type:`pattern` operand
+    :bro:type:`pattern` operand (which must be on the left-hand side) and
-    (which must be on the left-hand side), but tests whether the pattern
+    one :bro:type:`string` operand, but tests whether the pattern
     appears anywhere within the given string.  For example::
 
         /foo|bar/ in "foobar"
@@ -600,10 +600,6 @@ scripting language supports the following built-in attributes.
 
 .. TODO: needs to be documented.
 
-.. bro:attr:: &disable_print_hook
-
-    Deprecated. Will be removed.
-
 .. bro:attr:: &raw_output
 
     Opens a file in raw mode, i.e., non-ASCII characters are not

doc/signatures.rst

@@ -51,13 +51,18 @@ This script contains a default event handler that raises
 :bro:enum:`Signatures::Sensitive_Signature` :doc:`Notices <notice>`
 (as well as others; see the beginning of the script).
 
-As signatures are independent of Bro's policy scripts, they are put
+As signatures are independent of Bro's policy scripts, they are put into
-into their own file(s). There are two ways to specify which files
+their own file(s). There are three ways to specify which files contain
-contain signatures: By using the ``-s`` flag when you invoke Bro, or
+signatures: By using the ``-s`` flag when you invoke Bro, or by
-by extending the Bro variable :bro:id:`signature_files` using the ``+=``
+extending the Bro variable :bro:id:`signature_files` using the ``+=``
-operator. If a signature file is given without a path, it is searched
+operator, or by using the ``@load-sigs`` directive inside a Bro script.
-along the normal ``BROPATH``. The default extension of the file name
+If a signature file is given without a full path, it is searched for
-is ``.sig``, and Bro appends that automatically when necessary.
+along the normal ``BROPATH``.  Additionally, the ``@load-sigs``
+directive can be used to load signature files in a path relative to the
+Bro script in which it's placed, e.g. ``@load-sigs ./mysigs.sig`` will
+expect that signature file in the same directory as the Bro script. The
+default extension of the file name is ``.sig``, and Bro appends that
+automatically when necessary.
 
 Signature language
 ==================
@@ -78,9 +83,8 @@ Header Conditions
 ~~~~~~~~~~~~~~~~~
 
 Header conditions limit the applicability of the signature to a subset
-of traffic that contains matching packet headers. For TCP, this match
+of traffic that contains matching packet headers.  This type of matching
-is performed only for the first packet of a connection. For other
+is performed only for the first packet of a connection.
-protocols, it is done on each individual packet.
 
 There are pre-defined header conditions for some of the most used
 header fields. All of them generally have the format ``<keyword> <cmp>
@@ -90,14 +94,22 @@ one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``; and
 against. The following keywords are defined:
 
 ``src-ip``/``dst-ip <cmp> <address-list>``
-    Source and destination address, respectively. Addresses can be
+    Source and destination address, respectively. Addresses can be given
-    given as IP addresses or CIDR masks.
+    as IPv4 or IPv6 addresses or CIDR masks.  For IPv6 addresses/masks
+    the colon-hexadecimal representation of the address must be enclosed
+    in square brackets (e.g. ``[fe80::1]`` or ``[fe80::0]/16``).
 
-``src-port``/``dst-port`` ``<int-list>``
+``src-port``/``dst-port <cmp> <int-list>``
     Source and destination port, respectively.
 
-``ip-proto tcp|udp|icmp``
+``ip-proto <cmp> tcp|udp|icmp|icmp6|ip|ip6``
-    IP protocol.
+    IPv4 header's Protocol field or the Next Header field of the final
+    IPv6 header (i.e. either Next Header field in the fixed IPv6 header
+    if no extension headers are present or that field from the last
+    extension header in the chain).  Note that the IP-in-IP forms of
+    tunneling are automatically decapsulated by default and signatures
+    apply to only the inner-most packet, so specifying ``ip`` or ``ip6``
+    is a no-op.
 
 For lists of multiple values, they are sequentially compared against
 the corresponding header field. If at least one of the comparisons
@@ -111,20 +123,22 @@ condition can be defined either as
 
     header <proto>[<offset>:<size>] [& <integer>] <cmp> <value-list>
 
-This compares the value found at the given position of the packet
+This compares the value found at the given position of the packet header
-header with a list of values. ``offset`` defines the position of the
+with a list of values. ``offset`` defines the position of the value
-value within the header of the protocol defined by ``proto`` (which
+within the header of the protocol defined by ``proto`` (which can be
-can be ``ip``, ``tcp``, ``udp`` or ``icmp``). ``size`` is either 1, 2,
+``ip``, ``ip6``, ``tcp``, ``udp``, ``icmp`` or ``icmp6``). ``size`` is
-or 4 and specifies the value to have a size of this many bytes. If the
+either 1, 2, or 4 and specifies the value to have a size of this many
-optional ``& <integer>`` is given, the packet's value is first masked
+bytes. If the optional ``& <integer>`` is given, the packet's value is
-with the integer before it is compared to the value-list. ``cmp`` is
+first masked with the integer before it is compared to the value-list.
-one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``. ``value-list`` is
+``cmp`` is one of ``==``, ``!=``, ``<``, ``<=``, ``>``, ``>=``.
-a list of comma-separated integers similar to those described above.
+``value-list`` is a list of comma-separated integers similar to those
-The integers within the list may be followed by an additional ``/
+described above.  The integers within the list may be followed by an
-mask`` where ``mask`` is a value from 0 to 32. This corresponds to the
+additional ``/ mask`` where ``mask`` is a value from 0 to 32. This
-CIDR notation for netmasks and is translated into a corresponding
+corresponds to the CIDR notation for netmasks and is translated into a
-bitmask applied to the packet's value prior to the comparison (similar
+corresponding bitmask applied to the packet's value prior to the
-to the optional ``& integer``).
+comparison (similar to the optional ``& integer``).  IPv6 address values
+are not allowed in the value-list, though you can still inspect any 1,
+2, or 4 byte section of an IPv6 header using this keyword.
 
 Putting it all together, this is an example condition that is
 equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
@@ -133,8 +147,8 @@ equivalent to ``dst-ip == 1.2.3.4/16, 5.6.7.8/24``:
 
     header ip[16:4] == 1.2.3.4/16, 5.6.7.8/24
 
-Internally, the predefined header conditions are in fact just
+Note that the analogous example for IPv6 isn't currently possible since
-short-cuts and mapped into a generic condition.
+4 bytes is the max width of a value that can be compared.
 
 Content Conditions
 ~~~~~~~~~~~~~~~~~~
@@ -224,20 +238,10 @@ matched. The following context conditions are defined:
     confirming the match. If false is returned, no signature match is
     going to be triggered. The function has to be of type ``function
     cond(state: signature_state, data: string): bool``. Here,
-    ``content`` may contain the most recent content chunk available at
+    ``data`` may contain the most recent content chunk available at
     the time the signature was matched. If no such chunk is available,
-    ``content`` will be the empty string. ``signature_state`` is
+    ``data`` will be the empty string. See :bro:type:`signature_state`
-    defined as follows:
+    for its definition.
-
-    .. code:: bro
-
-        type signature_state: record {
-            id: string;          # ID of the signature
-            conn: connection;    # Current connection
-            is_orig: bool;       # True if current endpoint is originator
-            payload_size: count; # Payload size of the first packet
-            };
-
 
 ``payload-size <cmp> <integer>``
     Compares the integer to the size of the payload of a packet. For

doc/upgrade.rst

@@ -1,19 +1,80 @@
 
-=============================
+==========================================
-Upgrading From Bro 1.5 to 2.0
+Upgrading From the Previous Version of Bro
-=============================
+==========================================
 
 .. rst-class:: opening
 
-   This guide details differences between Bro versions 1.5 and 2.0
+   This guide details specific differences between Bro versions
    that may be important for users to know as they work on updating
    their Bro deployment/configuration to the later version.
 
 .. contents::
 
 
-Introduction
+Upgrading From Bro 2.0 to 2.1
-============
+=============================
+
+In Bro 2.1, IPv6 is enabled by default.  Therefore, when building Bro from
+source, the "--enable-brov6" configure option has been removed because it
+is no longer relevant.  
+
+Other configure changes include renaming the "--enable-perftools" option
+to "--enable-perftools-debug" to indicate that the option is only relevant
+for debugging the heap.  One other change involves what happens when
+tcmalloc (part of Google perftools) is found at configure time.  On Linux,
+it will automatically be linked with Bro, but on other platforms you
+need to use the "--enable-perftools" option to enable linking to tcmalloc.
+
+There are a couple of changes to the Bro scripting language to better
+support IPv6. First, IPv6 literals appearing in a Bro script must now be
+enclosed in square brackets (for example, ``[fe80::db15]``). For subnet
+literals, the slash "/" appears after the closing square bracket (for
+example, ``[fe80:1234::]/32``). Second, when an IP address variable or IP
+address literal is enclosed in pipes (for example, ``|[fe80::db15]|``) the
+result is now the size of the address in bits (32 for IPv4 and 128 for IPv6).
+
+In the Bro scripting language, "match" and "using" are no longer reserved
+keywords.
+
+Some built-in functions have been removed: "addr_to_count" (use
+"addr_to_counts" instead), "bro_has_ipv6" (this is no longer relevant
+because Bro now always supports IPv6), "active_connection" (use
+"connection_exists" instead), and "connection_record" (use "lookup_connection"
+instead).
+
+The "NFS3::mode2string" built-in function has been renamed to "file_mode".
+
+Some built-in functions have been changed:  "exit" (now takes the exit code
+as a parameter), "to_port" (now takes a string as parameter instead
+of a count and transport protocol, but "count_to_port" is still available),
+"connect" (now takes an additional string parameter specifying the zone of
+a non-global IPv6 address), and "listen" (now takes three additional
+parameters to enable listening on IPv6 addresses).
+
+Some Bro script variables have been renamed:  "LogAscii::header_prefix"
+has been renamed to "LogAscii::meta_prefix", "LogAscii::include_header"
+has been renamed to "LogAscii::include_meta".
+
+Some Bro script variables have been removed: "tunnel_port",
+"parse_udp_tunnels", "use_connection_compressor", "cc_handle_resets",
+"cc_handle_only_syns", and "cc_instantiate_on_data".
+
+A couple events have changed:  the "icmp_redirect" event now includes
+the target and destination addresses and any Neighbor Discovery options
+in the message, and the last parameter of the "dns_AAAA_reply" event has
+been removed because it was unused.
+
+The format of the ASCII log files has changed very slightly.  Two new lines
+are automatically added, one to record the time when the log was opened,
+and the other to record the time when the log was closed.
+
+In BroControl, the option (in broctl.cfg) "CFlowAddr" was renamed
+to "CFlowAddress".
+
+
+Upgrading From Bro 1.5 to 2.0
+=============================
 
 As the version number jump suggests, Bro 2.0 is a major upgrade and
 lots of things have changed. Most importantly, we have rewritten

pkg/make-mac-packages

@@ -3,7 +3,13 @@
 # This script creates binary packages for Mac OS X.
 # They can be found in ../build/ after running.
 
-./check-cmake || { exit 1; }
+cmake -P /dev/stdin << "EOF"
+if ( ${CMAKE_VERSION} VERSION_LESS 2.8.9 )
+    message(FATAL_ERROR "CMake >= 2.8.9 required to build package")
+endif ()
+EOF
+
+[ $? -ne 0 ] && exit 1;
 
 type sw_vers > /dev/null 2>&1 || {
     echo "Unable to get Mac OS X version" >&2;
@@ -34,26 +40,26 @@ prefix=/opt/bro
 cd ..
 
 # Minimum Bro
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --disable-broccoli --disable-broctl --pkg-name-prefix=Bro-minimal \
     --binary-package
 ( cd build && make package )
 
 # Full Bro package
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --pkg-name-prefix=Bro --binary-package
 ( cd build && make package )
 
 # Broccoli
 cd aux/broccoli
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..
 
 # Broctl
 cd aux/broctl
-CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
+CMAKE_PREFIX_PATH=/usr CMAKE_OSX_ARCHITECTURES=${arch} ./configure --prefix=${prefix} \
     --binary-package
 ( cd build && make package && mv *.dmg ../../../build/ )
 cd ../..

scripts/base/frameworks/cluster/main.bro

@@ -77,6 +77,9 @@ export {
 		node_type:    NodeType;
 		## The IP address of the cluster node.
 		ip:           addr;
+		## If the *ip* field is a non-global IPv6 address, this field
+		## can specify a particular :rfc:`4007` ``zone_id``.
+		zone_id:      string      &default="";
 		## The port to which the this local node can connect when
 		## establishing communication.
 		p:            port;

scripts/base/frameworks/cluster/nodes/manager.bro

@@ -13,8 +13,12 @@
 ## Turn off remote logging since this is the manager and should only log here.
 redef Log::enable_remote_logging = F;
 
+## Log rotation interval.
 redef Log::default_rotation_interval = 1 hrs;
 
+## Alarm summary mail interval.
+redef Log::default_mail_alarms_interval = 24 hrs;
+
 ## Use the cluster's archive logging script.
 redef Log::default_rotation_postprocessor_cmd = "archive-log";
 

scripts/base/frameworks/cluster/setup-connections.bro

@@ -19,23 +19,26 @@ event bro_init() &priority=9
 		# Connections from the control node for runtime control and update events.
 		# Every node in a cluster is eligible for control from this host.
 		if ( n$node_type == CONTROL )
-			Communication::nodes["control"] = [$host=n$ip, $connect=F,
+			Communication::nodes["control"] = [$host=n$ip, $zone_id=n$zone_id,
-			                                   $class="control", $events=control_events];
+			                                   $connect=F, $class="control",
+			                                   $events=control_events];
 		
 		if ( me$node_type == MANAGER )
 			{
 			if ( n$node_type == WORKER && n$manager == node )
 				Communication::nodes[i] =
-				    [$host=n$ip, $connect=F,
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=worker2manager_events, $request_logs=T];
 			
 			if ( n$node_type == PROXY && n$manager == node )
 				Communication::nodes[i] =
-				    [$host=n$ip, $connect=F,
+				    [$host=n$ip, $zone_id=n$zone_id, $connect=F,
 				     $class=i, $events=proxy2manager_events, $request_logs=T];
 				
 			if ( n$node_type == TIME_MACHINE && me?$time_machine && me$time_machine == i )
-				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, $p=nodes[i]$p,
+				Communication::nodes["time-machine"] = [$host=nodes[i]$ip,
+				                                        $zone_id=nodes[i]$zone_id,
+				                                        $p=nodes[i]$p,
 				                                        $connect=T, $retry=1min,
 				                                        $events=tm2manager_events];
 			}
@@ -44,7 +47,8 @@ event bro_init() &priority=9
 			{
 			if ( n$node_type == WORKER && n$proxy == node )
 				Communication::nodes[i] =
-					[$host=n$ip, $connect=F, $class=i, $sync=T, $auth=T, $events=worker2proxy_events];
+					[$host=n$ip, $zone_id=n$zone_id, $connect=F, $class=i,
+					 $sync=T, $auth=T, $events=worker2proxy_events];
 			
 			# accepts connections from the previous one. 
 			# (This is not ideal for setups with many proxies)
@@ -53,16 +57,18 @@ event bro_init() &priority=9
 				{
 				if ( n?$proxy )
 					Communication::nodes[i]
-					     = [$host=n$ip, $p=n$p,
+					     = [$host=n$ip, $zone_id=n$zone_id, $p=n$p,
 					        $connect=T, $auth=F, $sync=T, $retry=1mins];
 				else if ( me?$proxy && me$proxy == i )
 					Communication::nodes[me$proxy]
-					     = [$host=nodes[i]$ip, $connect=F, $auth=T, $sync=T];
+					     = [$host=nodes[i]$ip, $zone_id=nodes[i]$zone_id,
+					        $connect=F, $auth=T, $sync=T];
 				}
 			
 			# Finally the manager, to send it status updates.
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				                                   $zone_id=nodes[i]$zone_id, 
 				                                   $p=nodes[i]$p, 
 				                                   $connect=T, $retry=1mins, 
 				                                   $class=node,
@@ -72,6 +78,7 @@ event bro_init() &priority=9
 			{
 			if ( n$node_type == MANAGER && me$manager == i )
 				Communication::nodes["manager"] = [$host=nodes[i]$ip, 
+				                                   $zone_id=nodes[i]$zone_id,
 				                                   $p=nodes[i]$p,
 				                                   $connect=T, $retry=1mins, 
 				                                   $class=node, 
@@ -79,6 +86,7 @@ event bro_init() &priority=9
 			
 			if ( n$node_type == PROXY && me$proxy == i )
 				Communication::nodes["proxy"] = [$host=nodes[i]$ip, 
+				                                 $zone_id=nodes[i]$zone_id,
 				                                 $p=nodes[i]$p,
 				                                 $connect=T, $retry=1mins, 
 				                                 $sync=T, $class=node, 
@@ -87,6 +95,7 @@ event bro_init() &priority=9
 			if ( n$node_type == TIME_MACHINE && 
 			     me?$time_machine && me$time_machine == i )
 				Communication::nodes["time-machine"] = [$host=nodes[i]$ip, 
+				                                        $zone_id=nodes[i]$zone_id,
 				                                        $p=nodes[i]$p,
 				                                        $connect=T, 
 				                                        $retry=1min,

scripts/base/frameworks/communication/main.bro

@@ -2,6 +2,7 @@
 ##! and/or transfer events.
 
 @load base/frameworks/packet-filter
+@load base/utils/addrs
 
 module Communication;
 
@@ -10,7 +11,8 @@ export {
 	## The communication logging stream identifier.
 	redef enum Log::ID += { LOG };
 
-	## Which interface to listen on (0.0.0.0 for any interface).
+	## Which interface to listen on. The addresses ``0.0.0.0`` and ``[::]``
+	## are wildcards.
 	const listen_interface = 0.0.0.0 &redef;
 
 	## Which port to listen on.
@@ -19,6 +21,19 @@ export {
 	## This defines if a listening socket should use SSL.
 	const listen_ssl = F &redef;
 
+	## Defines if a listening socket can bind to IPv6 addresses.
+	const listen_ipv6 = F &redef;
+
+	## If :bro:id:`Communication::listen_interface` is a non-global
+	## IPv6 address and requires a specific :rfc:`4007` ``zone_id``,
+	## it can be specified here.
+	const listen_ipv6_zone_id = "" &redef;
+
+	## Defines the interval at which to retry binding to
+	## :bro:id:`Communication::listen_interface` on
+	## :bro:id:`Communication::listen_port` if it's already in use.
+	const listen_retry = 30 secs &redef;
+
 	## Default compression level.  Compression level is 0-9, with 0 = no
 	## compression.
 	global compression_level = 0 &redef;
@@ -27,7 +42,7 @@ export {
 	type Info: record {
 		## The network time at which a communication event occurred.
 		ts:                  time   &log;
-		## The peer name (if any) for which a communication event is concerned.
+		## The peer name (if any) with which a communication event is concerned.
 		peer:                string &log &optional;
 		## Where the communication event message originated from, that is,
 		## either from the scripting layer or inside the Bro process.
@@ -52,6 +67,10 @@ export {
 		## Remote address.
 		host: addr;
 
+		## If the *host* field is a non-global IPv6 address, this field
+		## can specify a particular :rfc:`4007` ``zone_id``.
+		zone_id: string &optional;
+
 		## Port of the remote Bro communication endpoint if we are initiating
 		## the connection based on the :bro:id:`connect` field.
 		p: port &optional;
@@ -160,7 +179,7 @@ event remote_log(level: count, src: count, msg: string)
 # This is a core generated event.
 event remote_log_peer(p: event_peer, level: count, src: count, msg: string)
 	{
-	local rmsg = fmt("[#%d/%s:%d] %s", p$id, p$host, p$p, msg);
+	local rmsg = fmt("[#%d/%s:%d] %s", p$id, addr_to_uri(p$host), p$p, msg);
 	do_script_log_common(level, src, rmsg);
 	}
 
@@ -178,7 +197,8 @@ function connect_peer(peer: string)
 		p = node$p;
 
 	local class = node?$class ? node$class : "";
-	local id = connect(node$host, p, class, node$retry, node$ssl);
+	local zone_id = node?$zone_id ? node$zone_id : "";
+	local id = connect(node$host, zone_id, p, class, node$retry, node$ssl);
 
 	if ( id == PEER_ID_NONE )
 		Log::write(Communication::LOG, [$ts = network_time(),

scripts/base/frameworks/control/main.bro

@@ -11,6 +11,10 @@ export {
 	## The port of the host that will be controlled.
 	const host_port = 0/tcp &redef;
 
+	## If :bro:id:`Control::host` is a non-global IPv6 address and
+	## requires a specific :rfc:`4007` ``zone_id``, it can be set here.
+	const zone_id = "" &redef;
+
 	## The command that is being done.  It's typically set on the
 	## command line.
 	const cmd = "" &redef;

scripts/base/frameworks/dpd/dpd.sig

@@ -149,3 +149,64 @@ signature dpd_ssl_client {
   payload /^(\x16\x03[\x00\x01\x02]..\x01...\x03[\x00\x01\x02]|...?\x01[\x00\x01\x02][\x02\x03]).*/
   tcp-state originator
 }
+
+signature dpd_ayiya {
+  ip-proto = udp
+  payload /^..\x11\x29/
+  enable "ayiya"
+}
+
+signature dpd_teredo {
+  ip-proto = udp
+  payload /^(\x00\x00)|(\x00\x01)|([\x60-\x6f])/
+  enable "teredo"
+}
+
+signature dpd_socks4_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state originator
+}
+
+signature dpd_socks4_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state responder
+	enable "socks"
+}
+
+signature dpd_socks4_reverse_client {
+	ip-proto == tcp
+	# '32' is a rather arbitrary max length for the user name.
+	payload /^\x04[\x01\x02].{0,32}\x00/
+	tcp-state responder
+}
+
+signature dpd_socks4_reverse_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks4_reverse_client
+	payload /^\x00[\x5a\x5b\x5c\x5d]/
+	tcp-state originator
+	enable "socks"
+}
+
+signature dpd_socks5_client {
+	ip-proto == tcp
+	# Watch for a few authentication methods to reduce false positives.
+	payload /^\x05.[\x00\x01\x02]/
+	tcp-state originator
+}
+
+signature dpd_socks5_server {
+	ip-proto == tcp
+	requires-reverse-signature dpd_socks5_client
+	# Watch for a single authentication method to be chosen by the server or
+	# the server to indicate the no authentication is required.
+	payload /^\x05(\x00|\x01[\x00\x01\x02])/
+	tcp-state responder
+	enable "socks"
+}
+
+

scripts/base/frameworks/dpd/main.bro

@@ -3,8 +3,7 @@
 
 module DPD;
 
-## Add the DPD signatures to the signature framework.
+@load-sigs ./dpd.sig
-redef signature_files += "base/frameworks/dpd/dpd.sig";
 
 export {
 	## Add the DPD logging stream identifier.
@@ -105,5 +104,8 @@ event protocol_violation(c: connection, atype: count, aid: count,
 				reason: string) &priority=-5
 	{
 	if ( c?$dpd )
+		{
 		Log::write(DPD::LOG, c$dpd);
+		delete c$dpd;
+		}
 	}

scripts/base/frameworks/input/__load__.bro

@@ -0,0 +1,5 @@
+@load ./main
+@load ./readers/ascii
+@load ./readers/raw
+@load ./readers/benchmark
+

scripts/base/frameworks/input/main.bro

@@ -0,0 +1,158 @@
+##! The input framework provides a way to read previously stored data either
+##! as an event stream or into a bro table.
+
+module Input;
+
+export {
+
+	## The default input reader used. Defaults to `READER_ASCII`.
+	const default_reader = READER_ASCII &redef;
+
+	## The default reader mode used. Defaults to `MANUAL`.
+	const default_mode = MANUAL &redef;
+
+	## Flag that controls if the input framework accepts records
+	## that contain types that are not supported (at the moment
+	## file and function). If true, the input framework will
+	## warn in these cases, but continue. If false, it will
+	## abort. Defaults to false (abort)
+	const accept_unsupported_types = F &redef;
+
+	## TableFilter description type used for the `table` method.
+	type TableDescription: record {
+		## Common definitions for tables and events
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this stream
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a stream at a later time
+		name: string;
+
+		# Special definitions for tables
+
+		## Table which will receive the data read by the input framework
+		destination: any;
+
+		## Record that defines the values used as the index of the table
+		idx: any;
+
+		## Record that defines the values used as the elements of the table
+		## If val is undefined, destination has to be a set.
+		val: any &optional;
+
+		## Defines if the value of the table is a record (default), or a single  value. Val
+		## can only contain one element when this is set to false.
+		want_record: bool &default=T;
+
+		## The event that is raised each time a value is added to, changed in or removed
+		## from the table. The event will receive an Input::Event enum as the first
+		## argument, the idx record as the second argument and the value (record) as the
+		## third argument.
+		ev: any &optional; # event containing idx, val as values.
+
+		## Predicate function that can decide if an insertion, update or removal should
+		## really be executed. Parameters are the same as for the event. If true is
+		## returned, the update is performed. If false is returned, it is skipped.
+		pred: function(typ: Input::Event, left: any, right: any): bool &optional;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+                config: table[string] of string &default=table();
+	};
+
+	## EventFilter description type used for the `event` method.
+	type EventDescription: record {
+		## Common definitions for tables and events
+
+		## String that allows the reader to find the source.
+		## For `READER_ASCII`, this is the filename.
+		source: string;
+
+		## Reader to use for this steam
+		reader: Reader &default=default_reader;
+
+		## Read mode to use for this stream
+		mode: Mode &default=default_mode;
+
+		## Descriptive name. Used to remove a stream at a later time
+		name: string;
+
+		# Special definitions for events
+
+		## Record describing the fields to be retrieved from the source input.
+		fields: any;
+
+		## If want_record if false, the event receives each value in fields as a separate argument.
+		## If it is set to true (default), the event receives all fields in a single record value.
+		want_record: bool &default=T;
+
+		## The event that is raised each time a new line is received from the reader.
+		## The event will receive an Input::Event enum as the first element, and the fields as the following arguments.
+		ev: any;
+
+		## A key/value table that will be passed on the reader.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
+	};
+
+	## Create a new table input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_table: function(description: Input::TableDescription) : bool;
+
+	## Create a new event input from a given source. Returns true on success.
+	##
+	## description: `TableDescription` record describing the source.
+	global add_event: function(description: Input::EventDescription) : bool;
+
+	## Remove a input stream. Returns true on success and false if the named stream was
+	## not found. 
+	##
+	## id: string value identifying the stream to be removed
+	global remove: function(id: string) : bool;
+
+	## Forces the current input to be checked for changes.
+	## Returns true on success and false if the named stream was not found
+	##
+	## id: string value identifying the stream
+	global force_update: function(id: string) : bool;
+
+	## Event that is called, when the end of a data source has been reached, including
+	## after an update.
+	global end_of_data: event(name: string, source:string);
+}
+
+@load base/input.bif
+
+
+module Input;
+
+function add_table(description: Input::TableDescription) : bool
+	{
+	return __create_table_stream(description);
+	}
+
+function add_event(description: Input::EventDescription) : bool
+	{
+	return __create_event_stream(description);
+	}
+
+function remove(id: string) : bool
+	{
+	return __remove_stream(id);
+	}
+
+function force_update(id: string) : bool
+	{
+	return __force_update(id);
+	}
+

scripts/base/frameworks/input/readers/ascii.bro

@@ -0,0 +1,21 @@
+##! Interface for the ascii input reader.
+##!
+##! The defaults are set to match Bro's ASCII output.
+
+module InputAscii;
+
+export {
+	## Separator between fields.
+	## Please note that the separator has to be exactly one character long
+	const separator = "\t" &redef;
+
+	## Separator between set elements.
+	## Please note that the separator has to be exactly one character long
+	const set_separator = "," &redef;
+
+	## String to use for empty fields.
+	const empty_field = "(empty)" &redef;
+
+	## String to use for an unset &optional field.
+	const unset_field = "-" &redef;
+}

scripts/base/frameworks/input/readers/benchmark.bro

@@ -0,0 +1,23 @@
+##! Interface for the ascii input reader.
+
+module InputBenchmark;
+
+export {
+	## multiplication factor for each second
+	const factor = 1.0 &redef;
+
+	## spread factor between lines
+	const spread = 0 &redef;
+
+	## spreading where usleep = 1000000 / autospread * num_lines
+	const autospread = 0.0 &redef;
+
+	## addition factor for each heartbeat
+	const addfactor = 0 &redef;
+
+	## stop spreading at x lines per heartbeat
+	const stopspreadat = 0 &redef;
+
+	## 1 -> enable timed spreading
+	const timedspread = 0.0 &redef;
+}

scripts/base/frameworks/input/readers/raw.bro

@@ -0,0 +1,9 @@
+##! Interface for the raw input reader.
+
+module InputRaw;
+
+export {
+	## Separator between input records.
+	## Please note that the separator has to be exactly one character long
+	const record_separator = "\n" &redef;
+}

scripts/base/frameworks/intel/__load__.bro

@@ -1 +1,11 @@
 @load ./main
+
+# The cluster framework must be loaded first.
+@load base/frameworks/cluster
+
+@if ( Cluster::is_enabled() )
+@load ./cluster
+@endif
+
+# This needs cluster support to only read on the manager.
+@load ./input

scripts/base/frameworks/intel/cluster.bro

@@ -0,0 +1,61 @@
+##! Cluster transparency support for the intelligence framework.  This is mostly oriented
+##! toward distributing intelligence information across clusters.
+
+@load base/frameworks/cluster
+@load ./input
+
+module Intel;
+
+redef record Item += {
+	## This field is used internally for cluster transparency to avoid 
+	## re-dispatching intelligence items over and over from workers.
+	first_dispatch: bool &default=T;
+};
+
+# If this process is not a manager process, we don't want the full metadata
+@if ( Cluster::local_node_type() != Cluster::MANAGER )
+redef have_full_data = F;
+@endif
+
+global cluster_new_item: event(item: Item);
+
+# Primary intelligence distribution comes from manager.
+redef Cluster::manager2worker_events += /^Intel::(cluster_new_item)$/;
+# If a worker finds intelligence and adds it, it should share it back to the manager.
+redef Cluster::worker2manager_events += /^Intel::(cluster_new_item|match_no_items)$/;
+
+@if ( Cluster::local_node_type() == Cluster::MANAGER )
+event Intel::match_no_items(s: Seen) &priority=5
+	{
+	event Intel::match(s, Intel::get_items(s));
+	}
+
+event remote_connection_handshake_done(p: event_peer)
+	{
+	# When a worker connects, send it the complete minimal data store.
+	# It will be kept up to date after this by the cluster_new_item event.
+	if ( Cluster::nodes[p$descr]$node_type == Cluster::WORKER )
+		{
+		send_id(p, "Intel::min_data_store");
+		}
+	}
+@endif
+
+event Intel::cluster_new_item(item: Intel::Item) &priority=5
+	{
+	# Ignore locally generated events to avoid event storms.
+	if ( is_remote_event() )
+		Intel::insert(item);
+	}
+
+event Intel::new_item(item: Intel::Item) &priority=5
+	{
+	# The cluster manager always rebroadcasts intelligence.
+	# Workers redistribute it if it was locally generated.
+	if ( Cluster::local_node_type() == Cluster::MANAGER ||
+	     item$first_dispatch )
+		{
+		item$first_dispatch=F;
+		event Intel::cluster_new_item(item);
+		}
+	}

scripts/base/frameworks/intel/input.bro

@@ -0,0 +1,33 @@
+@load ./main
+
+module Intel;
+
+export {
+	## Intelligence files that will be read off disk.  The files are
+	## reread everytime they are updated so updates much be atomic with
+	## "mv" instead of writing the file in place.
+	const read_files: set[string] = {} &redef;
+}
+
+event Intel::read_entry(desc: Input::EventDescription, tpe: Input::Event, item: Intel::Item)
+	{
+	Intel::insert(item);
+	}
+
+event bro_init() &priority=5
+	{
+	if ( ! Cluster::is_enabled() ||
+	     Cluster::local_node_type() == Cluster::MANAGER )
+		{
+		for ( a_file in read_files )
+			{
+			Input::add_event([$source=a_file,
+			                  $reader=Input::READER_ASCII,
+			                  $mode=Input::REREAD,
+			                  $name=cat("intel-", a_file),
+			                  $fields=Intel::Item,
+			                  $ev=Intel::read_entry]);
+			}
+		}
+	}
+

scripts/base/frameworks/intel/main.bro

@@ -1,323 +1,345 @@
 ##! The intelligence framework provides a way to store and query IP addresses,
-##! strings (with a subtype), and numeric (with a subtype) data.  Metadata 
+##! and strings (with a str_type).  Metadata can
-##! also be associated with the intelligence like tags which are arbitrary
+##! also be associated with the intelligence like for making more informed
-##! strings, time values, and longer descriptive strings.
+##! decisions about matching and handling of intelligence.
-
-# Example string subtypes:
-#   url
-#   email
-#   domain
-#   software
-#   user_name
-#   file_name
-#   file_md5
-#   x509_md5
-
-# Example tags: 
-#   infrastructure
-#   malicious
-#   sensitive
-#   canary
-#   friend
 
 @load base/frameworks/notice
 
 module Intel;
 
 export {
-	## The intel logging stream identifier.
 	redef enum Log::ID += { LOG };
 	
-	redef enum Notice::Type += {
+	## String data needs to be further categoried since it could represent
-		## This notice should be used in all detector scripts to indicate 
+	## and number of types of data.
-		## an intelligence based detection.
+	type StrType: enum {
-		Detection,
+		## A complete URL without the prefix "http://".
+		URL,
+		## User-Agent string, typically HTTP or mail message body.
+		USER_AGENT,
+		## Email address.
+		EMAIL,
+		## DNS domain name.
+		DOMAIN,
+		## A user name.
+		USER_NAME,
+		## File hash which is non-hash type specific.  It's up to the user to query
+		## for any relevant hash types.
+		FILE_HASH,
+		## Certificate SHA-1 hash.
+		CERT_HASH,
 	};
 	
-	## Record type used for logging information from the intelligence framework.
+	## Data about an :bro:type:`Intel::Item`
-	## Primarily for problems or oddities with inserting and querying data.  
-	## This is important since the content of the intelligence framework can 
-	## change quite dramatically during runtime and problems may be introduced 
-	## into the data.
-	type Info: record {
-		## The current network time.
-		ts:      time   &log;
-		## Represents the severity of the message. 
-		## This value should be one of: "info", "warn", "error"
-		level:   string &log;
-		## The message.
-		message: string &log;
-	};
-	
-	## Record to represent metadata associated with a single piece of
-	## intelligence.
 	type MetaData: record {
-		## A description for the data.
+		## An arbitrary string value representing the data source.  Typically,
+		## the convention for this field will be the source name and feed name
+		## separated by a hyphen.  For example: "source1-c&c".
+		source:      string;
+		## A freeform description for the data.
 		desc:        string      &optional;
-		## A URL where more information may be found about the intelligence.
+		## A URL for more information about the data.
 		url:         string      &optional;
-		## The time at which the data was first declared to be intelligence.
-		first_seen:  time        &optional;
-		## When this data was most recent inserted into the framework.
-		latest_seen: time        &optional;
-		## Arbitrary text tags for the data.
-		tags:        set[string];
 	};
 	
-	## Record to represent a singular piece of intelligence.
+	## Represents a piece of intelligence.
 	type Item: record {
-		## If the data is an IP address, this hold the address.
+		## The IP address if the intelligence is about an IP address.
-		ip:          addr        &optional;
+		host:        addr           &optional;
-		## If the data is textual, this holds the text.
+		## The network if the intelligence is about a CIDR block.
+		net:         subnet         &optional;
+		## The string if the intelligence is about a string.
 		str:         string         &optional;
-		## If the data is numeric, this holds the number.
+		## The type of data that is in the string if the $str field is set.
-		num:         int         &optional;
+		str_type:    StrType        &optional;
-		## The subtype of the data for when either the $str or $num fields are
-		## given.  If one of those fields are given, this field must be present.
-		subtype:     string      &optional;
 		
-		## The next five fields are temporary until a better model for 
+		## Metadata for the item.  Typically represents more deeply \
-		## attaching metadata to an intelligence item is created.
+		## descriptive data for a piece of intelligence.
-		desc:        string      &optional;
+		meta:        MetaData;
-		url:         string      &optional;
-		first_seen:  time        &optional;
-		latest_seen: time        &optional;
-		tags:        set[string];
-		
-		## These single string tags are throw away until pybroccoli supports sets.
-		tag1: string &optional;
-		tag2: string &optional;
-		tag3: string &optional;
 	};
 	
-	## Record model used for constructing queries against the intelligence 
+	## Enum to represent where data came from when it was discovered.
-	## framework.
+	## The convenction is to prefix the name with "IN_".
-	type QueryItem: record {
+	type Where: enum {
-		## If an IP address is being queried for, this field should be given.
+		## A catchall value to represent data of unknown provenance.
-		ip:        addr        &optional;
+		IN_ANYWHERE,
-		## If a string is being queried for, this field should be given.
-		str:       string      &optional;
-		## If numeric data is being queried for, this field should be given.
-		num:       int         &optional;
-		## If either a string or number is being queried for, this field should
-		## indicate the subtype of the data.
-		subtype:   string      &optional;
-		
-		## A set of tags where if a single metadata record attached to an item
-		## has any one of the tags defined in this field, it will match.
-		or_tags:   set[string] &optional;
-		## A set of tags where a single metadata record attached to an item 
-		## must have all of the tags defined in this field.
-		and_tags:  set[string] &optional; 
-		
-		## The predicate can be given when searching for a match.  It will
-		## be tested against every :bro:type:`Intel::MetaData` item associated
-		## with the data being matched on.  If it returns T a single time, the 
-		## matcher will consider that the item has matched.  This field can
-		## be used for constructing arbitrarily complex queries that may not
-		## be possible with the $or_tags or $and_tags fields.
-		pred:      function(meta: Intel::MetaData): bool &optional;
 	};
 
-	## Function to insert data into the intelligence framework.
+	## The $host field and combination of $str and $str_type fields are mutually 
-	## 
+	## exclusive.  These records *must* represent either an IP address being
-	## item: The data item.
+	## seen or a string being seen.
-	##
+	type Seen: record {
-	## Returns: T if the data was successfully inserted into the framework,
+		## The IP address if the data seen is an IP address.
-	##          otherwise it returns F.
+		host:      addr          &log &optional;
-	global insert: function(item: Item): bool;
+		## The string if the data is about a string.
+		str:       string        &log &optional;
+		## The type of data that is in the string if the $str field is set.
+		str_type:  StrType       &log &optional;
 
-	## A wrapper for the :bro:id:`Intel::insert` function.  This is primarily
+		## Where the data was discovered.
-	## used as the external API for inserting data into the intelligence 
+		where:     Where         &log;
-	## using Broccoli.
-	global insert_event: event(item: Item);
 		
-	## Function for matching data within the intelligence framework.
+		## If the data was discovered within a connection, the 
-	global matcher: function(item: QueryItem): bool;
+		## connection record should go into get to give context to the data.
+		conn:      connection    &optional;
+	};
+
+	## Record used for the logging framework representing a positive
+	## hit within the intelligence framework.
+	type Info: record {
+		## Timestamp when the data was discovered.
+		ts:       time           &log;
+
+		## If a connection was associated with this intelligence hit,
+		## this is the uid for the connection
+		uid:      string         &log &optional;
+		## If a connection was associated with this intelligence hit,
+		## this is the conn_id for the connection.
+		id:       conn_id        &log &optional;
+
+		## Where the data was seen.
+		seen:     Seen           &log;
+		## Sources which supplied data that resulted in this match.
+		sources:  set[string]    &log;
+	};
+
+	## Intelligence data manipulation functions.
+	global insert: function(item: Item);
+
+	## Function to declare discovery of a piece of data in order to check
+	## it against known intelligence for matches.
+	global seen: function(s: Seen);
+
+	## Event to represent a match in the intelligence data from data that was seen.  
+	## On clusters there is no assurance as to where this event will be generated 
+	## so do not assume that arbitrary global state beyond the given data
+	## will be available.
+	##
+	## This is the primary mechanism where a user will take actions based on data
+	## within the intelligence framework.
+	global match: event(s: Seen, items: set[Item]);
+
+	global log_intel: event(rec: Info);
 }
 
-type MetaDataStore: table[count] of MetaData;
+# Internal handler for matches with no metadata available.
+global match_no_items: event(s: Seen);
+
+# Internal events for cluster data distribution
+global new_item: event(item: Item);
+global updated_item: event(item: Item);
+
+# Optionally store metadata.  This is used internally depending on
+# if this is a cluster deployment or not.
+const have_full_data = T &redef;
+
+# The in memory data structure for holding intelligence.
 type DataStore: record {
-	ip_data:     table[addr] of MetaDataStore;
+	net_data:    table[subnet] of set[MetaData];
-	# The first string is the actual value and the second string is the subtype.
+	string_data: table[string, StrType] of set[MetaData];
-	string_data: table[string, string] of MetaDataStore;
-	int_data:    table[int, string] of MetaDataStore;
 };
-global data_store: DataStore;
+global data_store: DataStore &redef;
 
-event bro_init()
+# The in memory data structure for holding the barest matchable intelligence.
+# This is primarily for workers to do the initial quick matches and store
+# a minimal amount of data for the full match to happen on the manager.
+type MinDataStore: record {
+	net_data:    set[subnet];
+	string_data: set[string, StrType];
+};
+global min_data_store: MinDataStore &redef;
+
+
+event bro_init() &priority=5
 	{
-	Log::create_stream(Intel::LOG, [$columns=Info]);
+	Log::create_stream(LOG, [$columns=Info, $ev=log_intel]);
+	}
+
+function find(s: Seen): bool
+	{
+	if ( s?$host && 
+	     ((have_full_data && s$host in data_store$net_data) || 
+	      (s$host in min_data_store$net_data)))
+		{
+		return T;
+		}
+	else if ( s?$str && s?$str_type &&
+	          ((have_full_data && [s$str, s$str_type] in data_store$string_data) ||
+	           ([s$str, s$str_type] in min_data_store$string_data)))
+		{
+		return T;
+		}
+	else
+		{
+		return F;
+		}
+	}
+
+function get_items(s: Seen): set[Item]
+	{
+	local item: Item;
+	local return_data: set[Item] = set();
+
+	if ( ! have_full_data )
+		{
+		# A reporter warning should be generated here because this function
+		# should never be called from a host that doesn't have the full data.
+		# TODO: do a reporter warning.
+		return return_data;
+		}
+
+	if ( s?$host )
+		{
+		# See if the host is known about and it has meta values
+		if ( s$host in data_store$net_data )
+			{
+			for ( m in data_store$net_data[s$host] )
+				{
+				# TODO: the lookup should be finding all and not just most specific
+				#       and $host/$net should have the correct value.
+				item = [$host=s$host, $meta=m];
+				add return_data[item];
+				}
+			}
+		}
+	else if ( s?$str && s?$str_type )
+		{
+		# See if the string is known about and it has meta values
+		if ( [s$str, s$str_type] in data_store$string_data )
+			{
+			for ( m in data_store$string_data[s$str, s$str_type] )
+				{
+				item = [$str=s$str, $str_type=s$str_type, $meta=m];
+				add return_data[item];
+				}
+			}
+		}
+
+	return return_data;
+	}
+
+function Intel::seen(s: Seen)
+	{
+	if ( find(s) )
+		{
+		if ( have_full_data )
+			{
+			local items = get_items(s);
+			event Intel::match(s, items);
+			}
+		else
+			{
+			event Intel::match_no_items(s);
+			}
+		}
 	}
 
 
-function insert(item: Item): bool
+function has_meta(check: MetaData, metas: set[MetaData]): bool
 	{
-	local err_msg = "";
+	local check_hash = md5_hash(check);
-	if ( (item?$str || item?$num) && ! item?$subtype )
+	for ( m in metas )
-		err_msg = "You must provide a subtype to insert_sync or this item doesn't make sense.";
+		{
+		if ( check_hash == md5_hash(m) )
+			return T;
+		}
 
-	if ( err_msg == "" )
+	# The records must not be equivalent if we made it this far.
+	return F;
+	}
+
+event Intel::match(s: Seen, items: set[Item]) &priority=5
 	{
+	local empty_set: set[string] = set();
+	local info: Info = [$ts=network_time(), $seen=s, $sources=empty_set];
+
+	if ( s?$conn )
+		{
+		info$uid = s$conn$uid;
+		info$id  = s$conn$id;
+		}
+
+	for ( item in items )
+		add info$sources[item$meta$source];
+
+	Log::write(Intel::LOG, info);
+	}
+
+function insert(item: Item)
+	{
+	if ( item?$str && !item?$str_type )
+		{
+		event reporter_warning(network_time(), fmt("You must provide a str_type for strings or this item doesn't make sense.  Item: %s", item), "");
+		return;
+		}
+
 	# Create and fill out the meta data item.
-		local meta: MetaData;
+	local meta = item$meta;
-		if ( item?$first_seen )
+	local metas: set[MetaData];
-			meta$first_seen = item$first_seen;
-		if ( item?$latest_seen )
-			meta$latest_seen = item$latest_seen;
-		if ( item?$tags )
-			meta$tags = item$tags;
-		if ( item?$desc )
-			meta$desc = item$desc;
-		if ( item?$url )
-			meta$url = item$url;
 
-		
+	if ( item?$host )
-		# This is hopefully only temporary until pybroccoli supports sets.
-		if ( item?$tag1 )
-			add item$tags[item$tag1];
-		if ( item?$tag2 )
-			add item$tags[item$tag2];
-		if ( item?$tag3 )
-			add item$tags[item$tag3];
-		
-		if ( item?$ip )
 		{
-			if ( item$ip !in data_store$ip_data )
+		local host = mask_addr(item$host, is_v4_addr(item$host) ? 32 : 128);
-				data_store$ip_data[item$ip] = table();
+		if ( have_full_data )
-			data_store$ip_data[item$ip][|data_store$ip_data[item$ip]|] = meta;
+			{
-			return T;
+			if ( host !in data_store$net_data )
+				data_store$net_data[host] = set();
+
+			metas = data_store$net_data[host];
+			}
+
+		add min_data_store$net_data[host];
+		}
+	else if ( item?$net )
+		{
+		if ( have_full_data )
+			{
+			if ( item$net !in data_store$net_data )
+				data_store$net_data[item$net] = set();
+
+			metas = data_store$net_data[item$net];
+			}
+
+		add min_data_store$net_data[item$net];
 		}
 	else if ( item?$str )
 		{
-			if ( [item$str, item$subtype] !in data_store$string_data )
+		if ( have_full_data )
-				data_store$string_data[item$str, item$subtype] = table();
-			
-			data_store$string_data[item$str, item$subtype][|data_store$string_data[item$str, item$subtype]|] = meta;
-			return T;
-			}
-		else if ( item?$num )
 			{
-			if ( [item$num, item$subtype] !in data_store$int_data )
+			if ( [item$str, item$str_type] !in data_store$string_data )
-				data_store$int_data[item$num, item$subtype] = table();
+				data_store$string_data[item$str, item$str_type] = set();
 
-			data_store$int_data[item$num, item$subtype][|data_store$int_data[item$num, item$subtype]|] = meta;
+			metas = data_store$string_data[item$str, item$str_type];
-			return T;
+			}
+
+		add min_data_store$string_data[item$str, item$str_type];
+		}
+
+	local updated = F;
+	if ( have_full_data )
+		{
+		for ( m in metas )
+			{
+			if ( meta$source == m$source )
+				{
+				if ( has_meta(meta, metas) )
+					{
+					# It's the same item being inserted again.
+					return;
 					}
 				else
-			err_msg = "Failed to insert intelligence item for some unknown reason.";
+					{
+					# Same source, different metadata means updated item.
+					updated = T;
+					}
+				}
+			}
+		add metas[item$meta];
 		}
 	
-	if ( err_msg != "" )
+	if ( updated )
-		Log::write(Intel::LOG, [$ts=network_time(), $level="warn", $message=fmt(err_msg)]);
+		event Intel::updated_item(item);
-	return F;
-	}
-	
-event insert_event(item: Item)
-	{
-	insert(item);
-	}
-	
-function match_item_with_metadata(item: QueryItem, meta: MetaData): bool
-	{
-	if ( item?$and_tags )
-		{
-		local matched = T;
-		# Every tag given has to match in a single MetaData entry.
-		for ( tag in item$and_tags )
-			{
-			if ( tag !in meta$tags )
-				matched = F;
-			}
-		if ( matched )
-			return T;
-		}
-	else if ( item?$or_tags )
-		{
-		# For OR tags, only a single tag has to match.
-		for ( tag in item$or_tags )
-			{
-			if ( tag in meta$tags )
-				return T;
-			}
-		}
-	else if ( item?$pred )
-		return item$pred(meta);
-
-	# This indicates some sort of failure in the query
-	return F;
-	}
-	
-function matcher(item: QueryItem): bool
-	{
-	local err_msg = "";
-	if ( ! (item?$ip || item?$str || item?$num) )
-		err_msg = "You must supply one of the $ip, $str, or $num fields to search on";
-	else if ( (item?$or_tags || item?$and_tags) && item?$pred )
-		err_msg = "You can't match with both tags and a predicate.";
-	else if ( item?$or_tags && item?$and_tags )
-		err_msg = "You can't match with both OR'd together tags and AND'd together tags";
-	else if ( (item?$str || item?$num) && ! item?$subtype )
-		err_msg = "You must provide a subtype to matcher or this item doesn't make sense.";
-	else if ( item?$str && item?$num )
-		err_msg = "You must only provide $str or $num, not both.";
-		
-	local meta: MetaData;
-
-	if ( err_msg == "" )
-		{
-		if ( item?$ip )
-			{
-			if ( item$ip in data_store$ip_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-			
-				for ( i in data_store$ip_data[item$ip] )
-					{
-					meta = data_store$ip_data[item$ip][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		
-		else if ( item?$str )
-			{
-			if ( [item$str, item$subtype] in data_store$string_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$string_data[item$str, item$subtype] )
-					{
-					meta = data_store$string_data[item$str, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
-		
-		else if ( item?$num )
-			{
-			if ( [item$num, item$subtype] in data_store$int_data )
-				{
-				if ( ! item?$and_tags && ! item?$or_tags && ! item?$pred )
-					return T;
-
-				for ( i in data_store$int_data[item$num, item$subtype] )
-					{
-					meta = data_store$int_data[item$num, item$subtype][i];
-					if ( match_item_with_metadata(item, meta) )
-						return T;
-					}
-				}
-			}
 	else
-			err_msg = "Failed to query intelligence data for some unknown reason.";
+		event Intel::new_item(item);
 	}
 	
-	if ( err_msg != "" )
-		Log::write(Intel::LOG, [$ts=network_time(), $level="error", $message=fmt(err_msg)]);
-	return F;
-	}

scripts/base/frameworks/logging/__load__.bro

@@ -1,3 +1,6 @@
 @load ./main
 @load ./postprocessors
 @load ./writers/ascii
+@load ./writers/dataseries
+@load ./writers/elasticsearch
+@load ./writers/none

scripts/base/frameworks/logging/main.bro

@@ -60,6 +60,9 @@ export {
 	## Default rotation interval. Zero disables rotation.
 	const default_rotation_interval = 0secs &redef;
 
+	## Default alarm summary mail interval. Zero disables alarm summary mails.
+	const default_mail_alarms_interval = 0secs &redef;
+
 	## Default naming format for timestamps embedded into filenames.
 	## Uses a ``strftime()`` style.
 	const default_rotation_date_format = "%Y-%m-%d-%H-%M-%S" &redef;
@@ -96,6 +99,12 @@ export {
 		## file name. Generally, filenames are expected to given
 		## without any extensions; writers will add appropiate
 		## extensions automatically.
+		##
+		## If this path is found to conflict with another filter's
+		## for the same writer type, it is automatically corrected
+		## by appending "-N", where N is the smallest integer greater
+		## or equal to 2 that allows the corrected path name to not
+		## conflict with another filter's.
 		path: string &optional;
 
 		## A function returning the output path for recording entries
@@ -115,7 +124,10 @@ export {
 		## rec: An instance of the streams's ``columns`` type with its
 		##      fields set to the values to be logged.
 		##
-		## Returns: The path to be used for the filter.
+		## Returns: The path to be used for the filter, which will be subject
+		##          to the same automatic correction rules as the *path*
+		##          field of :bro:type:`Log::Filter` in the case of conflicts
+		##          with other filters trying to use the same writer/path pair.
 		path_func: function(id: ID, path: string, rec: any): string &optional;
 
 		## Subset of column names to record. If not given, all
@@ -138,6 +150,11 @@ export {
 		## Callback function to trigger for rotated files. If not set, the
 		## default comes out of :bro:id:`Log::default_rotation_postprocessors`.
 		postprocessor: function(info: RotationInfo) : bool &optional;
+
+		## A key/value table that will be passed on to the writer.
+		## Interpretation of the values is left to the writer, but
+		## usually they will be used for configuration purposes.
+		config: table[string] of string &default=table();
 	};
 
 	## Sentinel value for indicating that a filter was not found when looked up.
@@ -313,6 +330,11 @@ export {
 	##    Log::default_rotation_postprocessor_cmd
 	##    Log::default_rotation_postprocessors
 	global run_rotation_postprocessor_cmd: function(info: RotationInfo, npath: string) : bool;
+
+	## The streams which are currently active and not disabled.
+	## This table is not meant to be modified by users!  Only use it for
+	## examining which streams are active.
+	global active_streams: table[ID] of Stream = table();
 }
 
 # We keep a script-level copy of all filters so that we can manipulate them.
@@ -327,20 +349,23 @@ function __default_rotation_postprocessor(info: RotationInfo) : bool
 	{
 	if ( info$writer in default_rotation_postprocessors )
 		return default_rotation_postprocessors[info$writer](info);
+	else
+		# Return T by default so that postprocessor-less writers don't shutdown.
+		return T;
 	}
 
 function default_path_func(id: ID, path: string, rec: any) : string
-	{
-	local id_str = fmt("%s", id);
-	
-	local parts = split1(id_str, /::/);
-	if ( |parts| == 2 )
 	{
 	# The suggested path value is a previous result of this function
 	# or a filter path explicitly set by the user, so continue using it.
 	if ( path != "" )
 		return path;
 
+	local id_str = fmt("%s", id);
+
+	local parts = split1(id_str, /::/);
+	if ( |parts| == 2 )
+		{
 		# Example: Notice::LOG -> "notice"
 		if ( parts[2] == "LOG" )
 			{
@@ -376,13 +401,16 @@ function run_rotation_postprocessor_cmd(info: RotationInfo, npath: string) : boo
 	if ( pp_cmd == "" )
 		return T;
 
+	# Turn, e.g., Log::WRITER_ASCII into "ascii".
+	local writer = subst_string(to_lower(fmt("%s", info$writer)), "log::writer_", "");
+
 	# The date format is hard-coded here to provide a standardized
 	# script interface.
-	system(fmt("%s %s %s %s %s %d",
+	system(fmt("%s %s %s %s %s %d %s",
                pp_cmd, npath, info$path,
                strftime("%y-%m-%d_%H.%M.%S", info$open),
                strftime("%y-%m-%d_%H.%M.%S", info$close),
-               info$terminating));
+               info$terminating, writer));
 
 	return T;
 	}
@@ -392,11 +420,15 @@ function create_stream(id: ID, stream: Stream) : bool
 	if ( ! __create_stream(id, stream) )
 		return F;
 
+	active_streams[id] = stream;
+
 	return add_default_filter(id);
 	}
 
 function disable_stream(id: ID) : bool
 	{
+	delete active_streams[id];
+
 	return __disable_stream(id);
 	}
 

scripts/base/frameworks/logging/writers/ascii.bro

@@ -8,12 +8,13 @@ export {
 	## into files. This is primarily for debugging purposes.
 	const output_to_stdout = F &redef;
 
-	## If true, include a header line with column names and description
+	## If true, include lines with log meta information such as column names with
-	## of the other ASCII logging options that were used.
+	## types, the values of ASCII logging options that in use, and the time when the
-	const include_header = T &redef;
+	## file was opened and closes (the latter at the end). 
+	const include_meta = T &redef;
 
-	## Prefix for the header line if included.
+	## Prefix for lines with meta information.
-	const header_prefix = "#" &redef;
+	const meta_prefix = "#" &redef;
 
 	## Separator between fields.
 	const separator = "\t" &redef;

scripts/base/frameworks/logging/writers/dataseries.bro

@@ -0,0 +1,60 @@
+##! Interface for the DataSeries log writer.
+
+module LogDataSeries;
+
+export {
+	## Compression to use with the DS output file.  Options are:
+	##
+	## 'none' -- No compression.
+	## 'lzf'  -- LZF compression.   Very quick, but leads to larger output files.
+	## 'lzo'  -- LZO compression.   Very fast decompression times.
+	## 'gz'   -- GZIP compression.  Slower than LZF, but also produces smaller output.
+	## 'bz2'  -- BZIP2 compression. Slower than GZIP, but also produces smaller output.
+	const compression = "gz" &redef;
+
+	## The extent buffer size.
+	## Larger values here lead to better compression and more efficient writes, but
+	## also increase the lag between the time events are received and the time they
+	## are actually written to disk.
+	const extent_size = 65536 &redef;
+
+	## Should we dump the XML schema we use for this DS file to disk?
+	## If yes, the XML schema shares the name of the logfile, but has
+	## an XML ending.
+	const dump_schema = F &redef;
+
+	## How many threads should DataSeries spawn to perform compression?
+	## Note that this dictates the number of threads per log stream.  If
+	## you're using a lot of streams, you may want to keep this number
+	## relatively small.
+	##
+	## Default value is 1, which will spawn one thread / stream.
+	##
+	## Maximum is 128, minimum is 1.
+	const num_threads = 1 &redef;
+
+	## Should time be stored as an integer or a double?
+	## Storing time as a double leads to possible precision issues and
+	## can (significantly) increase the size of the resulting DS log.
+	## That said, timestamps stored in double form are consistent
+	## with the rest of Bro, including the standard ASCII log. Hence, we
+	## use them by default.
+	const use_integer_for_time = F &redef;
+}
+
+# Default function to postprocess a rotated DataSeries log file. It moves the
+# rotated file to a new name that includes a timestamp with the opening time, and
+# then runs the writer's default postprocessor command on it.
+function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
+	{
+	# Move file to name including both opening and closing time.
+	local dst = fmt("%s.%s.ds", info$path,
+			strftime(Log::default_rotation_date_format, info$open));
+
+	system(fmt("/bin/mv %s %s", info$fname, dst));
+
+	# Run default postprocessor.
+	return Log::run_rotation_postprocessor_cmd(info, dst);
+	}
+
+redef Log::default_rotation_postprocessors += { [Log::WRITER_DATASERIES] = default_rotation_postprocessor_func };

scripts/base/frameworks/logging/writers/elasticsearch.bro

@@ -0,0 +1,48 @@
+##! Log writer for sending logs to an ElasticSearch server.
+##!
+##! Note: This module is in testing and is not yet considered stable!
+##!
+##! There is one known memory issue.  If your elasticsearch server is
+##! running slowly and taking too long to return from bulk insert
+##! requests, the message queue to the writer thread will continue
+##! growing larger and larger giving the appearance of a memory leak.
+
+module LogElasticSearch;
+
+export {
+	## Name of the ES cluster
+	const cluster_name = "elasticsearch" &redef;
+
+	## ES Server
+	const server_host = "127.0.0.1" &redef;
+
+	## ES Port
+	const server_port = 9200 &redef;
+
+	## Name of the ES index
+	const index_prefix = "bro" &redef;
+
+	## The ES type prefix comes before the name of the related log.
+	## e.g. prefix = "bro\_" would create types of bro_dns, bro_software, etc.
+	const type_prefix = "" &redef;
+
+	## The time before an ElasticSearch transfer will timeout. Note that
+	## the fractional part of the timeout will be ignored. In particular, time
+	## specifications less than a second result in a timeout value of 0, which
+	## means "no timeout."
+	const transfer_timeout = 2secs;
+
+	## The batch size is the number of messages that will be queued up before
+	## they are sent to be bulk indexed.
+	const max_batch_size = 1000 &redef;
+
+	## The maximum amount of wall-clock time that is allowed to pass without
+	## finishing a bulk log send.  This represents the maximum delay you
+	## would like to have with your logs before they are sent to ElasticSearch.
+	const max_batch_interval = 1min &redef;
+
+	## The maximum byte size for a buffered JSON string to send to the bulk
+	## insert API.
+	const max_byte_size = 1024 * 1024 &redef;
+}
+

scripts/base/frameworks/logging/writers/none.bro

@@ -0,0 +1,17 @@
+##! Interface for the None log writer. Thiis writer is mainly for debugging.
+
+module LogNone;
+
+export {
+	## If true, output debugging output that can be useful for unit
+        ## testing the logging framework.
+	const debug = F &redef;
+}
+
+function default_rotation_postprocessor_func(info: Log::RotationInfo) : bool
+	{
+	return T;
+	}
+
+redef Log::default_rotation_postprocessors += { [Log::WRITER_NONE] = default_rotation_postprocessor_func };
+

scripts/base/frameworks/notice/actions/pp-alarms.bro

@@ -101,7 +101,7 @@ event bro_init()
 	# This replaces the standard non-pretty-printing filter.
 	Log::add_filter(Notice::ALARM_LOG,
 			[$name="alarm-mail", $writer=Log::WRITER_NONE,
-			 $interv=Log::default_rotation_interval,
+			 $interv=Log::default_mail_alarms_interval,
 			 $postprocessor=pp_postprocessor]);
 	}
 

scripts/base/frameworks/notice/cluster.bro

@@ -23,7 +23,10 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 # The notice policy is completely handled by the manager and shouldn't be 
 # done by workers or proxies to save time for packet processing.
-redef policy = {};
+event bro_init() &priority=11
+	{
+	Notice::policy = table();
+	}
 
 event Notice::begin_suppression(n: Notice::Info)
 	{

scripts/base/frameworks/reporter/main.bro

@@ -36,24 +36,55 @@ export {
 		## Not all reporter messages will have locations in them though.
 		location: string &log &optional;
 	};
+
+	## Tunable for sending reporter warning messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const warnings_to_stderr = T &redef;
+
+	## Tunable for sending reporter error messages to STDERR.  The option to
+	## turn it off is presented here in case Bro is being run by some
+	## external harness and shouldn't output anything to the console.
+	const errors_to_stderr = T &redef;
 }
 
+global stderr: file;
+
 event bro_init() &priority=5
 	{
 	Log::create_stream(Reporter::LOG, [$columns=Info]);
+
+	if ( errors_to_stderr || warnings_to_stderr )
+		stderr = open("/dev/stderr");
 	}
 
-event reporter_info(t: time, msg: string, location: string)
+event reporter_info(t: time, msg: string, location: string) &priority=-5
 	{
 	Log::write(Reporter::LOG, [$ts=t, $level=INFO, $message=msg, $location=location]);
 	}
 
-event reporter_warning(t: time, msg: string, location: string)
+event reporter_warning(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( warnings_to_stderr )
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("WARNING: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("WARNING: %s (%s)", msg, location);
+		}
+
 	Log::write(Reporter::LOG, [$ts=t, $level=WARNING, $message=msg, $location=location]);
 	}
 
-event reporter_error(t: time, msg: string, location: string)
+event reporter_error(t: time, msg: string, location: string) &priority=-5
 	{
+	if ( errors_to_stderr )
+		{
+		if ( t > double_to_time(0.0) )
+			print stderr, fmt("ERROR: %.6f %s (%s)", t, msg, location);
+		else
+			print stderr, fmt("ERROR: %s (%s)", msg, location);
+		}
+
 	Log::write(Reporter::LOG, [$ts=t, $level=ERROR, $message=msg, $location=location]);
 	}

scripts/base/frameworks/tunnels/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/tunnels/main.bro

@@ -0,0 +1,149 @@
+##! This script handles the tracking/logging of tunnels (e.g. Teredo,
+##! AYIYA, or IP-in-IP such as 6to4 where "IP" is either IPv4 or IPv6).
+##!
+##! For any connection that occurs over a tunnel, information about its
+##! encapsulating tunnels is also found in the *tunnel* field of
+##! :bro:type:`connection`.
+
+module Tunnel;
+
+export {
+	## The tunnel logging stream identifier.
+	redef enum Log::ID += { LOG };
+
+	## Types of interesting activity that can occur with a tunnel.
+	type Action: enum {
+		## A new tunnel (encapsulating "connection") has been seen.
+		DISCOVER,
+		## A tunnel connection has closed.
+		CLOSE,
+		## No new connections over a tunnel happened in the amount of
+		## time indicated by :bro:see:`Tunnel::expiration_interval`.
+		EXPIRE,
+	};
+
+	## The record type which contains column fields of the tunnel log.
+	type Info: record {
+		## Time at which some tunnel activity occurred.
+		ts:          time         &log;
+		## The unique identifier for the tunnel, which may correspond
+		## to a :bro:type:`connection`'s *uid* field for non-IP-in-IP tunnels.
+		## This is optional because there could be numerous connections
+		## for payload proxies like SOCKS but we should treat it as a single
+		## tunnel.
+		uid:         string       &log &optional;
+		## The tunnel "connection" 4-tuple of endpoint addresses/ports.
+		## For an IP tunnel, the ports will be 0.
+		id:          conn_id      &log;
+		## The type of tunnel.
+		tunnel_type: Tunnel::Type &log;
+		## The type of activity that occurred.
+		action:      Action       &log;
+	};
+
+	## Logs all tunnels in an encapsulation chain with action
+	## :bro:see:`Tunnel::DISCOVER` that aren't already in the
+	## :bro:id:`Tunnel::active` table and adds them if not.
+	global register_all: function(ecv: EncapsulatingConnVector);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::DISCOVER` if it's not already in the
+	## :bro:id:`Tunnel::active` table and adds it if not.
+	global register: function(ec: EncapsulatingConn);
+
+	## Logs a single tunnel "connection" with action
+	## :bro:see:`Tunnel::EXPIRE` and removes it from the
+	## :bro:id:`Tunnel::active` table.
+	##
+	## t: A table of tunnels.
+	##
+	## idx: The index of the tunnel table corresponding to the tunnel to expire.
+	##
+	## Returns: 0secs, which when this function is used as an
+	##          :bro:attr:`&expire_func`, indicates to remove the element at
+	##          *idx* immediately.
+	global expire: function(t: table[conn_id] of Info, idx: conn_id): interval;
+
+	## Removes a single tunnel from the :bro:id:`Tunnel::active` table
+	## and logs the closing/expiration of the tunnel.
+	##
+	## tunnel: The tunnel which has closed or expired.
+	##
+	## action: The specific reason for the tunnel ending.
+	global close: function(tunnel: Info, action: Action);
+
+	## The amount of time a tunnel is not used in establishment of new
+	## connections before it is considered inactive/expired.
+	const expiration_interval = 1hrs &redef;
+
+	## Currently active tunnels.  That is, tunnels for which new, encapsulated
+	## connections have been seen in the interval indicated by
+	## :bro:see:`Tunnel::expiration_interval`.
+	global active: table[conn_id] of Info = table() &read_expire=expiration_interval &expire_func=expire;
+}
+
+const ayiya_ports = { 5072/udp };
+redef dpd_config += { [ANALYZER_AYIYA] = [$ports = ayiya_ports] };
+
+const teredo_ports = { 3544/udp };
+redef dpd_config += { [ANALYZER_TEREDO] = [$ports = teredo_ports] };
+
+redef likely_server_ports += { ayiya_ports, teredo_ports };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Tunnel::LOG, [$columns=Info]);
+	}
+
+function register_all(ecv: EncapsulatingConnVector)
+	{
+	for ( i in ecv )
+		register(ecv[i]);
+	}
+
+function register(ec: EncapsulatingConn)
+	{
+	if ( ec$cid !in active )
+		{
+		local tunnel: Info;
+		tunnel$ts = network_time();
+		if ( ec?$uid )
+			tunnel$uid = ec$uid;
+		tunnel$id = ec$cid;
+		tunnel$action = DISCOVER;
+		tunnel$tunnel_type = ec$tunnel_type;
+		active[ec$cid] = tunnel;
+		Log::write(LOG, tunnel);
+		}
+	}
+
+function close(tunnel: Info, action: Action)
+	{
+	tunnel$action = action;
+	tunnel$ts = network_time();
+	Log::write(LOG, tunnel);
+	delete active[tunnel$id];
+	}
+
+function expire(t: table[conn_id] of Info, idx: conn_id): interval
+	{
+	close(t[idx], EXPIRE);
+	return 0secs;
+	}
+
+event new_connection(c: connection) &priority=5
+	{
+	if ( c?$tunnel )
+		register_all(c$tunnel);
+	}
+
+event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
+	{
+	register_all(e);
+	}
+
+event connection_state_remove(c: connection) &priority=-5
+	{
+	if ( c$id in active )
+		close(active[c$id], CLOSE);
+	}

scripts/base/init-bare.bro

@@ -115,6 +115,61 @@ type icmp_context: record {
 	DF: bool;	##< True if the packets *don't fragment* flag is set.
 };
 
+## Values extracted from a Prefix Information option in an ICMPv6 neighbor
+## discovery message as specified by :rfc:`4861`.
+##
+## .. bro:see:: icmp6_nd_option
+type icmp6_nd_prefix_info: record {
+	## Number of leading bits of the *prefix* that are valid.
+	prefix_len: count;
+	## Flag indicating the prefix can be used for on-link determination.
+	L_flag: bool;
+	## Autonomous address-configuration flag.
+	A_flag: bool;
+	## Length of time in seconds that the prefix is valid for purpose of
+	## on-link determination (0xffffffff represents infinity).
+	valid_lifetime: interval;
+	## Length of time in seconds that the addresses generated from the prefix
+	## via stateless address autoconfiguration remain preferred
+	## (0xffffffff represents infinity).
+	preferred_lifetime: interval;
+	## An IP address or prefix of an IP address.  Use the *prefix_len* field
+	## to convert this into a :bro:type:`subnet`.
+	prefix: addr;
+};
+
+## Options extracted from ICMPv6 neighbor discovery messages as specified
+## by :rfc:`4861`.
+##
+## .. bro:see:: icmp_router_solicitation icmp_router_advertisement
+##    icmp_neighbor_advertisement icmp_neighbor_solicitation icmp_redirect
+##    icmp6_nd_options
+type icmp6_nd_option: record {
+	## 8-bit identifier of the type of option.
+	otype:        count;
+	## 8-bit integer representing the length of the option (including the type
+	## and length fields) in units of 8 octets.
+	len:          count;
+	## Source Link-Layer Address (Type 1) or Target Link-Layer Address (Type 2).
+	## Byte ordering of this is dependent on the actual link-layer.
+	link_address: string &optional;
+	## Prefix Information (Type 3).
+	prefix:       icmp6_nd_prefix_info &optional;
+	## Redirected header (Type 4).  This field contains the context of the
+	## original, redirected packet.
+	redirect:     icmp_context &optional;
+	## Recommended MTU for the link (Type 5).
+	mtu:          count &optional;
+	## The raw data of the option (everything after type & length fields),
+	## useful for unknown option types or when the full option payload is
+	## truncated in the captured packet.  In those cases, option fields
+	## won't be pre-extracted into the fields above.
+	payload:      string &optional;
+};
+
+## A type alias for a vector of ICMPv6 neighbor discovery message options.
+type icmp6_nd_options: vector of icmp6_nd_option;
+
 # A DNS mapping between IP address and hostname resolved by Bro's internal
 # resolver.
 #
@@ -178,9 +233,35 @@ type endpoint_stats: record {
 ##    use ``count``. That should be changed.
 type AnalyzerID: count;
 
-## Statistics about an endpoint.
+module Tunnel;
+export {
+	## Records the identity of an encapsulating parent of a tunneled connection.
+	type EncapsulatingConn: record {
+		## The 4-tuple of the encapsulating "connection". In case of an IP-in-IP
+		## tunnel the ports will be set to 0. The direction (i.e., orig and
+		## resp) are set according to the first tunneled packet seen
+		## and not according to the side that established the tunnel. 
+		cid: conn_id;
+		## The type of tunnel.
+		tunnel_type: Tunnel::Type;
+		## A globally unique identifier that, for non-IP-in-IP tunnels,
+		## cross-references the *uid* field of :bro:type:`connection`.
+		uid: string &optional;
+	} &log;
+} # end export
+module GLOBAL;
+
+## A type alias for a vector of encapsulating "connections", i.e for when
+## there are tunnels within tunnels.
 ##
-## todo::Where is this used?
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type EncapsulatingConnVector: vector of Tunnel::EncapsulatingConn;
+
+## Statistics about a :bro:type:`connection` endpoint.
+##
+## .. bro:see:: connection
 type endpoint: record {
 	size: count;	##< Logical size of data sent (for TCP: derived from sequence numbers).
 	## Endpoint state. For TCP connection, one of the constants:
@@ -194,12 +275,15 @@ type endpoint: record {
 	## Number of IP-level bytes sent. Only set if :bro:id:`use_conn_size_analyzer` is
 	## true.
 	num_bytes_ip: count &optional;
+	## The current IPv6 flow label that the connection endpoint is using.
+	## Always 0 if the connection is over IPv4.
+	flow_label: count;
 };
 
-# A connection. This is Bro's basic connection type describing IP- and
+## A connection. This is Bro's basic connection type describing IP- and
-# transport-layer information about the conversation. Note that Bro uses a
+## transport-layer information about the conversation. Note that Bro uses a
-# liberal interpreation of "connection" and associates instances of this type
+## liberal interpreation of "connection" and associates instances of this type
-# also with UDP and ICMP flows.
+## also with UDP and ICMP flows.
 type connection: record {
 	id: conn_id;	##< The connection's identifying 4-tuple.
 	orig: endpoint;	##< Statistics about originator side.
@@ -219,11 +303,17 @@ type connection: record {
         service: set[string];
 	addl: string;	##< Deprecated.
 	hot: count;	##< Deprecated.
-	history: string;	##< State history of TCP connections. See *history* in :bro:see:`Conn::Info`.
+	history: string;	##< State history of connections. See *history* in :bro:see:`Conn::Info`.
 	## A globally unique connection identifier. For each connection, Bro creates an ID
 	## that is very likely unique across independent Bro runs. These IDs can thus be
 	## used to tag and locate information  associated with that connection.
 	uid: string;
+	## If the connection is tunneled, this field contains information about
+	## the encapsulating "connection(s)" with the outermost one starting
+	## at index zero.  It's also always the first such enapsulation seen
+	## for the connection unless the :bro:id:`tunnel_changed` event is handled
+	## and re-assigns this field to the new encapsulation.
+	tunnel: EncapsulatingConnVector &optional;
 };
 
 ## Fields of a SYN packet.
@@ -612,7 +702,9 @@ function add_signature_file(sold: string, snew: string): string
 	}
 
 ## Signature files to read. Use ``redef signature_files  += "foo.sig"`` to
-## extend. Signature files will be searched relative to ``BROPATH``.
+## extend. Signature files added this way will be searched relative to
+## ``BROPATH``.  Using the ``@load-sigs`` directive instead is preferred
+## since that can search paths relative to the current script.
 global signature_files = "" &add_func = add_signature_file;
 
 ## ``p0f`` fingerprint file to use. Will be searched relative to ``BROPATH``.
@@ -721,7 +813,7 @@ const tcp_storm_interarrival_thresh = 1 sec &redef;
 ## peer's ACKs.  Set to zero to turn off this determination.
 ##
 ## .. bro:see:: tcp_max_above_hole_without_any_acks tcp_excessive_data_without_further_acks
-const tcp_max_initial_window = 4096;
+const tcp_max_initial_window = 4096 &redef;
 
 ## If we're not seeing our peer's ACKs, the maximum volume of data above a sequence
 ## hole that we'll tolerate before assuming that there's been a packet drop and we
@@ -729,7 +821,7 @@ const tcp_max_initial_window = 4096;
 ## up.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_excessive_data_without_further_acks
-const tcp_max_above_hole_without_any_acks = 4096;
+const tcp_max_above_hole_without_any_acks = 4096 &redef;
 
 ## If we've seen this much data without any of it being acked, we give up
 ## on that connection to avoid memory exhaustion due to buffering all that
@@ -738,7 +830,7 @@ const tcp_max_above_hole_without_any_acks = 4096;
 ## has in fact gone too far, but for now we just make this quite beefy.
 ##
 ## .. bro:see:: tcp_max_initial_window tcp_max_above_hole_without_any_acks
-const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024;
+const tcp_excessive_data_without_further_acks = 10 * 1024 * 1024 &redef;
 
 ## For services without an a handler, these sets define originator-side ports that
 ## still trigger reassembly.
@@ -866,18 +958,9 @@ const frag_timeout = 0.0 sec &redef;
 const packet_sort_window = 0 usecs &redef;
 
 ## If positive, indicates the encapsulation header size that should
-## be skipped. This either applies to all packets, or if
+## be skipped. This applies to all packets.
-## :bro:see:`tunnel_port` is set, only to packets on that port.
-##
-## .. :bro:see:: tunnel_port
 const encap_hdr_size = 0 &redef;
 
-## A UDP port that specifies which connections to apply :bro:see:`encap_hdr_size`
-## to.
-##
-## .. :bro:see:: encap_hdr_size
-const tunnel_port = 0/udp &redef;
-
 ## Whether to use the ``ConnSize`` analyzer to count the number of packets and
 ## IP-level bytes transfered by each endpoint. If true, these values are returned
 ## in the connection's :bro:see:`endpoint`  record value.
@@ -954,16 +1037,19 @@ const IPPROTO_MOBILITY = 135;		##< IPv6 mobility header.
 ## Values extracted from an IPv6 extension header's (e.g. hop-by-hop or
 ## destination option headers) option field.
 ##
-## .. bro:see:: ip6_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts
+## .. bro:see:: ip6_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts
 type ip6_option: record {
 	otype: count;	##< Option type.
 	len: count;		##< Option data length.
 	data: string;	##< Option data.
 };
 
+## A type alias for a vector of IPv6 options.
+type ip6_options: vector of ip6_option;
+
 ## Values extracted from an IPv6 Hop-by-Hop options extension header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option
 type ip6_hopopts: record {
 	## Protocol number of the next header (RFC 1700 et seq., IANA assigned
 	## number), e.g. :bro:id:`IPPROTO_ICMP`.
@@ -971,12 +1057,12 @@ type ip6_hopopts: record {
 	## Length of header in 8-octet units, excluding first unit.
 	len: count;
 	## The TLV encoded options;
-	options: vector of ip6_option;
+	options: ip6_options;
 };
 
 ## Values extracted from an IPv6 Destination options extension header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain ip6_option
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr ip6_option
 type ip6_dstopts: record {
 	## Protocol number of the next header (RFC 1700 et seq., IANA assigned
 	## number), e.g. :bro:id:`IPPROTO_ICMP`.
@@ -984,12 +1070,12 @@ type ip6_dstopts: record {
 	## Length of header in 8-octet units, excluding first unit.
 	len: count;
 	## The TLV encoded options;
-	options: vector of ip6_option;
+	options: ip6_options;
 };
 
 ## Values extracted from an IPv6 Routing extension header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
 type ip6_routing: record {
 	## Protocol number of the next header (RFC 1700 et seq., IANA assigned
 	## number), e.g. :bro:id:`IPPROTO_ICMP`.
@@ -1006,7 +1092,7 @@ type ip6_routing: record {
 
 ## Values extracted from an IPv6 Fragment extension header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
 type ip6_fragment: record {
 	## Protocol number of the next header (RFC 1700 et seq., IANA assigned
 	## number), e.g. :bro:id:`IPPROTO_ICMP`.
@@ -1025,7 +1111,7 @@ type ip6_fragment: record {
 
 ## Values extracted from an IPv6 Authentication extension header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
 type ip6_ah: record {
 	## Protocol number of the next header (RFC 1700 et seq., IANA assigned
 	## number), e.g. :bro:id:`IPPROTO_ICMP`.
@@ -1036,15 +1122,15 @@ type ip6_ah: record {
 	rsv: count;
 	## Security Parameter Index.
 	spi: count;
-	## Sequence number.
+	## Sequence number, unset in the case that *len* field is zero.
-	seq: count;
+	seq: count &optional;
-	## Authentication data.
+	## Authentication data, unset in the case that *len* field is zero.
-	data: string;
+	data: string &optional;
 };
 
 ## Values extracted from an IPv6 ESP extension header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
 type ip6_esp: record {
 	## Security Parameters Index.
 	spi: count;
@@ -1054,7 +1140,7 @@ type ip6_esp: record {
 
 ## Values extracted from an IPv6 Mobility Binding Refresh Request message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_brr: record {
 	## Reserved.
 	rsv: count;
@@ -1064,7 +1150,7 @@ type ip6_mobility_brr: record {
 
 ## Values extracted from an IPv6 Mobility Home Test Init message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_hoti: record {
 	## Reserved.
 	rsv: count;
@@ -1076,7 +1162,7 @@ type ip6_mobility_hoti: record {
 
 ## Values extracted from an IPv6 Mobility Care-of Test Init message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_coti: record {
 	## Reserved.
 	rsv: count;
@@ -1088,7 +1174,7 @@ type ip6_mobility_coti: record {
 
 ## Values extracted from an IPv6 Mobility Home Test message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_hot: record {
 	## Home Nonce Index.
 	nonce_idx: count;
@@ -1102,7 +1188,7 @@ type ip6_mobility_hot: record {
 
 ## Values extracted from an IPv6 Mobility Care-of Test message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_cot: record {
 	## Care-of Nonce Index.
 	nonce_idx: count;
@@ -1116,7 +1202,7 @@ type ip6_mobility_cot: record {
 
 ## Values extracted from an IPv6 Mobility Binding Update message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_bu: record {
 	## Sequence number.
 	seq: count;
@@ -1136,7 +1222,7 @@ type ip6_mobility_bu: record {
 
 ## Values extracted from an IPv6 Mobility Binding Acknowledgement message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_back: record {
 	## Status.
 	status: count;
@@ -1152,7 +1238,7 @@ type ip6_mobility_back: record {
 
 ## Values extracted from an IPv6 Mobility Binding Error message.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain ip6_mobility_msg
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr ip6_mobility_msg
 type ip6_mobility_be: record {
 	## Status.
 	status: count;
@@ -1164,7 +1250,7 @@ type ip6_mobility_be: record {
 
 ## Values extracted from an IPv6 Mobility header's message data.
 ##
-## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_hdr_chain
+## .. bro:see:: ip6_mobility_hdr ip6_hdr ip6_ext_hdr
 type ip6_mobility_msg: record {
 	## The type of message from the header's MH Type field.
 	id: count;
@@ -1188,7 +1274,7 @@ type ip6_mobility_msg: record {
 
 ## Values extracted from an IPv6 Mobility header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_hdr_chain
+## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr ip6_ext_hdr
 type ip6_mobility_hdr: record {
 	## Protocol number of the next header (RFC 1700 et seq., IANA assigned
 	## number), e.g. :bro:id:`IPPROTO_ICMP`.
@@ -1229,9 +1315,12 @@ type ip6_ext_hdr: record {
 	mobility: ip6_mobility_hdr &optional;
 };
 
+## A type alias for a vector of IPv6 extension headers.
+type ip6_ext_hdr_chain: vector of ip6_ext_hdr;
+
 ## Values extracted from an IPv6 header.
 ##
-## .. bro:see:: pkt_hdr ip4_hdr ip6_hdr_chain ip6_hopopts ip6_dstopts
+## .. bro:see:: pkt_hdr ip4_hdr ip6_ext_hdr ip6_hopopts ip6_dstopts
 ##    ip6_routing ip6_fragment ip6_ah ip6_esp
 type ip6_hdr: record {
 	class: count;					##< Traffic class.
@@ -1243,7 +1332,7 @@ type ip6_hdr: record {
 	hlim: count;					##< Hop limit.
 	src: addr;					##< Source address.
 	dst: addr;					##< Destination address.
-	exts: vector of ip6_ext_hdr;			##< Extension header chain.
+	exts: ip6_ext_hdr_chain;			##< Extension header chain.
 };
 
 ## Values extracted from an IPv4 header.
@@ -1312,6 +1401,42 @@ type pkt_hdr: record {
 	icmp: icmp_hdr &optional;		##< The ICMP header if an ICMP packet.
 };
 
+## A Teredo origin indication header.  See :rfc:`4380` for more information
+## about the Teredo protocol.
+##
+## .. bro:see:: teredo_bubble teredo_origin_indication teredo_authentication
+##    teredo_hdr
+type teredo_auth: record {
+	id:      string;  ##< Teredo client identifier.
+	value:   string;  ##< HMAC-SHA1 over shared secret key between client and
+	                  ##< server, nonce, confirmation byte, origin indication
+	                  ##< (if present), and the IPv6 packet.
+	nonce:   count;   ##< Nonce chosen by Teredo client to be repeated by
+	                  ##< Teredo server.
+	confirm: count;   ##< Confirmation byte to be set to 0 by Teredo client
+	                  ##< and non-zero by server if client needs new key.
+};
+
+## A Teredo authentication header.  See :rfc:`4380` for more information
+## about the Teredo protocol.
+##
+## .. bro:see:: teredo_bubble teredo_origin_indication teredo_authentication
+##    teredo_hdr
+type teredo_origin: record {
+	p: port; ##< Unobfuscated UDP port of Teredo client.
+	a: addr; ##< Unobfuscated IPv4 address of Teredo client.
+};
+
+## A Teredo packet header.  See :rfc:`4380` for more information about the
+## Teredo protocol.
+##
+## .. bro:see:: teredo_bubble teredo_origin_indication teredo_authentication
+type teredo_hdr: record {
+	auth:   teredo_auth &optional;   ##< Teredo authentication header.
+	origin: teredo_origin &optional; ##< Teredo origin indication header.
+	hdr:    pkt_hdr;                 ##< IPv6 and transport protocol headers.
+};
+
 ## Definition of "secondary filters". A secondary filter is a BPF filter given as
 ## index in this table. For each such filter, the corresponding event is raised for
 ## all matching packets.
@@ -1789,6 +1914,14 @@ export {
 	};
 } # end export
 
+module Threading;
+
+export {
+	## The heartbeat interval used by the threading framework.
+	## Changing this should usually not be neccessary and will break several tests.
+	const heartbeat_interval = 1.0 secs &redef;
+}
+
 module GLOBAL;
 
 ## An NTP message.
@@ -2311,11 +2444,37 @@ type bittorrent_benc_dir: table[string] of bittorrent_benc_value;
 ##    bt_tracker_response_not_ok
 type bt_tracker_headers: table[string] of string;
 
+type ModbusCoils: vector of bool;
+type ModbusRegisters: vector of count;
+
+type ModbusHeaders: record {
+	tid:           count;
+	pid:           count;
+	uid:           count;
+	function_code: count;
+};
+
+module SOCKS;
+export {
+	## This record is for a SOCKS client or server to provide either a 
+	## name or an address to represent a desired or established connection.
+	type Address: record {
+		host: addr   &optional;
+		name: string &optional;
+	} &log;
+}
+module GLOBAL;
+
 @load base/event.bif
 
 ## BPF filter the user has set via the -f command line options. Empty if none.
 const cmd_line_bpf_filter = "" &redef;
 
+## The maximum number of open files to keep cached at a given time.
+## If set to zero, this is automatically determined by inspecting
+## the current/maximum limit on open files for the process.
+const max_files_in_cache = 0 &redef;
+
 ## Deprecated.
 const log_rotate_interval = 0 sec &redef;
 
@@ -2599,11 +2758,41 @@ const record_all_packets = F &redef;
 ## .. bro:see:: conn_stats
 const ignore_keep_alive_rexmit = F &redef;
 
-## Whether the analysis engine parses IP packets encapsulated in
+module Tunnel;
-## UDP tunnels.
+export {
-##
+	## The maximum depth of a tunnel to decapsulate until giving up.
-## .. bro:see:: tunnel_port
+	## Setting this to zero will disable all types of tunnel decapsulation.
-const parse_udp_tunnels = F &redef;
+	const max_depth: count = 2 &redef;
+
+	## Toggle whether to do IPv{4,6}-in-IPv{4,6} decapsulation.
+	const enable_ip = T &redef;
+
+	## Toggle whether to do IPv{4,6}-in-AYIYA decapsulation.
+	const enable_ayiya = T &redef;
+
+	## Toggle whether to do IPv6-in-Teredo decapsulation.
+	const enable_teredo = T &redef;
+
+	## With this option set, the Teredo analysis will first check to see if
+	## other protocol analyzers have confirmed that they think they're
+	## parsing the right protocol and only continue with Teredo tunnel
+	## decapsulation if nothing else has yet confirmed.  This can help
+	## reduce false positives of UDP traffic (e.g. DNS) that also happens
+	## to have a valid Teredo encapsulation.
+	const yielding_teredo_decapsulation = T &redef;
+
+	## With this set, the Teredo analyzer waits until it sees both sides
+	## of a connection using a valid Teredo encapsulation before issuing
+	## a :bro:see:`protocol_confirmation`.  If it's false, the first
+	## occurence of a packet with valid Teredo encapsulation causes a
+	## confirmation.  Both cases are still subject to effects of
+	## :bro:see:`Tunnel::yielding_teredo_decapsulation`.
+	const delay_teredo_confirmation = T &redef;
+
+	## How often to cleanup internal state for inactive IP tunnels.
+	const ip_tunnel_timeout = 24hrs &redef;
+} # end export
+module GLOBAL;
 
 ## Number of bytes per packet to capture from live interfaces.
 const snaplen = 8192 &redef;
@@ -2611,3 +2800,6 @@ const snaplen = 8192 &redef;
 # Load the logging framework here because it uses fairly deep integration with
 # BiFs and script-land defined types.
 @load base/frameworks/logging
+
+@load base/frameworks/input
+

scripts/base/init-default.bro

@@ -14,6 +14,7 @@
 @load base/utils/patterns
 @load base/utils/strings
 @load base/utils/thresholds
+@load base/utils/urls
 
 # This has some deep interplay between types and BiFs so it's 
 # loaded in base/init-bare.bro
@@ -30,13 +31,18 @@
 @load base/frameworks/intel
 @load base/frameworks/reporter
 @load base/frameworks/protocols
+@load base/frameworks/tunnels
 
 @load base/protocols/conn
 @load base/protocols/dns
 @load base/protocols/ftp
 @load base/protocols/http
 @load base/protocols/irc
+@load base/protocols/modbus
 @load base/protocols/smtp
+@load base/protocols/socks
 @load base/protocols/ssh
 @load base/protocols/ssl
 @load base/protocols/syslog
+
+@load base/misc/find-checksum-offloading

scripts/base/misc/find-checksum-offloading.bro

@@ -0,0 +1,57 @@
+##! Discover cases where the local interface is sniffed and outbound packets
+##! have checksum offloading.  Load this script to receive a notice if it's
+##! likely that checksum offload effects are being seen on a live interface or
+##! in a packet trace file.
+
+@load base/frameworks/notice
+
+module ChecksumOffloading;
+
+export {
+	## The interval which is used for checking packet statistics
+	## to see if checksum offloading is affecting analysis.
+	const check_interval = 10secs &redef;
+}
+
+# Keep track of how many bad checksums have been seen.
+global bad_checksums = 0;
+
+# Track to see if this script is done so that messages aren't created multiple times.
+global done = F;
+
+event ChecksumOffloading::check()
+	{
+	if ( done )
+		return;
+
+	local pkts_recvd = net_stats()$pkts_recvd;
+	if ( (bad_checksums*1.0 / net_stats()$pkts_recvd*1.0) > 0.05 )
+		{
+		local packet_src = reading_traces() ? "trace file likely has" : "interface is likely receiving";
+		local message = fmt("Your %s invalid IP checksums, most likely from NIC checksum offloading.", packet_src);
+		Reporter::warning(message);
+		done = T;
+		}
+	else if ( pkts_recvd < 20 )
+		{
+		# Keep scheduling this event until we've seen some lower threshold of
+		# total packets.
+		schedule check_interval { ChecksumOffloading::check() };
+		}
+	}
+
+event bro_init()
+	{
+	schedule check_interval { ChecksumOffloading::check() };
+	}
+
+event net_weird(name: string)
+	{
+	if ( name == "bad_IP_checksum" )
+		++bad_checksums;
+	}
+
+event bro_done()
+	{
+	event ChecksumOffloading::check();
+	}

scripts/base/protocols/conn/__load__.bro

@@ -1,3 +1,4 @@
 @load ./main
 @load ./contents
 @load ./inactivity
+@load ./polling

scripts/base/protocols/conn/main.bro

@@ -17,7 +17,7 @@ export {
 	type Info: record {
 		## This is the time of the first packet.
 		ts:           time            &log;
-		## A unique identifier of a connection.
+		## A unique identifier of the connection.
 		uid:          string          &log;
 		## The connection's 4-tuple of endpoint addresses/ports.
 		id:           conn_id         &log;
@@ -61,14 +61,14 @@ export {
 		## be left empty at all times.
 		local_orig:   bool            &log &optional;
 
-		## Indicates the number of bytes missed in content gaps which is
+		## Indicates the number of bytes missed in content gaps, which is
 		## representative of packet loss.  A value other than zero will
 		## normally cause protocol analysis to fail but some analysis may
 		## have been completed prior to the packet loss.
 		missed_bytes: count           &log &default=0;
 
 		## Records the state history of connections as a string of letters.
-		## For TCP connections the meaning of those letters is:
+		## The meaning of those letters is:
 		##
 		## ======  ====================================================
 		## Letter  Meaning
@@ -83,24 +83,29 @@ export {
 		## i       inconsistent packet (e.g. SYN+RST bits both set)
 		## ======  ====================================================
 		##
-		## If the letter is in upper case it means the event comes from the
+		## If the event comes from the originator, the letter is in upper-case; if it comes
-		## originator and lower case then means the responder.
+		## from the responder, it's in lower-case. Multiple packets of the same type will
-		## Also, there is compression. We only record one "d" in each direction,
+		## only be noted once (e.g. we only record one "d" in each direction, regardless of
-		## for instance. I.e., we just record that data went in that direction.
+		## how many data packets were seen.)
-		## This history is not meant to encode how much data that happened to
-		## be.
 		history:      string          &log &optional;
-		## Number of packets the originator sent.
+		## Number of packets that the originator sent.
 		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_pkts:     count      &log &optional;
-		## Number IP level bytes the originator sent (as seen on the wire,
+		## Number of IP level bytes that the originator sent (as seen on the wire,
 		## taken from IP total_length header field).
 		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		orig_ip_bytes: count      &log &optional;
-		## Number of packets the responder sent. See ``orig_pkts``.
+		## Number of packets that the responder sent.
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		resp_pkts:     count      &log &optional;
-		## Number IP level bytes the responder sent. See ``orig_pkts``.
+		## Number og IP level bytes that the responder sent (as seen on the wire,
+		## taken from IP total_length header field).
+		## Only set if :bro:id:`use_conn_size_analyzer` = T
 		resp_ip_bytes: count      &log &optional;
+		## If this connection was over a tunnel, indicate the
+		## *uid* values for any encapsulating parent connections
+		## used over the lifetime of this inner connection.
+		tunnel_parents: set[string] &log;
 	};
 
 	## Event that can be handled to access the :bro:type:`Conn::Info`
@@ -190,6 +195,8 @@ function set_conn(c: connection, eoc: bool)
 	c$conn$ts=c$start_time;
 	c$conn$uid=c$uid;
 	c$conn$id=c$id;
+	if ( c?$tunnel && |c$tunnel| > 0 )
+		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
@@ -228,6 +235,14 @@ event content_gap(c: connection, is_orig: bool, seq: count, length: count) &prio
 	c$conn$missed_bytes = c$conn$missed_bytes + length;
 	}
 
+event tunnel_changed(c: connection, e: EncapsulatingConnVector) &priority=5
+	{
+	set_conn(c, F);
+	if ( |e| > 0 )
+		add c$conn$tunnel_parents[e[|e|-1]$uid];
+	c$tunnel = e;
+	}
+
 event connection_state_remove(c: connection) &priority=5
 	{
 	set_conn(c, T);

scripts/base/protocols/conn/polling.bro

@@ -0,0 +1,49 @@
+##! Implements a generic way to poll connections looking for certain features
+##! (e.g. monitor bytes transferred).  The specific feature of a connection
+##! to look for, the polling interval, and the code to execute if the feature
+##! is found are all controlled by user-defined callback functions.
+
+module ConnPolling;
+
+export {
+	## Starts monitoring a given connection.
+	##
+	## c: The connection to watch.
+	##
+	## callback: A callback function that takes as arguments the monitored
+	##           *connection*, and counter *cnt* that increments each time the
+	##           callback is called.  It returns an interval indicating how long
+	##           in the future to schedule an event which will call the
+	##           callback.  A negative return interval causes polling to stop.
+	##
+	## cnt: The initial value of a counter which gets passed to *callback*.
+	##
+	## i: The initial interval at which to schedule the next callback.
+	##    May be ``0secs`` to poll right away.
+	global watch: function(c: connection,
+			       callback: function(c: connection, cnt: count): interval,
+			       cnt: count, i: interval);
+}
+
+event ConnPolling::check(c: connection,
+			 callback: function(c: connection, cnt: count): interval,
+			 cnt: count)
+	{
+	if ( ! connection_exists(c$id) )
+		return;
+
+	lookup_connection(c$id); # updates the conn val
+
+	local next_interval = callback(c, cnt);
+	if ( next_interval < 0secs )
+		return;
+
+	watch(c, callback, cnt + 1, next_interval);
+	}
+
+function watch(c: connection,
+	       callback: function(c: connection, cnt: count): interval,
+	       cnt: count, i: interval)
+	{
+	schedule i { ConnPolling::check(c, callback, cnt) };
+	}

scripts/base/protocols/dns/main.bro

@@ -46,27 +46,29 @@ export {
 		AA:            bool               &log &default=F;
 		## The Truncation bit specifies that the message was truncated.
 		TC:            bool               &log &default=F;
-		## The Recursion Desired bit indicates to a name server to recursively
+		## The Recursion Desired bit in a request message indicates that
-		## purse the query.
+		## the client wants recursive service for this query.
 		RD:            bool               &log &default=F;
-		## The Recursion Available bit in a response message indicates if
+		## The Recursion Available bit in a response message indicates that
 		## the name server supports recursive queries.
 		RA:            bool               &log &default=F;
 		## A reserved field that is currently supposed to be zero in all
 		## queries and responses.
 		Z:             count              &log &default=0;
-		## The set of resource descriptions in answer of the query.
+		## The set of resource descriptions in the query answer.
 		answers:       vector of string   &log &optional;
 		## The caching intervals of the associated RRs described by the
 		## ``answers`` field.
 		TTLs:          vector of interval &log &optional;
+		## The DNS query was rejected by the server.
+		rejected:      bool               &log &default=F;
 
 		## This value indicates if this request/response pair is ready to be
 		## logged.
 		ready:         bool            &default=F;
 		## The total number of resource records in a reply message's answer
 		## section.
-		total_answers: count           &optional;
+		total_answers: count           &default=0;
 		## The total number of resource records in a reply message's answer,
 		## authority, and additional sections.
 		total_replies: count           &optional;
@@ -151,11 +153,11 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
 
 	c$dns = c$dns_state$pending[msg$id];
 
+	if ( ! is_query )
+		{
 		c$dns$rcode = msg$rcode;
 		c$dns$rcode_name = base_errors[msg$rcode];
 
-	if ( ! is_query )
-		{
 		if ( ! c$dns?$total_answers )
 			c$dns$total_answers = msg$num_answers;
 
@@ -175,10 +177,13 @@ function set_session(c: connection, msg: dns_msg, is_query: bool)
 		}
 	}
 
+event dns_message(c: connection, is_orig: bool, msg: dns_msg, len: count) &priority=5
+	{
+	set_session(c, msg, is_orig);
+	}
+
 event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string) &priority=5
 	{
-	set_session(c, msg, F);
-
 	if ( ans$answer_type == DNS_ANS )
 		{
 		c$dns$AA    = msg$AA;
@@ -198,7 +203,8 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 			c$dns$TTLs[|c$dns$TTLs|] = ans$TTL;
 			}
 
-		if ( c$dns?$answers && |c$dns$answers| == c$dns$total_answers )
+		if ( c$dns?$answers && c$dns?$total_answers &&
+		     |c$dns$answers| == c$dns$total_answers )
 			{
 			add c$dns_state$finished_answers[c$dns$trans_id];
 			# Indicate this request/reply pair is ready to be logged.
@@ -219,8 +225,6 @@ event DNS::do_reply(c: connection, msg: dns_msg, ans: dns_answer, reply: string)
 
 event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
 	{
-	set_session(c, msg, T);
-
 	c$dns$RD          = msg$RD;
 	c$dns$TC          = msg$TC;
 	c$dns$qclass      = qclass;
@@ -310,11 +314,9 @@ event dns_SRV_reply(c: connection, msg: dns_msg, ans: dns_answer) &priority=5
 #
 #	}
 
-
+event dns_rejected(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count) &priority=5
-event dns_rejected(c: connection, msg: dns_msg,
-                   query: string, qtype: count, qclass: count) &priority=5
 	{
-	set_session(c, msg, F);
+	c$dns$rejected = T;
 	}
 
 event connection_state_remove(c: connection) &priority=-5

scripts/base/protocols/ftp/__load__.bro

@@ -1,3 +1,4 @@
 @load ./utils-commands
 @load ./main
 @load ./file-extract
+@load ./gridftp

scripts/base/protocols/ftp/gridftp.bro

@@ -0,0 +1,121 @@
+##! A detection script for GridFTP data and control channels.
+##!
+##! GridFTP control channels are identified by FTP control channels
+##! that successfully negotiate the GSSAPI method of an AUTH request
+##! and for which the exchange involved an encoded TLS/SSL handshake,
+##! indicating the GSI mechanism for GSSAPI was used.  This analysis
+##! is all supported internally, this script simple adds the "gridftp"
+##! label to the *service* field of the control channel's
+##! :bro:type:`connection` record.
+##!
+##! GridFTP data channels are identified by a heuristic that relies on
+##! the fact that default settings for GridFTP clients typically
+##! mutally authenticate the data channel with TLS/SSL and negotiate a
+##! NULL bulk cipher (no encryption).  Connections with those
+##! attributes are then polled for two minutes with decreasing frequency
+##! to check if the transfer sizes are large enough to indicate a
+##! GridFTP data channel that would be undesireable to analyze further
+##! (e.g. stop TCP reassembly).  A side effect is that true connection
+##! sizes are not logged, but at the benefit of saving CPU cycles that
+##! otherwise go to analyzing the large (and likely benign) connections.
+
+@load ./main
+@load base/protocols/conn
+@load base/protocols/ssl
+@load base/frameworks/notice
+
+module GridFTP;
+
+export {
+	## Number of bytes transferred before guessing a connection is a
+	## GridFTP data channel.
+	const size_threshold = 1073741824 &redef;
+
+	## Max number of times to check whether a connection's size exceeds the
+	## :bro:see:`GridFTP::size_threshold`.
+	const max_poll_count = 15 &redef;
+
+	## Whether to skip further processing of the GridFTP data channel once
+	## detected, which may help performance.
+	const skip_data = T &redef;
+
+	## Base amount of time between checking whether a GridFTP data connection
+	## has transferred more than :bro:see:`GridFTP::size_threshold` bytes.
+	const poll_interval = 1sec &redef;
+
+	## The amount of time the base :bro:see:`GridFTP::poll_interval` is
+	## increased by each poll interval.  Can be used to make more frequent
+	## checks at the start of a connection and gradually slow down.
+	const poll_interval_increase = 1sec &redef;
+
+	## Raised when a GridFTP data channel is detected.
+	##
+	## c: The connection pertaining to the GridFTP data channel.
+	global data_channel_detected: event(c: connection);
+
+	## The initial criteria used to determine whether to start polling
+	## the connection for the :bro:see:`GridFTP::size_threshold` to have
+	## been exceeded.  This is called in a :bro:see:`ssl_established` event
+	## handler and by default looks for both a client and server certificate
+	## and for a NULL bulk cipher.  One way in which this function could be
+	## redefined is to make it also consider client/server certificate issuer
+	## subjects.
+	##
+	## c: The connection which may possibly be a GridFTP data channel.
+	##
+	## Returns: true if the connection should be further polled for an
+	##          exceeded :bro:see:`GridFTP::size_threshold`, else false.
+	const data_channel_initial_criteria: function(c: connection): bool &redef;
+}
+
+redef record FTP::Info += {
+	last_auth_requested: string &optional;
+};
+
+event ftp_request(c: connection, command: string, arg: string) &priority=4
+	{
+	if ( command == "AUTH" && c?$ftp )
+		c$ftp$last_auth_requested = arg;
+	}
+
+function size_callback(c: connection, cnt: count): interval
+	{
+	if ( c$orig$size > size_threshold || c$resp$size > size_threshold )
+		{
+		add c$service["gridftp-data"];
+		event GridFTP::data_channel_detected(c);
+
+		if ( skip_data )
+			skip_further_processing(c$id);
+
+		return -1sec;
+		}
+
+	if ( cnt >= max_poll_count )
+		return -1sec;
+
+	return poll_interval + poll_interval_increase * cnt;
+	}
+
+event ssl_established(c: connection) &priority=5
+	{
+	# If an FTP client requests AUTH GSSAPI and later an SSL handshake
+	# finishes, it's likely a GridFTP control channel, so add service label.
+	if ( c?$ftp && c$ftp?$last_auth_requested &&
+	     /GSSAPI/ in c$ftp$last_auth_requested )
+		add c$service["gridftp"];
+	}
+
+function data_channel_initial_criteria(c: connection): bool
+	{
+	return ( c?$ssl && c$ssl?$client_subject && c$ssl?$subject &&
+	         c$ssl?$cipher && /WITH_NULL/ in c$ssl$cipher );
+	}
+
+event ssl_established(c: connection) &priority=-3
+	{
+	# By default GridFTP data channels do mutual authentication and
+	# negotiate a cipher suite with a NULL bulk cipher.
+	if ( data_channel_initial_criteria(c) )
+		ConnPolling::watch(c, size_callback, 0, 0secs);
+	}

scripts/base/protocols/ftp/main.bro

@@ -7,6 +7,7 @@
 @load ./utils-commands
 @load base/utils/paths
 @load base/utils/numbers
+@load base/utils/addrs
 
 
 module FTP;
@@ -29,7 +30,9 @@ export {
 	type Info: record {
 		## Time when the command was sent.
 		ts:               time        &log;
+		## Unique ID for the connection.
 		uid:              string      &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:               conn_id     &log;
 		## User name for the current FTP session.
 		user:             string      &log &default="<unknown>";
@@ -169,12 +172,7 @@ function ftp_message(s: Info)
 		
 		local arg = s$cmdarg$arg;
 		if ( s$cmdarg$cmd in file_cmds )
-			{
+			arg = fmt("ftp://%s%s", addr_to_uri(s$id$resp_h), build_path_compressed(s$cwd, arg));
-			if ( is_v4_addr(s$id$resp_h) )
-				arg = fmt("ftp://%s%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
-			else
-				arg = fmt("ftp://[%s]%s", s$id$resp_h, build_path_compressed(s$cwd, arg));
-			}
 		
 		s$ts=s$cmdarg$ts;
 		s$command=s$cmdarg$cmd;

scripts/base/protocols/http/file-ident.bro

@@ -6,7 +6,8 @@
 @load ./utils
 
 # Add the magic number signatures to the core signature set.
-redef signature_files += "base/protocols/http/file-ident.sig";
+@load-sigs ./file-ident.sig
+
 # Ignore the signatures used to match files
 redef Signatures::ignored_ids += /^matchfile-/;
 

scripts/base/protocols/http/main.bro

@@ -23,7 +23,9 @@ export {
 	type Info: record {
 		## Timestamp for when the request happened.
 		ts:                      time      &log;
+		## Unique ID for the connection.
 		uid:                     string    &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:                      conn_id   &log;
 		## Represents the pipelined depth into the connection of this 
 		## request/response transaction.

scripts/base/protocols/http/utils.bro

@@ -1,6 +1,7 @@
 ##! Utilities specific for HTTP processing.
 
 @load ./main
+@load base/utils/addrs
 
 module HTTP;
 
@@ -51,7 +52,7 @@ function extract_keys(data: string, kv_splitter: pattern): string_vec
 function build_url(rec: Info): string
 	{
 	local uri  = rec?$uri ? rec$uri : "/<missed_request>";
-	local host = rec?$host ? rec$host : fmt("%s", rec$id$resp_h);
+	local host = rec?$host ? rec$host : addr_to_uri(rec$id$resp_h);
 	if ( rec$id$resp_p != 80/tcp )
 		host = fmt("%s:%s", host, rec$id$resp_p);
 	return fmt("%s%s", host, uri);

scripts/base/protocols/irc/main.bro

@@ -13,7 +13,9 @@ export {
 	type Info: record {
 		## Timestamp when the command was seen.
 		ts:       time        &log;
+		## Unique ID for the connection.
 		uid:      string      &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:       conn_id     &log;
 		## Nick name given for the connection.
 		nick:     string      &log &optional;

scripts/base/protocols/modbus/__load__.bro

@@ -0,0 +1,2 @@
+@load ./consts
+@load ./main

scripts/base/protocols/modbus/consts.bro

@@ -0,0 +1,67 @@
+
+module Modbus;
+
+export {
+	## Standard defined Modbus function codes.
+	const function_codes = {
+		[0x01] = "READ_COILS",
+		[0x02] = "READ_DISCRETE_INPUTS",
+		[0x03] = "READ_HOLDING_REGISTERS",
+		[0x04] = "READ_INPUT_REGISTERS",
+		[0x05] = "WRITE_SINGLE_COIL",
+		[0x06] = "WRITE_SINGLE_REGISTER",
+		[0x07] = "READ_EXCEPTION_STATUS",
+		[0x08] = "DIAGNOSTICS",
+		[0x0B] = "GET_COMM_EVENT_COUNTER",
+		[0x0C] = "GET_COMM_EVENT_LOG",
+		[0x0F] = "WRITE_MULTIPLE_COILS",
+		[0x10] = "WRITE_MULTIPLE_REGISTERS",
+		[0x11] = "REPORT_SLAVE_ID",
+		[0x14] = "READ_FILE_RECORD",
+		[0x15] = "WRITE_FILE_RECORD",
+		[0x16] = "MASK_WRITE_REGISTER",
+		[0x17] = "READ_WRITE_MULTIPLE_REGISTERS",
+		[0x18] = "READ_FIFO_QUEUE",
+		[0x2B] = "ENCAP_INTERFACE_TRANSPORT",
+
+		# Machine/vendor/network specific functions
+		[0x09] = "PROGRAM_484",
+		[0x0A] = "POLL_484",
+		[0x0D] = "PROGRAM_584_984",
+		[0x0E] = "POLL_584_984",
+		[0x12] = "PROGRAM_884_U84",
+		[0x13] = "RESET_COMM_LINK_884_U84",
+		[0x28] = "PROGRAM_CONCEPT",
+		[0x7D] = "FIRMWARE_REPLACEMENT",
+		[0x7E] = "PROGRAM_584_984_2",
+		[0x7F] = "REPORT_LOCAL_ADDRESS",
+
+		# Exceptions
+		[0x81] = "READ_COILS_EXCEPTION",
+		[0x82] = "READ_DISCRETE_INPUTS_EXCEPTION",
+		[0x83] = "READ_HOLDING_REGISTERS_EXCEPTION",
+		[0x84] = "READ_INPUT_REGISTERS_EXCEPTION",
+		[0x85] = "WRITE_SINGLE_COIL_EXCEPTION",
+		[0x86] = "WRITE_SINGLE_REGISTER_EXCEPTION",
+		[0x87] = "READ_EXCEPTION_STATUS_EXCEPTION",
+		[0x8F] = "WRITE_MULTIPLE_COILS_EXCEPTION",
+		[0x90] = "WRITE_MULTIPLE_REGISTERS_EXCEPTION",
+		[0x94] = "READ_FILE_RECORD_EXCEPTION",
+		[0x95] = "WRITE_FILE_RECORD_EXCEPTION",
+		[0x96] = "MASK_WRITE_REGISTER_EXCEPTION",
+		[0x97] = "READ_WRITE_MULTIPLE_REGISTERS_EXCEPTION",
+		[0x98] = "READ_FIFO_QUEUE_EXCEPTION",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
+
+	const exception_codes = {
+		[0x01] = "ILLEGAL_FUNCTION",
+		[0x02] = "ILLEGAL_DATA_ADDRESS",
+		[0x03] = "ILLEGAL_DATA_VALUE",
+		[0x04] = "SLAVE_DEVICE_FAILURE",
+		[0x05] = "ACKNOWLEDGE",
+		[0x06] = "SLAVE_DEVICE_BUSY",
+		[0x08] = "MEMORY_PARITY_ERROR",
+		[0x0A] = "GATEWAY_PATH_UNAVAILABLE",
+		[0x0B] = "GATEWAY_TARGET_DEVICE_FAILED_TO_RESPOND",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); } &redef;
+}

scripts/base/protocols/modbus/main.bro

@@ -0,0 +1,71 @@
+##! Base Modbus analysis script.
+
+module Modbus;
+
+@load ./consts
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Time of the request.
+		ts:        time           &log;
+		## Unique identifier for the connnection.
+		uid:       string         &log;
+		## Identifier for the connection.
+		id:        conn_id        &log;
+		## The name of the function message that was sent.
+		func:      string         &log &optional;
+		## The exception if the response was a failure.
+		exception: string         &log &optional;
+	};
+
+	## Event that can be handled to access the Modbus record as it is sent on
+	## to the logging framework.
+	global log_modbus: event(rec: Info);
+}
+
+redef record connection += {
+	modbus: Info &optional;
+};
+
+# Configure DPD and the packet filter.
+redef capture_filters += { ["modbus"] = "tcp port 502" };
+redef dpd_config += { [ANALYZER_MODBUS] = [$ports = set(502/tcp)] };
+redef likely_server_ports += { 502/tcp };
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(Modbus::LOG, [$columns=Info, $ev=log_modbus]);
+	}
+
+event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=5
+	{
+	if ( ! c?$modbus )
+		{
+		c$modbus = [$ts=network_time(), $uid=c$uid, $id=c$id];
+		}
+
+	c$modbus$ts   = network_time();
+	c$modbus$func = function_codes[headers$function_code];
+	}
+
+event modbus_message(c: connection, headers: ModbusHeaders, is_orig: bool) &priority=-5
+	{
+	# Only log upon replies.
+	# Also, don't log now if this is an exception (log in the exception event handler)
+	if ( ! is_orig && ( headers$function_code <= 0x81 || headers$function_code >= 0x98 ) )
+		Log::write(LOG, c$modbus);
+	}
+
+event modbus_exception(c: connection, headers: ModbusHeaders, code: count) &priority=5
+	{
+	c$modbus$exception = exception_codes[code];
+	}
+
+event modbus_exception(c: connection, headers: ModbusHeaders, code: count) &priority=-5
+	{
+	Log::write(LOG, c$modbus);
+	delete c$modbus$exception;
+	}
+

scripts/base/protocols/smtp/main.bro

@@ -9,33 +9,51 @@ export {
 	redef enum Log::ID += { LOG };
 
 	type Info: record {
+		## Time when the message was first seen.
 		ts:                time            &log;
+		## Unique ID for the connection.
 		uid:               string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:                conn_id         &log;
-		## This is a number that indicates the number of messages deep into
+		## A count to represent the depth of this message transaction in a single 
-		## this connection where this particular message was transferred.
+		## connection where multiple messages were transferred.
 		trans_depth:       count           &log;
+		## Contents of the Helo header.
 		helo:              string          &log &optional;
+		## Contents of the From header.
 		mailfrom:          string          &log &optional;
+		## Contents of the Rcpt header.
 		rcptto:            set[string]     &log &optional;
+		## Contents of the Date header.
 		date:              string          &log &optional;
+		## Contents of the From header.
 		from:              string          &log &optional;
+		## Contents of the To header.
 		to:                set[string]     &log &optional;
+		## Contents of the ReplyTo header.
 		reply_to:          string          &log &optional;
+		## Contents of the MsgID header.
 		msg_id:            string          &log &optional;
+		## Contents of the In-Reply-To header.
 		in_reply_to:       string          &log &optional;
+		## Contents of the Subject header.
 		subject:           string          &log &optional;
+		## Contents of the X-Origininating-IP header.
 		x_originating_ip:  addr            &log &optional;
+		## Contents of the first Received header.
 		first_received:    string          &log &optional;
+		## Contents of the second Received header.
 		second_received:   string          &log &optional;
-		## The last message the server sent to the client.
+		## The last message that the server sent to the client.
 		last_reply:        string          &log &optional;
+		## The message transmission path, as extracted from the headers.
 		path:              vector of addr  &log &optional;
+		## Value of the User-Agent header from the client.
 		user_agent:        string          &log &optional;
 		
-		## Indicate if the "Received: from" headers should still be processed.
+		## Indicates if the "Received: from" headers should still be processed.
 		process_received_from: bool        &default=T;
-		## Indicates if client activity has been seen, but not yet logged
+		## Indicates if client activity has been seen, but not yet logged.
 		has_client_activity:  bool            &default=F;
 	};
 	

scripts/base/protocols/socks/__load__.bro

@@ -0,0 +1,2 @@
+@load ./consts
+@load ./main

scripts/base/protocols/socks/consts.bro

@@ -0,0 +1,40 @@
+module SOCKS;
+
+export {
+	type RequestType: enum {
+		CONNECTION    = 1,
+		PORT          = 2,
+		UDP_ASSOCIATE = 3,
+	};
+
+	const v5_authentication_methods: table[count] of string = {
+		[0] = "No Authentication Required",
+		[1] = "GSSAPI",
+		[2] = "Username/Password",
+		[3] = "Challenge-Handshake Authentication Protocol",
+		[5] = "Challenge-Response Authentication Method",
+		[6] = "Secure Sockets Layer",
+		[7] = "NDS Authentication",
+		[8] = "Multi-Authentication Framework",
+		[255] = "No Acceptable Methods",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const v4_status: table[count] of string = {
+		[0x5a] = "succeeded",
+		[0x5b] = "general SOCKS server failure",
+		[0x5c] = "request failed because client is not running identd",
+		[0x5d] = "request failed because client's identd could not confirm the user ID string in the request",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+
+	const v5_status: table[count] of string = {
+		[0] = "succeeded",
+		[1] = "general SOCKS server failure",
+		[2] = "connection not allowed by ruleset",
+		[3] = "Network unreachable",
+		[4] = "Host unreachable",
+		[5] = "Connection refused",
+		[6] = "TTL expired",
+		[7] = "Command not supported",
+		[8] = "Address type not supported",
+	} &default=function(i: count):string { return fmt("unknown-%d", i); };
+}

scripts/base/protocols/socks/main.bro

@@ -0,0 +1,92 @@
+@load base/frameworks/tunnels
+@load ./consts
+
+module SOCKS;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Time when the proxy connection was first detected.
+		ts:          time            &log;
+		## Unique ID for the tunnel - may correspond to connection uid or be non-existent.
+		uid:         string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
+		id:          conn_id         &log;
+		## Protocol version of SOCKS.
+		version:     count           &log;
+		## Username for the proxy if extracted from the network..
+		user:        string          &log &optional;
+		## Server status for the attempt at using the proxy.
+		status:      string          &log &optional;
+		## Client requested SOCKS address. Could be an address, a name or both.
+		request:     SOCKS::Address  &log &optional;
+		## Client requested port.
+		request_p:   port            &log &optional;
+		## Server bound address. Could be an address, a name or both.
+		bound:       SOCKS::Address  &log &optional;
+		## Server bound port.
+		bound_p:     port            &log &optional;
+	};
+
+	## Event that can be handled to access the SOCKS
+	## record as it is sent on to the logging framework.
+	global log_socks: event(rec: Info);
+}
+
+event bro_init() &priority=5
+	{
+	Log::create_stream(SOCKS::LOG, [$columns=Info, $ev=log_socks]);
+	}
+
+redef record connection += {
+	socks: SOCKS::Info &optional;
+};
+
+# Configure DPD
+redef capture_filters += { ["socks"] = "tcp port 1080" };
+redef dpd_config += { [ANALYZER_SOCKS] = [$ports = set(1080/tcp)] };
+redef likely_server_ports += { 1080/tcp };
+
+function set_session(c: connection, version: count)
+	{
+	if ( ! c?$socks )
+		c$socks = [$ts=network_time(), $id=c$id, $uid=c$uid, $version=version];
+	}
+
+event socks_request(c: connection, version: count, request_type: count,
+                    sa: SOCKS::Address, p: port, user: string) &priority=5
+	{
+	set_session(c, version);
+
+	c$socks$request   = sa;
+	c$socks$request_p = p;
+
+	# Copy this conn_id and set the orig_p to zero because in the case of SOCKS proxies there will
+	# be potentially many source ports since a new proxy connection is established for each
+	# proxied connection.  We treat this as a singular "tunnel".
+	local cid = copy(c$id);
+	cid$orig_p = 0/tcp;
+	Tunnel::register([$cid=cid, $tunnel_type=Tunnel::SOCKS, $payload_proxy=T]);
+	}
+
+event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=5
+	{
+	set_session(c, version);
+
+	if ( version == 5 )
+		c$socks$status = v5_status[reply];
+	else if ( version == 4 )
+		c$socks$status = v4_status[reply];
+
+	c$socks$bound   = sa;
+	c$socks$bound_p = p;
+	}
+
+event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port) &priority=-5
+	{
+	# This will handle the case where the analyzer failed in some way and was removed.  We probably 
+	# don't want to log these connections.
+	if ( "SOCKS" in c$service )
+		Log::write(SOCKS::LOG, c$socks);
+	}

scripts/base/protocols/ssh/main.bro

@@ -27,19 +27,21 @@ export {
 	type Info: record {
 		## Time when the SSH connection began.
 		ts:              time         &log;
+		## Unique ID for the connection.
 		uid:             string       &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:              conn_id      &log;
 		## Indicates if the login was heuristically guessed to be "success"
 		## or "failure".
 		status:          string       &log &optional;
 		## Direction of the connection.  If the client was a local host 
-		## logging into an external host, this would be OUTBOUD.  INBOUND
+		## logging into an external host, this would be OUTBOUND. INBOUND
 		## would be set for the opposite situation.
 		# TODO: handle local-local and remote-remote better.
 		direction:       Direction    &log &optional;
-		## Software string given by the client.
+		## Software string from the client.
 		client:          string       &log &optional;
-		## Software string given by the server.
+		## Software string from the server.
 		server:          string       &log &optional;
 		## Amount of data returned from the server. This is currently
 		## the only measure of the success heuristic and it is logged to 

scripts/base/protocols/ssl/consts.bro

@@ -81,6 +81,8 @@ export {
 		[35] = "SessionTicket TLS",
 		[40] = "extended_random",
 		[13172] = "next_protocol_negotiation",
+		[13175] = "origin_bound_certificates",
+		[13180] = "encrypted_client_certificates",
 		[65281] = "renegotiation_info"
 	} &default=function(i: count):string { return fmt("unknown-%d", i); };
 	

scripts/base/protocols/ssl/main.bro

@@ -10,13 +10,15 @@ export {
 	redef enum Log::ID += { LOG };
 
 	type Info: record {
-		## Time when the SSL connection began.
+		## Time when the SSL connection was first detected.
 		ts:               time             &log;
+		## Unique ID for the connection.
 		uid:         string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:               conn_id          &log;
-		## SSL/TLS version the server offered.
+		## SSL/TLS version that the server offered.
 		version:          string           &log &optional;
-		## SSL/TLS cipher suite the server chose.
+		## SSL/TLS cipher suite that the server chose.
 		cipher:           string           &log &optional;
 		## Value of the Server Name Indicator SSL/TLS extension.  It
 		## indicates the server name that the client was requesting.
@@ -25,19 +27,32 @@ export {
 		session_id:       string           &log &optional;
 		## Subject of the X.509 certificate offered by the server.
 		subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the server.
+		issuer_subject:   string           &log &optional;
 		## NotValidBefore field value from the server certificate.
 		not_valid_before: time             &log &optional;
-		## NotValidAfter field value from the serve certificate.
+		## NotValidAfter field value from the server certificate.
 		not_valid_after:  time             &log &optional;
 		## Last alert that was seen during the connection.
 		last_alert:       string           &log &optional;
 
+		## Subject of the X.509 certificate offered by the client.
+		client_subject:          string           &log &optional;
+		## Subject of the signer of the X.509 certificate offered by the client.
+		client_issuer_subject:   string           &log &optional;
+
 		## Full binary server certificate stored in DER format.
 		cert:             string           &optional;
 		## Chain of certificates offered by the server to validate its
 		## complete signing chain.
 		cert_chain:       vector of string &optional;
 
+		## Full binary client certificate stored in DER format.
+		client_cert:             string           &optional;
+		## Chain of certificates offered by the client to validate its
+		## complete signing chain.
+		client_cert_chain:       vector of string &optional;
+
 		## The analyzer ID used for the analyzer instance attached
 		## to each connection.  It is not used for logging since it's a
 		## meaningless arbitrary number.
@@ -82,7 +97,8 @@ redef Protocols::common_ports += { ["SSL"] = ports };
 function set_session(c: connection)
 	{
 	if ( ! c?$ssl )
-		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector()];
+		c$ssl = [$ts=network_time(), $uid=c$uid, $id=c$id, $cert_chain=vector(),
+		         $client_cert_chain=vector()];
 	}
 
 function finish(c: connection)
@@ -116,8 +132,24 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 
 	# We aren't doing anything with client certificates yet.
 	if ( is_orig )
-		return;
+		{
+		if ( chain_idx == 0 )
+			{
+			# Save the primary cert.
+			c$ssl$client_cert = der_cert;
 
+			# Also save other certificate information about the primary cert.
+			c$ssl$client_subject = cert$subject;
+			c$ssl$client_issuer_subject = cert$issuer;
+			}
+		else
+			{
+			# Otherwise, add it to the cert validation chain.
+			c$ssl$client_cert_chain[|c$ssl$client_cert_chain|] = der_cert;
+			}
+		}
+	else
+		{
 		if ( chain_idx == 0 )
 			{
 			# Save the primary cert.
@@ -125,6 +157,7 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 
 			# Also save other certificate information about the primary cert.
 			c$ssl$subject = cert$subject;
+			c$ssl$issuer_subject = cert$issuer;
 			c$ssl$not_valid_before = cert$not_valid_before;
 			c$ssl$not_valid_after = cert$not_valid_after;
 			}
@@ -134,6 +167,7 @@ event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: coun
 			c$ssl$cert_chain[|c$ssl$cert_chain|] = der_cert;
 			}
 		}
+	}
 
 event ssl_extension(c: connection, is_orig: bool, code: count, val: string) &priority=5
 	{

scripts/base/protocols/syslog/main.bro

@@ -10,9 +10,11 @@ export {
 	redef enum Log::ID += { LOG };
 	
 	type Info: record {
-		## Timestamp of when the syslog message was seen.
+		## Timestamp when the syslog message was seen.
 		ts:        time            &log;
+		## Unique ID for the connection.
 		uid:       string          &log;
+		## The connection's 4-tuple of endpoint addresses/ports.
 		id:        conn_id         &log;
 		## Protocol over which the message was seen.
 		proto:     transport_proto &log;

scripts/base/utils/addrs.bro

@@ -98,3 +98,18 @@ function find_ip_addresses(input: string): string_array
 		}
 	return output;
 	}
+
+## Returns the string representation of an IP address suitable for inclusion
+## in a URI.  For IPv4, this does no special formatting, but for IPv6, the
+## address is included in square brackets.
+##
+## a: the address to make suitable for URI inclusion.
+##
+## Returns: the string representation of *a* suitable for URI inclusion.
+function addr_to_uri(a: addr): string
+	{
+	if ( is_v4_addr(a) )
+		return fmt("%s", a);
+	else
+		return fmt("[%s]", a);
+	}

scripts/base/utils/files.bro

@@ -1,10 +1,11 @@
+@load ./addrs
 
 ## This function can be used to generate a consistent filename for when
 ## contents of a file, stream, or connection are being extracted to disk.
 function generate_extraction_filename(prefix: string, c: connection, suffix: string): string
 	{
-	local conn_info = fmt("%s:%d-%s:%d", 
+	local conn_info = fmt("%s:%d-%s:%d", addr_to_uri(c$id$orig_h), c$id$orig_p,
-	                      c$id$orig_h, c$id$orig_p, c$id$resp_h, c$id$resp_p);
+	                      addr_to_uri(c$id$resp_h), c$id$resp_p);
 	
 	if ( prefix != "" )
 		conn_info = fmt("%s_%s", prefix, conn_info);

scripts/base/utils/patterns.bro

@@ -1,5 +1,7 @@
 ##! Functions for creating and working with patterns.
 
+module GLOBAL;
+
 ## Given a pattern as a string with two tildes (~~) contained in it, it will
 ## return a pattern with string set's elements OR'd together where the
 ## double-tilde was given (this function only works at or before init time).

scripts/base/utils/site.bro

@@ -10,15 +10,19 @@ export {
 	const private_address_space: set[subnet] = {
 		10.0.0.0/8,
 		192.168.0.0/16,
+		172.16.0.0/12,
+		100.64.0.0/10,  # RFC6598 Carrier Grade NAT
 		127.0.0.0/8,
-		172.16.0.0/12
+		[fe80::]/10,
+		[::1]/128,
 	} &redef;
 
 	## Networks that are considered "local".
 	const local_nets: set[subnet] &redef;
 
-	## This is used for retrieving the subnet when you multiple 
+	## This is used for retrieving the subnet when using multiple entries in
-	## :bro:id:`Site::local_nets`.  A membership query can be done with an 
+	## :bro:id:`Site::local_nets`.  It's populated automatically from there.
+	## A membership query can be done with an
 	## :bro:type:`addr` and the table will yield the subnet it was found
 	## within.
 	global local_nets_table: table[subnet] of subnet = {};
@@ -40,27 +44,33 @@ export {
 
 	## Function that returns true if an address corresponds to one of
 	## the local networks, false if not.
+	## The function inspects :bro:id:`Site::local_nets`.
 	global is_local_addr: function(a: addr): bool;
 
 	## Function that returns true if an address corresponds to one of
 	## the neighbor networks, false if not.
+	## The function inspects :bro:id:`Site::neighbor_nets`.
 	global is_neighbor_addr: function(a: addr): bool;
 
 	## Function that returns true if an address corresponds to one of
 	## the private/unrouted networks, false if not.
+	## The function inspects :bro:id:`Site::private_address_space`.
 	global is_private_addr: function(a: addr): bool;
 
 	## Function that returns true if a host name is within a local
 	## DNS zone.
+	## The function inspects :bro:id:`Site::local_zones`.
 	global is_local_name: function(name: string): bool;
 
 	## Function that returns true if a host name is within a neighbor
 	## DNS zone.
+	## The function inspects :bro:id:`Site::neighbor_zones`.
 	global is_neighbor_name: function(name: string): bool;
 
 	## Function that returns a common separated list of email addresses
 	## that are considered administrators for the IP address provided as
 	## an argument.
+	## The function inspects :bro:id:`Site::local_admins`.
 	global get_emails: function(a: addr): string;
 }
 

scripts/base/utils/urls.bro

@@ -0,0 +1,25 @@
+## Functions for URL handling.
+
+## A regular expression for matching and extracting URLs.
+const url_regex = /^([a-zA-Z\-]{3,5})(:\/\/[^\/?#"'\r\n><]*)([^?#"'\r\n><]*)([^[:blank:]\r\n"'><]*|\??[^"'\r\n><]*)/ &redef;
+
+## Extracts URLs discovered in arbitrary text.
+function find_all_urls(s: string): string_set
+	{
+	return find_all(s, url_regex);
+	}
+
+## Extracts URLs discovered in arbitrary text without
+## the URL scheme included.
+function find_all_urls_without_scheme(s: string): string_set
+	{
+	local urls = find_all_urls(s);
+	local return_urls: set[string] = set();
+	for ( url in urls )
+		{
+		local no_scheme = sub(url, /^([a-zA-Z\-]{3,5})(:\/\/)/, "");
+		add return_urls[no_scheme];
+		}
+
+	return return_urls;
+	}

scripts/policy/frameworks/communication/listen.bro

@@ -8,5 +8,6 @@ module Communication;
 event bro_init() &priority=-10
 	{
 	enable_communication();
-	listen(listen_interface, listen_port, listen_ssl);
+	listen(listen_interface, listen_port, listen_ssl, listen_ipv6,
+	       listen_ipv6_zone_id, listen_retry);
 	}

scripts/policy/frameworks/control/controller.bro

@@ -25,8 +25,8 @@ event bro_init() &priority=5
 	
 	# Establish the communication configuration and only request response
 	# messages.
-	Communication::nodes["control"] = [$host=host, $p=host_port,
+	Communication::nodes["control"] = [$host=host, $zone_id=zone_id,
-	                                   $sync=F, $connect=T,
+	                                   $p=host_port, $sync=F, $connect=T,
 	                                   $class="control", $events=Control::controllee_events];
 	}
 

scripts/policy/frameworks/intel/__load__.bro

@@ -0,0 +1,8 @@
+@load ./conn-established
+@load ./dns
+@load ./http-host-header
+@load ./http-url
+@load ./http-user-agents
+@load ./ssl
+@load ./smtp
+@load ./smtp-url-extraction

scripts/policy/frameworks/intel/conn-established.bro

@@ -0,0 +1,8 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event connection_established(c: connection)
+	{
+	Intel::seen([$host=c$id$orig_h, $conn=c, $where=Conn::IN_ORIG]);
+	Intel::seen([$host=c$id$resp_h, $conn=c, $where=Conn::IN_RESP]);
+	}

scripts/policy/frameworks/intel/dns.bro

@@ -0,0 +1,10 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event dns_request(c: connection, msg: dns_msg, query: string, qtype: count, qclass: count)
+	{
+	Intel::seen([$str=query,
+	             $str_type=Intel::DOMAIN,
+	             $conn=c,
+	             $where=DNS::IN_REQUEST]);
+	}

scripts/policy/frameworks/intel/http-host-header.bro

@@ -0,0 +1,11 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig && name == "HOST" )
+		Intel::seen([$str=value,
+		             $str_type=Intel::DOMAIN,
+		             $conn=c,
+		             $where=HTTP::IN_HOST_HEADER]);
+	}

scripts/policy/frameworks/intel/http-url.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load base/protocols/http/utils
+@load ./where-locations
+
+event http_message_done(c: connection, is_orig: bool, stat: http_message_stat)
+	{
+	if ( is_orig && c?$http )
+		Intel::seen([$str=HTTP::build_url(c$http),
+		             $str_type=Intel::URL,
+		             $conn=c,
+		             $where=HTTP::IN_URL]);
+	}

scripts/policy/frameworks/intel/http-user-agents.bro

@@ -0,0 +1,12 @@
+@load base/frameworks/intel
+@load ./where-locations
+
+event http_header(c: connection, is_orig: bool, name: string, value: string)
+	{
+	if ( is_orig && name == "USER-AGENT" )
+		Intel::seen([$str=value,
+		             $str_type=Intel::USER_AGENT,
+		             $conn=c,
+		             $where=HTTP::IN_USER_AGENT_HEADER]);
+	}
+

scripts/policy/frameworks/intel/smtp-url-extraction.bro

@@ -0,0 +1,15 @@
+@load base/frameworks/intel
+@load base/utils/urls
+@load ./where-locations
+
+event mime_segment_data(c: connection, length: count, data: string) &priority=3
+	{
+	local urls = find_all_urls_without_scheme(data);
+	for ( url in urls )
+		{
+		Intel::seen([$str=url,
+		             $str_type=Intel::URL,
+		             $conn=c,
+		             $where=SMTP::IN_MESSAGE]);
+		}
+	}

scripts/policy/frameworks/intel/smtp.bro

@@ -0,0 +1,71 @@
+@load base/frameworks/intel
+@load base/protocols/smtp
+@load ./where-locations
+
+event mime_end_entity(c: connection)
+	{
+	if ( c?$smtp )
+		{
+		if ( c$smtp?$path )
+			{
+			local path = c$smtp$path;
+			for ( i in path )
+				{
+				Intel::seen([$host=path[i],
+				             $conn=c,
+				             $where=SMTP::IN_RECEIVED_HEADER]);
+				}
+			}
+
+		if ( c$smtp?$user_agent )
+			Intel::seen([$str=c$smtp$user_agent,
+			             $str_type=Intel::USER_AGENT,
+			             $conn=c,
+			             $where=SMTP::IN_HEADER]);
+
+		if ( c$smtp?$x_originating_ip )
+			Intel::seen([$host=c$smtp$x_originating_ip,
+			             $conn=c,
+			             $where=SMTP::IN_X_ORIGINATING_IP_HEADER]);
+
+		if ( c$smtp?$mailfrom )
+			Intel::seen([$str=c$smtp$mailfrom,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=SMTP::IN_MAIL_FROM]);
+
+		if ( c$smtp?$rcptto )
+			{
+			for ( rcptto in c$smtp$rcptto )
+				{
+				Intel::seen([$str=rcptto,
+				             $str_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_RCPT_TO]);
+				}
+			}
+
+		if ( c$smtp?$from )
+			Intel::seen([$str=c$smtp$from,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=SMTP::IN_FROM]);
+
+		if ( c$smtp?$to )
+			{
+			for ( email_to in c$smtp$to )
+				{
+				Intel::seen([$str=email_to,
+				             $str_type=Intel::EMAIL,
+				             $conn=c,
+				             $where=SMTP::IN_TO]);
+				}
+			}
+
+		if ( c$smtp?$reply_to )
+			Intel::seen([$str=c$smtp$reply_to,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=SMTP::IN_REPLY_TO]);
+		}
+	}

scripts/policy/frameworks/intel/ssl.bro

@@ -0,0 +1,34 @@
+@load base/frameworks/intel
+@load base/protocols/ssl
+@load ./where-locations
+
+event x509_certificate(c: connection, is_orig: bool, cert: X509, chain_idx: count, chain_len: count, der_cert: string)
+	{
+	if ( chain_idx == 0 )
+		{
+		if ( /emailAddress=/ in cert$subject )
+			{
+			local email = sub(cert$subject, /^.*emailAddress=/, "");
+			email = sub(email, /,.*$/, "");
+			Intel::seen([$str=email,
+			             $str_type=Intel::EMAIL,
+			             $conn=c,
+			             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
+			}
+
+		Intel::seen([$str=sha1_hash(der_cert),
+		             $str_type=Intel::CERT_HASH,
+		             $conn=c,
+		             $where=(is_orig ? SSL::IN_CLIENT_CERT : SSL::IN_SERVER_CERT)]);
+		}
+	}
+
+event ssl_extension(c: connection, is_orig: bool, code: count, val: string)
+	{
+	if ( is_orig && SSL::extensions[code] == "server_name" && 
+	     c?$ssl && c$ssl?$server_name )
+		Intel::seen([$str=c$ssl$server_name,
+		             $str_type=Intel::DOMAIN,
+		             $conn=c,
+		             $where=SSL::IN_SERVER_NAME]);
+	}

scripts/policy/frameworks/intel/where-locations.bro

@@ -0,0 +1,25 @@
+@load base/frameworks/intel
+
+export {
+	redef enum Intel::Where += {
+		Conn::IN_ORIG,
+		Conn::IN_RESP,
+		DNS::IN_REQUEST,
+		DNS::IN_RESPONSE,
+		HTTP::IN_HOST_HEADER,
+		HTTP::IN_USER_AGENT_HEADER,
+		HTTP::IN_URL,
+		SMTP::IN_MAIL_FROM,
+		SMTP::IN_RCPT_TO,
+		SMTP::IN_FROM,
+		SMTP::IN_TO,
+		SMTP::IN_RECEIVED_HEADER,
+		SMTP::IN_REPLY_TO,
+		SMTP::IN_X_ORIGINATING_IP_HEADER,
+		SMTP::IN_MESSAGE,
+		SSL::IN_SERVER_CERT,
+		SSL::IN_CLIENT_CERT,
+		SSL::IN_SERVER_NAME,
+		SMTP::IN_HEADER,
+	};
+}

scripts/policy/integration/collective-intel/README

@@ -0,0 +1,6 @@
+Collective Intelligence Framework Integration
+=============================================
+
+The scripts in this module are for deeper integration with the Collective Intelligence
+Framework (CIF) since Bro's Intel framework doesn't natively behave the same as CIF nor
+does it store and maintain the same data in all cases.