Merge branch 'topic/robin/dynamic-plugins-2.3' into topic/robin/pktsrc

CHANGES

@@ -1,4 +1,104 @@
 
+2.2-117 | 2014-01-23 14:18:19 -0800
+
+  * Fixing initialization context in anonymous functions. (Robin
+    Sommer)
+
+2.2-115 | 2014-01-22 12:11:18 -0800
+
+  * Add unit tests for new Bro Manual docs. (Jon Siwek)
+    
+  * New content for the "Using Bro" section of the manual. (Rafael
+    Bonilla/Jon Siwek)
+    
+2.2-105 | 2014-01-20 12:16:48 -0800
+
+  * Support GRE tunnel decapsulation, including enhanced GRE headers.
+    GRE tunnels are treated just like IP-in-IP tunnels by parsing past
+    the GRE header in between the delivery and payload IP packets.
+    Addresses BIT-867. (Jon Siwek)
+
+  * Simplify FragReassembler memory management. (Jon Siwek)
+    
+2.2-102 | 2014-01-20 12:00:29 -0800
+
+  * Include file information (MIME type and description) into notice
+    emails if available. (Justin Azoff)
+
+2.2-100 | 2014-01-20 11:54:58 -0800
+
+  * Fix caching of recently validated SSL certifcates. (Justin Azoff)
+    
+2.2-98 | 2014-01-20 11:50:32 -0800
+
+  * For notice suppresion, instead of storing the entire notice in
+    Notice::suppressing, just store the time the notice should be
+    suppressed until. This saves significant memory but can no longer
+    raise end_suppression, which has been removed. (Justin Azoff)
+
+2.2-96 | 2014-01-20 11:41:07 -0800
+
+  * Integrate libmagic 5.16. Bro now now always relies on
+    builtin/shipped magic library/database. (Jon Siwek)
+
+  * Bro now requires a CMake 2.8.x, but no longer a pre-installed
+    libmagic. (Jon Siwek)
+
+2.2-93 | 2014-01-13 09:16:51 -0800
+
+  * Fixing compile problems with some versions of libc++. Reported by
+    Craig Leres. (Robin Sommer)
+
+2.2-91 | 2014-01-13 01:33:28 -0800
+
+  * Improve GeoIP City database support. When trying to open a city
+    database, it now considers both the "REV0" and "REV1" versions of
+    the city database instead of just the former. (Jon Siwek)
+
+  * Broxygen init fixes. Addresses BIT-1110. (Jon Siwek)
+    
+    - Don't check mtime of bro binary if BRO_DISABLE_BROXYGEN env var set.
+
+    - Fix failure to locate bro binary if invoking from a relative
+      path and '.' isn't in PATH.
+
+  * Fix for packet writing to make it use the global snap length.
+    (Seth Hall)
+
+  * Fix for traffic with TCP segmentation offloading with IP header
+    len field being set to zero. (Seth Hall)
+
+  * Canonify output of a unit test. (Jon Siwek)
+    
+  * A set of documentation updates. (Daniel Thayer)
+
+      - Fix typo in Bro 2.2 NEWS on string indexing.
+      - Fix typo in the Quick Start Guide, and clarified the
+        instructions about modifying crontab.
+      - Add/fix documentation for missing/misnamed event parameters.
+      - Fix typos in BIF documentation of hexstr_to_bytestring.
+      - Update the documentation of types and attributes.
+      - Documented the new substring extraction functionality.
+      - Clarified the description of "&priority" and "void".
+
+2.2-75 | 2013-12-18 08:36:50 -0800
+
+  * Fixing segfault with mismatching set &default in record fields.
+    (Robin Sommer)
+
+2.2-74 | 2013-12-16 08:49:55 -0800
+
+  * Improve warnings emitted from raw/execute input reader. (Jon
+    Siwek)
+
+  * Further improve core.when-interpreter-exceptions unit test. (Jon
+    Siwek)
+
+2.2-72 | 2013-12-12 07:12:47 -0800
+
+  * Improve the core.when-interpreter-exceptions unit test to prevent
+    it from occasionally timing out. (Jon Siwek)
+
 2.2-70 | 2013-12-10 15:02:50 -0800
 
   * Fix (harmless) uninitialized field in basename/dirname util

CMakeLists.txt

@@ -1,5 +1,10 @@
 project(Bro C CXX)
-cmake_minimum_required(VERSION 2.6.3 FATAL_ERROR)
+
+# When changing the minimum version here, also adapt
+# cmake/BroPluginDynamic and
+# aux/bro-aux/plugin-support/skeleton/CMakeLists.txt
+cmake_minimum_required(VERSION 2.8.0 FATAL_ERROR)
+
 include(cmake/CommonCMakeConfig.cmake)
 
 ########################################################################
@@ -16,17 +21,21 @@ endif ()
 get_filename_component(BRO_SCRIPT_INSTALL_PATH ${BRO_SCRIPT_INSTALL_PATH}
     ABSOLUTE)
 
+set(BRO_PLUGIN_INSTALL_PATH ${BRO_ROOT_DIR}/lib/bro/plugins CACHE STRING "Installation path for plugins" FORCE)
 set(BRO_MAGIC_INSTALL_PATH ${BRO_ROOT_DIR}/share/bro/magic)
 set(BRO_MAGIC_SOURCE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/magic/database)
 
+
 configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
      "export BROMAGIC=\"${BRO_MAGIC_SOURCE_PATH}\"\n"
+     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
      "setenv BROMAGIC \"${BRO_MAGIC_SOURCE_PATH}\"\n"
+     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -39,6 +48,32 @@ set(VERSION_MAJ_MIN "${VERSION_MAJOR}.${VERSION_MINOR}")
 ########################################################################
 ## Dependency Configuration
 
+include(ExternalProject)
+
+# LOG_* options to ExternalProject_Add appear in CMake 2.8.3.  If
+# available, using them hides external project configure/build output.
+if("${CMAKE_VERSION}" VERSION_GREATER 2.8.2)
+    set(EXTERNAL_PROJECT_LOG_OPTIONS
+        LOG_DOWNLOAD 1 LOG_UPDATE 1 LOG_CONFIGURE 1 LOG_BUILD 1 LOG_INSTALL 1)
+else()
+    set(EXTERNAL_PROJECT_LOG_OPTIONS)
+endif()
+
+set(LIBMAGIC_PREFIX ${CMAKE_CURRENT_BINARY_DIR}/libmagic-prefix)
+set(LIBMAGIC_INCLUDE_DIR ${LIBMAGIC_PREFIX}/include)
+set(LIBMAGIC_LIB_DIR ${LIBMAGIC_PREFIX}/lib)
+set(LIBMAGIC_LIBRARY ${LIBMAGIC_LIB_DIR}/libmagic.a)
+ExternalProject_Add(libmagic
+    PREFIX ${LIBMAGIC_PREFIX}
+    URL ${CMAKE_CURRENT_SOURCE_DIR}/src/3rdparty/file-5.16.tar.gz
+    CONFIGURE_COMMAND ./configure --enable-static --disable-shared
+                                  --prefix=${LIBMAGIC_PREFIX}
+                                  --includedir=${LIBMAGIC_INCLUDE_DIR}
+                                  --libdir=${LIBMAGIC_LIB_DIR}
+    BUILD_IN_SOURCE 1
+    ${EXTERNAL_PROJECT_LOG_OPTIONS}
+)
+
 include(FindRequiredPackage)
 
 # Check cache value first to avoid displaying "Found sed" messages everytime
@@ -57,7 +92,6 @@ FindRequiredPackage(BISON)
 FindRequiredPackage(PCAP)
 FindRequiredPackage(OpenSSL)
 FindRequiredPackage(BIND)
-FindRequiredPackage(LibMagic)
 FindRequiredPackage(ZLIB)
 
 if (NOT BinPAC_ROOT_DIR AND
@@ -73,18 +107,12 @@ if (MISSING_PREREQS)
     message(FATAL_ERROR "Configuration aborted due to missing prerequisites")
 endif ()
 
-set(libmagic_req 5.04)
-if ( LibMagic_VERSION VERSION_LESS ${libmagic_req} )
-    message(FATAL_ERROR "libmagic of at least version ${libmagic_req} required "
-                        "(found ${LibMagic_VERSION})")
-endif ()
-
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${OpenSSL_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
                     ${BinPAC_INCLUDE_DIR}
-                    ${LibMagic_INCLUDE_DIR}
+                    ${LIBMAGIC_INCLUDE_DIR}
                     ${ZLIB_INCLUDE_DIR}
 )
 
@@ -163,7 +191,7 @@ set(brodeps
     ${PCAP_LIBRARY}
     ${OpenSSL_LIBRARIES}
     ${BIND_LIBRARY}
-    ${LibMagic_LIBRARY}
+    ${LIBMAGIC_LIBRARY}
     ${ZLIB_LIBRARY}
     ${OPTLIBS}
 )
@@ -182,6 +210,10 @@ include(MiscTests)
 include(PCAPTests)
 include(OpenSSLTests)
 include(CheckNameserCompat)
+include(GetArchitecture)
+
+# Tell the plugin code that we're building as part of the main tree.
+set(BRO_PLUGIN_INTERNAL_BUILD true CACHE INTERNAL "" FORCE)
 
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/config.h.in
                ${CMAKE_CURRENT_BINARY_DIR}/config.h)

NEWS

@@ -9,9 +9,21 @@ Bro 2.3
 
 [In progress]
 
+Dependencies
+------------
+
+- Bro no longer requires a pre-installed libmagic (because it now
+  ships its own).
+
+- Compiling from source now needs a CMake version >= 2.8.0.
+
 New Functionality
 -----------------
 
+- Support for GRE tunnel decapsulation, including enhanced GRE
+  headers. GRE tunnels are treated just like IP-in-IP tunnels by
+  parsing past the GRE header in between the delivery and payload IP
+  packets.
 
 Changed Functionality
 ---------------------
@@ -22,6 +34,8 @@ Changed Functionality
 - ssl_client_hello() now receives a vector of ciphers, instead of a
   set, to preserve their order.
 
+- Notice::end_suppression() has been removed.
+
 Bro 2.2
 =======
 
@@ -198,9 +212,9 @@ New Functionality
         global s = MySet([$c=1], [$c=2]);
 
 - Strings now support the subscript operator to extract individual
-  characters and substrings (e.g., ``s[4]``, ``s[1,5]``). The index
+  characters and substrings (e.g., ``s[4]``, ``s[1:5]``). The index
   expression can take up to two indices for the start and end index of
-  the substring to return (e.g. ``mystring[1,3]``).
+  the substring to return (e.g. ``mystring[1:3]``).
 
 - Functions now support default parameters, e.g.::
 

VERSION

@@ -1 +1 @@
-2.2-70
+2.2-117

aux/binpac

@@ -1 +1 @@
-Subproject commit 54b321009b750268526419bdbd841f421c839313
+Subproject commit 896ddedde55c48ec2163577fc258b49c418abb3e

aux/bro-aux

@@ -1 +1 @@
-Subproject commit ebf9c0d88ae8230845b91f15755156f93ff21aa8
+Subproject commit 77234b4ba1c5ad0eda64554a7f16a0d79df9ca52

cmake

@@ -1 +1 @@
-Subproject commit e7a46cb82ee10aa522c4d88115baf10181277d20
+Subproject commit 67da63d43e734111b324d8ed045e188e0a28ebf2

config.h.in

@@ -209,3 +209,14 @@
 
 /* Common IPv6 extension structure */
 #cmakedefine HAVE_IP6_EXT
+
+/* String with host architecture (e.g., "linux-x86_64") */
+#define HOST_ARCHITECTURE "@HOST_ARCHITECTURE@"
+
+/* String with extension of dynamic libraries (e.g., ".so") */
+#define DYNAMIC_PLUGIN_SUFFIX "@CMAKE_SHARED_MODULE_SUFFIX@"
+
+/* True if we're building outside of the main Bro source code tree. */
+#ifndef BRO_PLUGIN_INTERNAL_BUILD
+#define BRO_PLUGIN_INTERNAL_BUILD @BRO_PLUGIN_INTERNAL_BUILD@
+#endif

doc/_static/basic.css

@@ -439,8 +439,17 @@ td.linenos pre {
     color: #aaa;
 }
 
+.highlight-guess {
+    overflow:auto;
+}
+
+.highlight-none {
+    overflow:auto;
+}
+
 table.highlighttable {
     margin-left: 0.5em;
+    overflow:scroll;
 }
 
 table.highlighttable td {

doc/broids/index.rst

@@ -0,0 +1,80 @@
+
+.. _bro-ids:
+
+=======
+Bro IDS
+=======
+
+An Intrusion Detection System (IDS) allows you to detect suspicious
+activities happening on your network as a result of a past or active
+attack. Because of its programming capabilities, Bro can easily be
+configured to behave like traditional IDSs and detect common attacks
+with well known patterns, or you can create your own scripts to detect
+conditions specific to your particular case.
+
+In the following sections, we present a few examples of common uses of
+Bro as an IDS.
+
+------------------------------------------------
+Detecting an FTP Bruteforce attack and notifying
+------------------------------------------------
+
+For the purpose of this exercise, we define FTP bruteforcing as too many
+rejected usernames and passwords occurring from a single address.  We
+start by defining a threshold for the number of attempts and a
+monitoring interval in minutes as well as a new notice type.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 9-25
+
+Now, using the ftp_reply event, we check for error codes from the `500
+series <http://en.wikipedia.org/wiki/List_of_FTP_server_return_codes>`_
+for the "USER" and "PASS" commands, representing rejected usernames or
+passwords. For this, we can use the :bro:see:`FTP::parse_ftp_reply_code`
+function to break down the reply code and check if the first digit is a
+"5" or not. If true, we then use the :ref:`Summary Statistics Framework
+<sumstats-framework>` to keep track of the number of failed attempts.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 52-60
+
+Next, we use the SumStats framework to raise a notice of the attack of
+the attack when the number of failed attempts exceeds the specified
+threshold during the measuring interval.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+    :lines: 28-50
+
+Below is the final code for our script.
+
+.. btest-include:: ${BRO_SRC_ROOT}/scripts/policy/protocols/ftp/detect-bruteforcing.bro
+
+.. btest:: ftp-bruteforce
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/ftp/bruteforce.pcap protocols/ftp/detect-bruteforcing.bro
+    @TEST-EXEC: btest-rst-include notice.log
+
+As a final note, the :doc:`detect-bruteforcing.bro
+</scripts/policy/protocols/ftp/detect-bruteforcing.bro>` script above is
+include with Bro out of the box, so you only need to load it at startup
+to instruct Bro to detect and notify of FTP bruteforce attacks.
+
+-------------
+Other Attacks
+-------------
+
+Detecting SQL Injection attacks
+-------------------------------
+
+Checking files against known malware hashes
+-------------------------------------------
+
+Files transmitted on your network could either be completely harmless or
+contain viruses and other threats. One possible action against this
+threat is to compute the hashes of the files and compare them against a
+list of known malware hashes. Bro simplifies this task by offering a
+:doc:`detect-MHR.bro </scripts/policy/frameworks/files/detect-MHR.bro>`
+script that creates and compares hashes against the `Malware Hash
+Registry <https://www.team-cymru.org/Services/MHR/>`_ maintained by Team
+Cymru. You only need to load this script along with your other scripts
+at startup time.

doc/devel/plugins.rst

@@ -0,0 +1,339 @@
+
+===================
+Writing Bro Plugins
+===================
+
+Bro is internally moving to a plugin structure that enables extending
+the system dynamically, without modifying the core code base. That way
+custom code remains self-contained and can be maintained, compiled,
+and installed independently. Currently, plugins can add the following
+functionality to Bro:
+
+    - Bro scripts.
+
+    - Builtin functions/events/types for the scripting language.
+
+    - Protocol and file analyzers.
+
+    - Packet sources and packet dumpers. TODO: Not yet.
+
+    - Logging framework backends. TODO: Not yet.
+
+    - Input framework readers. TODO: Not yet.
+
+A plugin's functionality is available to the user just as if Bro had
+the corresponding code built-in. Indeed, internally many of Bro's
+pieces are structured as plugins as well, they are just statically
+compiled into the binary rather than loaded dynamically at runtime, as
+external plugins are.
+
+Quick Start
+===========
+
+Writing a basic plugin is quite straight-forward as long as one
+follows a few conventions. In the following we walk through adding a
+new built-in function (bif) to Bro; we'll add `a `rot13(s: string) :
+string``, a function that rotates every character in a string by 13
+places.
+
+A plugin comes in the form of a directory following a certain
+structure. To get started, Bro's distribution provides a helper script
+``aux/bro-aux/plugin-support/init-plugin`` that creates a skeleton
+plugin that can then be customized. Let's use that::
+
+    # mkdir rot13-plugin
+    # cd rot13-plugin
+    # init-plugin Demo Rot13
+
+As you can see the script takes two arguments. The first is a
+namespace the plugin will live in, and the second a descriptive name
+for the plugin itself. Bro uses the combination of the two to identify
+a plugin. The namespace serves to avoid naming conflicts between
+plugins written by independent developers; pick, e.g., the name of
+your organisation (and note that the namespace ``Bro`` is reserved for
+functionality distributed by the Bro Project). In our example, the
+plugin will be called ``Demo::Rot13``.
+
+The ``init-plugin`` script puts a number of files in place. The full
+layout is described later. For now, all we need is
+``src/functions.bif``. It's initially empty, but we'll add our new bif
+there as follows::
+
+    # cat scripts/functions.bif
+    module CaesarCipher;
+
+    function camel_case%(s: string%) : string
+        %{
+        char* rot13 = copy_string(s->CheckString());
+
+        for ( char* p = rot13; *p; p++ )
+            {
+            char b = islower(*p) ? 'a' : 'A';
+            *p  = (*p - b + 13) % 26 + b;
+            }
+
+        return new StringVal(strlen(rot13), rot13);
+        %}
+
+The syntax of this file is just like any other ``*.bif`` file; we
+won't go into it here.
+
+Now we can already compile our plugin, we just need to tell the
+Makefile put in place by ``init-plugin`` where the Bro source tree is
+located (Bro needs to have been built there first)::
+
+    # make BRO=/path/to/bro/dist
+    [... cmake output ...]
+
+Now our ``rot13-plugin`` directory has everything that it needs
+for Bro to recognize it as a dynamic plugin. Once we point Bro to it,
+it will pull it in automatically, as we can check with the ``-N``
+option:
+
+    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin
+    # bro -N
+    [...]
+    Plugin: Demo::Rot13 - <Insert brief description of plugin> (dynamic, version 1)
+    [...]
+
+That looks quite good, except for the dummy description that we should
+replace with something nicer so that users will know what our plugin
+is about.  We do this by editing the ``config.description`` line in
+``src/Plugin.cc``, like this:
+
+    [...]
+    plugin::Configuration Configure()
+        {
+        plugin::Configuration config;
+        config.name = "Demo::Rot13";
+        config.description = "Caesar cipher rotating a string's characters by 13 places.";
+        config.version.major = 1;
+        config.version.minor = 0;
+        return config;
+        }
+    [...]
+
+    # make
+    [...]
+    # bro -N | grep Rot13
+    Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
+
+Better. Bro can also show us what exactly the plugin provides with the
+more verbose option ``-NN``::
+
+    # bro -NN
+    [...]
+    Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
+        [Function] CaesarCipher::rot13
+    [...]
+
+There's our function. Now let's use it::
+
+    # bro -e 'print CaesarCipher::rot13("Hello")'
+    Uryyb
+
+It works. We next install the plugin along with Bro itself, so that it
+will find it directly without needing the ``BRO_PLUGIN_PATH``
+environment variable. If we first unset the variable, the function
+will no longer be available::
+
+    # unset BRO_PLUGIN_PATH
+    # bro -e 'print CaesarCipher::rot13("Hello")'
+    error in <command line>, line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13"
+
+Once we install it, it works again::
+
+    # make install
+    # bro -e 'print CaesarCipher::rot13("Hello")'
+    Uryyb
+
+The installed version went into
+``<bro-install-prefix>/lib/bro/plugins/Demo_Rot13``.
+
+We can distribute the plugin in either source or binary form by using
+the Makefile's ``sdist`` and ``bdist`` target, respectively. Both
+create corrsponding tarballs::
+
+    # make sdist
+    [...]
+    Source distribution in build/sdist/Demo_Rot13.tar.gz
+
+    # make bdist
+    [...]
+    Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz
+
+The source archive will contain everything in the plugin directory
+except any generated files. The binary archive will contain anything
+needed to install and run the plugin, i.e., just what ``make install``
+puts into place as well. As the binary distribution is
+platform-dependent, its name includes the OS and architecture the
+plugin was built on.
+
+Plugin Directory Layout
+=======================
+
+A plugin's directory needs to follow a set of conventions so that Bro
+(1) recognizes it as a plugin, and (2) knows what to load.  While
+``init-plugin`` takes care of most of this, the following is the full
+story. We'll use ``<base>`` to represent a plugin's top-level
+directory.
+
+``<base>/__bro_plugin__``
+    A file that marks a directory as containing a Bro plugin. The file
+    must exist, and its content must consist of a single line with the
+    qualified name of the plugin (e.g., "Demo::Rot13").  
+
+``<base>/lib/<plugin-name>-<os>-<arch>.so``
+    The shared library containing the plugin's compiled code. Bro will
+    load this in dynamically at run-time if OS and architecture match
+    the current platform.
+
+``lib/bif/``
+    Directory with auto-generated Bro scripts that declare the plugins
+    bif elements. The files here are produced by ``bifcl``.
+
+``scripts/``
+    A directory with the plugin's custom Bro scripts. When the plugin
+    gets activated, this directory will be automatically added to
+    ``BROPATH``, so that any scripts/modules inside can be
+    ``@load``ed. 
+
+``scripts``/__load__.bro
+    A Bro script that will be loaded immediately when the plugin gets
+    activated. See below for more information on activating plugins.
+
+By convention, a plugin should put its custom scripts into sub folders
+of ``scripts/``, i.e., ``scripts/<script-namespace>/<script>.bro`` to
+avoid conflicts. As usual, you can then put a ``__load__.bro`` in
+there as well so that, e.g., ``@load Demo/Rot13`` could load a whole
+module in the form of multiple individual scripts.
+
+Note that in addition to the paths above, the ``init-plugin`` helper
+puts some more files and directories in place that help with
+development and installation (e.g., ``CMakeLists.txt``, ``Makefile``,
+and source code in ``src/``). However, all these do not have a special
+meaning for Bro at runtime and aren't necessary for a plugin to
+function.
+
+``init-plugin``
+===============
+
+``init-plugin`` puts a basic plugin structure in place that follows
+the above layout and augments it with a CMake build and installation
+system. Note that plugins with this structure can be used both
+directly out of their source directory (after ``make`` and setting
+Bro's ``BRO_PLUGIN_PATH``), and when installed alongside Bro (after
+``make install``).
+
+``make install`` copies over the ``lib`` and ``scripts`` directories,
+as well as the ``__bro_plugin__`` magic file and the ``README`` (which
+you should customize). One can add further CMake ``install`` rules to
+install additional files if neeed.
+
+.. todo::
+
+    Describe the other files that the script puts in place.
+
+Activating a Plugin
+===================
+
+A plugin needs to be *activated* to make it available to the user.
+Activating a plugin will:
+
+    1. Load the dynamic module
+    2. Make any bif items available
+    3. Add the ``scripts/`` directory to ``BROPATH``
+    4. Load ``scripts/__load__.bro``
+
+By default, Bro will automatically activate all dynamic plugins found
+in its search path ``BRO_PLUGIN_PATH``. However, in bare mode (``bro
+-b``), no dynamic plugins will be activated by default; instead the
+user can selectively enable individual plugins in scriptland using the
+``@load-plugin <qualified-plugin-name>`` directive (e.g.,
+``@load-plugin Demo::Rot13``).
+
+``bro -N`` shows activated and found yet unactivated plugins
+separately. Note that plugins compiled statically into Bro are always
+activated, and hence show up as such even in bare mode.
+
+.. todo::
+
+    Is this the right activation model?
+
+
+Plugin Component
+================
+
+The following gives additional information about providing individual
+types of functionality via plugins. Note that a single plugin can
+provide more than one type. For example, a plugin could provide
+multiple protocol analyzers at once; or both a logging backend and
+input reader at the same time.
+
+We now walk briefly through the specifics of providing a specific type
+of functionality (a *component*) through plugin. We'll focus on their
+interfaces to the plugin system, rather than specifics on writing the
+corresponding logic (usually the best way to get going on that is to
+start with an existing plugin providing a corresponding component and
+adapt that). We'll also point out how the CMake infrastructure put in
+place by the ``init-plugin`` helper script ties the various pieces
+together.
+
+Bro Scripts
+-----------
+
+Scripts are easy: just put them into ``scripts/``, as described above.
+The CMake infrastructure will automatically install them, as well
+include them into the source and binary plugin distributions.
+
+Builtin Language Elements
+-------------------------
+
+Functions
+    TODO
+
+Events
+    TODO
+
+Types
+    TODO
+
+Protocol Analyzers
+------------------
+
+TODO.
+
+File Analyzers
+--------------
+
+TODO.
+
+Logging Writer
+--------------
+
+Not yet implemented.
+
+Input Reader
+------------
+
+Not yet implemented.
+
+Packet Sources
+--------------
+
+Not yet implemented.
+
+Packet Dumpers
+--------------
+
+Not yet implemented.
+
+Documenting Plugins
+===================
+
+..todo::
+
+    Integrate all this with Broxygen.
+
+
+

doc/frameworks/file-analysis.rst

@@ -1,3 +1,6 @@
+
+.. _file-analysis-framework:
+
 =============
 File Analysis
 =============

doc/frameworks/notice.rst

@@ -1,4 +1,6 @@
 
+.. _notice-framework:
+
 Notice Framework
 ================
 

doc/frameworks/sumstats.rst

@@ -1,3 +1,6 @@
+
+.. _sumstats-framework:
+
 ==================
 Summary Statistics
 ==================

doc/httpmonitor/file_extraction.bro

@@ -0,0 +1,24 @@
+
+global mime_to_ext: table[string] of string = {
+	["application/x-dosexec"] = "exe",
+	["text/plain"] = "txt",
+	["image/jpeg"] = "jpg",
+	["image/png"] = "png",
+	["text/html"] = "html",
+};
+
+event file_new(f: fa_file)
+	{
+	if ( f$source != "HTTP" )
+		return;
+
+	if ( ! f?$mime_type )
+		return;
+
+	if ( f$mime_type !in mime_to_ext )
+		return;
+
+	local fname = fmt("%s-%s.%s", f$source, f$id, mime_to_ext[f$mime_type]);
+	print fmt("Extracting file %s", fname);
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT, [$extract_filename=fname]);
+	}

doc/httpmonitor/http_proxy_01.bro

@@ -0,0 +1,5 @@
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri && c$http$status_code == 200 )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

doc/httpmonitor/http_proxy_02.bro

@@ -0,0 +1,26 @@
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

doc/httpmonitor/http_proxy_03.bro

@@ -0,0 +1,31 @@
+
+@load base/utils/site
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		print fmt("A local server is acting as an open proxy: %s", c$id$resp_h);
+	}

doc/httpmonitor/http_proxy_04.bro

@@ -0,0 +1,40 @@
+@load base/utils/site
+@load base/frameworks/notice
+
+redef Site::local_nets += { 192.168.0.0/16 };
+
+module HTTP;
+
+export {
+
+	redef enum Notice::Type += {
+		Open_Proxy
+	};
+
+	global success_status_codes: set[count] = {
+		200,
+		201,
+		202,
+		203,
+		204,
+		205,
+		206,
+		207,
+		208,
+		226,
+		304
+	};
+}
+
+event http_reply(c: connection, version: string, code: count, reason: string)
+	{
+	if ( Site::is_local_addr(c$id$resp_h) &&
+	     /^[hH][tT][tT][pP]:/ in c$http$uri &&
+	     c$http$status_code in HTTP::success_status_codes )
+		NOTICE([$note=HTTP::Open_Proxy,
+		        $msg=fmt("A local server is acting as an open proxy: %s",
+		                 c$id$resp_h),
+		        $conn=c,
+		        $identifier=cat(c$id$resp_h),
+		        $suppress_for=1day]);
+	}

doc/httpmonitor/index.rst

@@ -0,0 +1,163 @@
+
+.. _http-monitor:
+
+================================
+Monitoring HTTP Traffic with Bro
+================================
+
+Bro can be used to log the entire HTTP traffic from your network to the
+http.log file.  This file can then be used for analysis and auditing
+purposes.
+
+In the sections below we briefly explain the structure of the http.log
+file. Then, we show you how to perform basic HTTP traffic monitoring and
+analysis tasks with Bro. Some of these ideas and techniques can later be
+applied to monitor different protocols in a similar way.
+
+----------------------------
+Introduction to the HTTP log
+----------------------------
+
+The http.log file contains a summary of all HTTP requests and responses
+sent over a Bro-monitored network. Here are the first few columns of
+``http.log``::
+
+    # ts          uid          orig_h        orig_p  resp_h         resp_p
+    1311627961.8  HSH4uV8KVJg  192.168.1.100 52303   192.150.187.43 80
+
+Every single line in this log starts with a timestamp, a unique
+connection identifier (UID), and a connection 4-tuple (originator
+host/port and responder host/port).  The UID can be used to identify all
+logged activity (possibly across multiple log files) associated with a
+given connection 4-tuple over its lifetime.
+
+The remaining columns detail the activity that's occurring.  For
+example, the columns on the line below (shortened for brevity) show a
+request to the root of Bro website::
+
+    # method   host         uri  referrer  user_agent
+    GET        bro.org  /    -         <...>Chrome/12.0.742.122<...>
+
+Network administrators and security engineers, for instance, can use the
+information in this log to understand the HTTP activity on the network
+and troubleshoot network problems or search for anomalous activities. At
+this point, we would like to stress out the fact that there is no just
+one right way to perform analysis; it will depend on the expertise of
+the person doing the analysis and the specific details of the task to
+accomplish.
+
+For more information about how to handle the HTTP protocol in Bro,
+including a complete list of the fields available in http.log, go to
+Bro's :doc:`HTTP script reference
+</scripts/base/protocols/http/main.bro>`.
+
+------------------------
+Detecting a Proxy Server
+------------------------
+
+A proxy server is a device on your network configured to request a
+service on behalf of a third system; one of the most common examples is
+a Web proxy server. A client without Internet access connects to the
+proxy and requests a Web page; the proxy then sends the request to the
+actual Web server, receives the response and passes it to the original
+client.
+
+Proxies were conceived to help manage a network and provide better
+encapsulation. By themselves, proxies are not a security threat, but a
+misconfigured or unauthorized proxy can allow others, either inside or
+outside the network, to access any Web site and even conduct malicious
+activities anonymously using the network resources.
+
+What Proxy Server traffic looks like
+-------------------------------------
+
+In general, when a client starts talking with a proxy server, the
+traffic consists of two parts: (i) a GET request, and (ii) an HTTP/
+reply::
+
+    Request: GET http://www.bro.org/ HTTP/1.1
+    Reply:   HTTP/1.0 200 OK
+
+This will differ from traffic between a client and a normal Web server
+because GET requests should not include "http" on the string. So we can
+use this to identify a proxy server.
+
+We can write a basic script in Bro to handle the http_reply event and
+detect a reply for a ``GET http://`` request.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
+
+.. btest:: http_proxy_01
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_01.bro
+
+Basically, the script is checking for a "200 OK" status code on a reply
+for a request that includes "http:" (case insensitive). In reality, the
+HTTP protocol defines several success status codes other than 200, so we
+will extend our basic script to also consider the additional codes.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
+
+.. btest:: http_proxy_02
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_02.bro
+
+Next, we will make sure that the responding proxy is part of our local
+network.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
+
+.. btest:: http_proxy_03
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_03.bro
+
+.. note::
+
+    The redefinition of :bro:see:`Site::local_nets` is only done inside
+    this script to make it a self-contained example.  It's typically
+    redefined somewhere else.
+
+Finally, our goal should be to generate an alert when a proxy has been
+detected instead of printing a message on the console output.  For that,
+we will tag the traffic accordingly and define a new ``Open_Proxy``
+``Notice`` type to alert of all tagged communications. Once a
+notification has been fired, we will further suppress it for one day.
+Below is the complete script.
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+
+.. btest:: http_proxy_04
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/proxy.pcap ${DOC_ROOT}/httpmonitor/http_proxy_04.bro
+    @TEST-EXEC: btest-rst-include notice.log
+
+Note that this script only logs the presence of the proxy to
+``notice.log``, but if an additional email is desired (and email
+functionality is enabled), then that's done simply by redefining
+:bro:see:`Notice::emailed_types` to add the ``Open_proxy`` notice type
+to it.
+
+----------------
+Inspecting Files
+----------------
+
+Files are often transmitted on regular HTTP conversations between a
+client and a server. Most of the time these files are harmless, just
+images and some other multimedia content, but there are also types of
+files, specially executable files, that can damage your system. We can
+instruct Bro to create a copy of all files of certain types that it sees
+using the :ref:`File Analysis Framework <file-analysis-framework>`
+(introduced with Bro 2.2):
+
+.. btest-include:: ${DOC_ROOT}/httpmonitor/file_extraction.bro
+
+.. btest:: file_extraction
+
+    @TEST-EXEC: btest-rst-cmd -n 5 bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/httpmonitor/file_extraction.bro
+
+Here, the ``mime_to_ext`` table serves two purposes.  It defines which
+mime types to extract and also the file suffix of the extracted files.
+Extracted files are written to a new ``extract_files`` subdirectory.
+Also note that the first conditional in the :bro:see:`file_new` event
+handler can be removed to make this behavior generic to other protocols
+besides HTTP.

doc/index.rst

@@ -1,9 +1,12 @@
 
 .. Bro documentation master file
 
-=================
+==========
-Bro Documentation
+Bro Manual
-=================
+==========
+
+Introduction Section
+====================
 
 .. toctree::
    :maxdepth: 2
@@ -11,13 +14,36 @@ Bro Documentation
    intro/index.rst
    install/index.rst
    quickstart/index.rst
-   using/index.rst
+
+.. 
+
+Using Bro Section
+=================
+
+.. toctree::
+   :maxdepth: 2
+
+   logs/index.rst
+   httpmonitor/index.rst
+   broids/index.rst
+   mimestats/index.rst
+   cluster/index.rst
+
+..
+
+Reference Section
+=================
+
+.. toctree::
+   :maxdepth: 2
+
    scripting/index.rst
    frameworks/index.rst
-   cluster/index.rst
    script-reference/index.rst
    components/index.rst
 
+..
+
 * :ref:`General Index <genindex>`
 * :ref:`search`
 

doc/install/install.rst

@@ -29,14 +29,13 @@ before you begin:
     * Libpcap                           (http://www.tcpdump.org)
     * OpenSSL libraries                 (http://www.openssl.org)
     * BIND8 library
-    * Libmagic 5.04 or greater
     * Libz
     * Bash (for BroControl)
     * Python (for BroControl)
 
 To build Bro from source, the following additional dependencies are required:
 
-    * CMake 2.6.3 or greater            (http://www.cmake.org)
+    * CMake 2.8.0 or greater            (http://www.cmake.org)
     * Make
     * C/C++ compiler
     * SWIG                              (http://www.swig.org)
@@ -44,7 +43,6 @@ To build Bro from source, the following additional dependencies are required:
     * Flex  (Fast Lexical Analyzer)
     * Libpcap headers                   (http://www.tcpdump.org)
     * OpenSSL headers                   (http://www.openssl.org)
-    * libmagic headers
     * zlib headers
     * Perl
 
@@ -55,13 +53,13 @@ that ``bash`` and ``python`` are in your ``PATH``):
 
   .. console::
 
-     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel file-devel
+     sudo yum install cmake make gcc gcc-c++ flex bison libpcap-devel openssl-devel python-devel swig zlib-devel
 
 * DEB/Debian-based Linux:
 
   .. console::
 
-     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev libmagic-dev
+     sudo apt-get install cmake make gcc g++ flex bison libpcap-dev libssl-dev python-dev swig zlib1g-dev
 
 * FreeBSD:
 
@@ -78,15 +76,11 @@ that ``bash`` and ``python`` are in your ``PATH``):
   then going through its "Preferences..." -> "Downloads" menus to
   install the "Command Line Tools" component.
 
-  Lion (10.7) and Mountain Lion (10.8) come with all required
+  OS X comes with all required dependencies except for CMake_ and SWIG_.
-  dependencies except for CMake_, SWIG_, and ``libmagic``.
-
   Distributions of these dependencies can likely be obtained from your
   preferred Mac OS X package management system (e.g. MacPorts_, Fink_,
-  or Homebrew_).
+  or Homebrew_).  Specifically for MacPorts, the ``cmake``, ``swig``,
-
+  ``swig-python`` and packages provide the required dependencies.
-  Specifically for MacPorts, the ``cmake``, ``swig``,
-  ``swig-python`` and ``file`` packages provide the required dependencies.
 
 
 Optional Dependencies

doc/logs/index.rst

@@ -1,9 +1,9 @@
 
-.. _using-bro:
+.. _bro-logging:
 
-=========
+===========
-Using Bro
+Bro Logging
-=========
+===========
 
 .. contents::
 
@@ -251,3 +251,42 @@ stream and Bro is able to extract and track that information for you,
 giving you an in-depth and structured view into HTTP traffic on your
 network.
 
+-----------------------
+Common Log Files
+-----------------------
+As a monitoring tool, Bro records a detailed view of the traffic inspected and the events generated in 
+a series of relevant log files. These files can later be reviewed for monitoring, auditing and troubleshooting 
+purposes.
+
+In this section we present a brief explanation of the most commonly used log files generated by Bro including links 
+to descriptions of some of the fields for each log type.
+
++-----------------+---------------------------------------+------------------------------+
+| Log File        | Description                           | Field Descriptions           |
++=================+=======================================+==============================+
+| http.log        | Shows all HTTP requests and replies   | :bro:type:`HTTP::Info`       |
++-----------------+---------------------------------------+------------------------------+
+| ftp.log         | Records FTP activity                  | :bro:type:`FTP::Info`        |
++-----------------+---------------------------------------+------------------------------+
+| ssl.log         | Records SSL sessions including        | :bro:type:`SSL::Info`        |
+|                 | certificates used                     |                              |
++-----------------+---------------------------------------+------------------------------+
+| known_certs.log | Includes SSL certificates used        | :bro:type:`Known::CertsInfo` |
++-----------------+---------------------------------------+------------------------------+
+| smtp.log        | Summarizes SMTP traffic on a network  | :bro:type:`SMTP::Info`       |
++-----------------+---------------------------------------+------------------------------+
+| dns.log         | Shows all DNS activity on a network   | :bro:type:`DNS::Info`        |
++-----------------+---------------------------------------+------------------------------+
+| conn.log        | Records all connections seen by Bro   | :bro:type:`Conn::Info`       |
++-----------------+---------------------------------------+------------------------------+
+| dpd.log         | Shows network activity on             | :bro:type:`DPD::Info`        |
+|                 | non-standard ports                    |                              |
++-----------------+---------------------------------------+------------------------------+
+| files.log       | Records information about all files   | :bro:type:`Files::Info`      |
+|                 | transmitted over the network          |                              |
++-----------------+---------------------------------------+------------------------------+
+| weird.log       | Records unexpected protocol-level     | :bro:type:`Weird::Info`      |
+|                 | activity                              |                              |
++-----------------+---------------------------------------+------------------------------+
+
+

doc/mimestats/index.rst

@@ -0,0 +1,71 @@
+
+.. _mime-stats:
+
+====================
+MIME Type Statistics
+====================
+
+Files are constantly transmitted over HTTP on regular networks. These
+files belong to a specific category (i.e., executable, text, image,
+etc.) identified by a `Multipurpose Internet Mail Extension (MIME)
+<http://en.wikipedia.org/wiki/MIME>`_. Although MIME was originally
+developed to identify the type of non-text attachments on email, it is
+also used by Web browser to identify the type of files transmitted and
+present them accordingly.
+
+In this tutorial, we will show how to use the Sumstats Framework to
+collect some statistics information based on MIME types, specifically
+the total number of occurrences, size in bytes, and number of unique
+hosts transmitting files over HTTP per each type. For instructions about
+extracting and creating a local copy of these files, visit :ref:`this
+<http-monitor>` tutorial instead.
+
+------------------------------------------------
+MIME Statistics with Sumstats
+------------------------------------------------
+
+When working with the :ref:`Summary Statistics Framework
+<sumstats-framework>`, you need to define three different pieces: (i)
+Observations, where the event is observed and fed into the framework.
+(ii) Reducers, where observations are collected and measured. (iii)
+Sumstats, where the main functionality is implemented.
+
+So, we start by defining our observation along with a record to store
+all statistics values and an observation interval. We are conducting our
+observation on the :bro:see:`HTTP::log_http` event and we are interested
+in the MIME type, size of the file ("response_body_len") and the
+originator host ("orig_h"). We use the MIME type as our key and create
+observers for the other two values.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+    :lines: 6-29, 54-64
+
+Next, we create the reducers. The first one will accumulate file sizes
+and the second one will make sure we only store a host ID once. Below is
+the partial code from a :bro:see:`bro_init` handler.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+    :lines: 34-37
+
+In our final step, we create the SumStats where we check for the
+observation interval and once it expires, we populate the record
+(defined above) with all the relevant data and write it to a log.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+    :lines: 38-51
+
+Putting everything together we end up with the following final code for
+our script.
+
+.. btest-include:: ${DOC_ROOT}/mimestats/mimestats.bro
+
+.. btest:: mimestats
+
+    @TEST-EXEC: btest-rst-cmd bro -r ${TRACES}/http/bro.org.pcap ${DOC_ROOT}/mimestats/mimestats.bro
+    @TEST-EXEC: btest-rst-include mime_metrics.log
+
+.. note::
+
+    The redefinition of :bro:see:`Site::local_nets` is only done inside
+    this script to make it a self-contained example.  It's typically
+    redefined somewhere else.

doc/mimestats/mimestats.bro

@@ -0,0 +1,64 @@
+@load base/utils/site
+@load base/frameworks/sumstats
+
+redef Site::local_nets += { 10.0.0.0/8 };
+
+module MimeMetrics;
+
+export {
+
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		## Timestamp when the log line was finished and written.
+		ts:         time   &log;
+		## Time interval that the log line covers.
+		ts_delta:   interval &log;
+		## The mime type
+		mtype:        string &log;
+		## The number of unique local hosts that fetched this mime type
+		uniq_hosts: count  &log;
+		## The number of hits to the mime type
+		hits:       count  &log;
+		## The total number of bytes received by this mime type
+		bytes:      count  &log;
+	};
+
+	## The frequency of logging the stats collected by this script.
+	const break_interval = 5mins &redef;
+}
+
+event bro_init() &priority=3
+	{
+	Log::create_stream(MimeMetrics::LOG, [$columns=Info]);
+	local r1: SumStats::Reducer = [$stream="mime.bytes",
+	                               $apply=set(SumStats::SUM)];
+	local r2: SumStats::Reducer = [$stream="mime.hits", 
+	                               $apply=set(SumStats::UNIQUE)];
+	SumStats::create([$name="mime-metrics",
+	                  $epoch=break_interval,
+	                  $reducers=set(r1, r2),
+	                  $epoch_result(ts: time, key: SumStats::Key, result: SumStats::Result) =
+	                        {
+	                        local l: Info;
+	                        l$ts         = network_time();
+	                        l$ts_delta   = break_interval;
+	                        l$mtype      = key$str;
+	                        l$bytes      = double_to_count(floor(result["mime.bytes"]$sum));
+	                        l$hits       = result["mime.hits"]$num;
+	                        l$uniq_hosts = result["mime.hits"]$unique;
+	                        Log::write(MimeMetrics::LOG, l);
+	                        }]);
+	}
+
+event HTTP::log_http(rec: HTTP::Info)
+	{
+	if ( Site::is_local_addr(rec$id$orig_h) && rec?$resp_mime_types )
+		{
+		local mime_type = rec$resp_mime_types[0];
+		SumStats::observe("mime.bytes", [$str=mime_type],
+		                  [$num=rec$response_body_len]);
+		SumStats::observe("mime.hits",  [$str=mime_type],
+		                  [$str=cat(rec$id$orig_h)]);
+		}
+	}

doc/quickstart/index.rst

@@ -13,7 +13,7 @@ Bro works on most modern, Unix-based systems and requires no custom
 hardware.  It can be downloaded in either pre-built binary package or
 source code forms.  See :ref:`installing-bro` for instructions on how to
 install Bro. Below, ``$PREFIX`` is used to reference the Bro
-installation root directory, which by default is ``/usr/local/`` if
+installation root directory, which by default is ``/usr/local/bro/`` if
 you install from source. 
 
 Managing Bro with BroControl
@@ -26,8 +26,8 @@ traffic-monitoring cluster.
 A Minimal Starting Configuration
 --------------------------------
 
-These are the basic configuration changes to make for a minimal BroControl installation
+These are the basic configuration changes to make for a minimal BroControl
-that will manage a single Bro instance on the ``localhost``:
+installation that will manage a single Bro instance on the ``localhost``:
 
 1) In ``$PREFIX/etc/node.cfg``, set the right interface to monitor.
 2) In ``$PREFIX/etc/networks.cfg``, comment out the default settings and add
@@ -72,7 +72,8 @@ You can leave it running for now, but to stop this Bro instance you would do:
 
    [BroControl] > stop
 
-We also recommend to insert the following entry into `crontab`::
+We also recommend to insert the following entry into the crontab of the user
+running BroControl::
 
       0-59/5 * * * * $PREFIX/bin/broctl cron
 

doc/script-reference/builtins.rst

@@ -23,7 +23,8 @@ The Bro scripting language supports the following built-in types.
 
 .. bro:type:: void
 
-    An internal Bro type representing the absence of a return type for a
+    An internal Bro type (i.e., "void" is not a reserved keyword in the Bro
+    scripting language) representing the absence of a return type for a
     function.
 
 .. bro:type:: bool
@@ -132,10 +133,23 @@ The Bro scripting language supports the following built-in types.
 
     Strings support concatenation (``+``), and assignment (``=``, ``+=``).
     Strings also support the comparison operators (``==``, ``!=``, ``<``,
-    ``<=``, ``>``, ``>=``).  Substring searching can be performed using
+    ``<=``, ``>``, ``>=``).  The number of characters in a string can be
-    the "in" or "!in" operators (e.g., "bar" in "foobar" yields true).
+    found by enclosing the string within pipe characters (e.g., ``|"abc"|``
-    The number of characters in a string can be found by enclosing the
+    is 3).
-    string within pipe characters (e.g., ``|"abc"|`` is 3).
+
+    The subscript operator can extract an individual character or a substring
+    of a string (string indexing is zero-based, but an index of
+    -1 refers to the last character in the string, and -2 refers to the
+    second-to-last character, etc.).  When extracting a substring, the
+    starting and ending index values are separated by a colon.  For example::
+
+        local orig = "0123456789";
+        local third_char = orig[2];
+        local last_char = orig[-1];
+        local first_three_chars = orig[0:2];
+
+    Substring searching can be performed using the "in" or "!in"
+    operators (e.g., "bar" in "foobar" yields true).
 
     Note that Bro represents strings internally as a count and vector of
     bytes rather than a NUL-terminated byte string (although string
@@ -767,7 +781,7 @@ The Bro scripting language supports the following built-in types.
 .. bro:type:: hook
 
     A hook is another flavor of function that shares characteristics of
-    both a :bro:type:`function` and a :bro:type:`event`.  They are like
+    both a :bro:type:`function` and an :bro:type:`event`.  They are like
     events in that many handler bodies can be defined for the same hook
     identifier and the order of execution can be enforced with
     :bro:attr:`&priority`.  They are more like functions in the way they
@@ -856,14 +870,14 @@ scripting language supports the following built-in attributes.
 .. bro:attr:: &optional
 
     Allows a record field to be missing. For example the type ``record {
-    a: int, b: port &optional }`` could be instantiated both as
+    a: addr; b: port &optional; }`` could be instantiated both as
     singleton ``[$a=127.0.0.1]`` or pair ``[$a=127.0.0.1, $b=80/tcp]``.
 
 .. bro:attr:: &default
 
     Uses a default value for a record field, a function/hook/event
     parameter, or container elements.  For example, ``table[int] of
-    string &default="foo" }`` would create a table that returns the
+    string &default="foo"`` would create a table that returns the
     :bro:type:`string` ``"foo"`` for any non-existing index.
 
 .. bro:attr:: &redef
@@ -901,7 +915,7 @@ scripting language supports the following built-in attributes.
     Called right before a container element expires.  The function's
     first parameter is of the same type of the container and the second
     parameter the same type of the container's index.  The return
-    value is a :bro:type:`interval` indicating the amount of additional
+    value is an :bro:type:`interval` indicating the amount of additional
     time to wait before expiring the container element at the given
     index (which will trigger another execution of this function).
 
@@ -925,7 +939,7 @@ scripting language supports the following built-in attributes.
 
 .. bro:attr:: &persistent
 
-    Makes a variable persistent, i.e., its value is writen to disk (per
+    Makes a variable persistent, i.e., its value is written to disk (per
     default at shutdown time).
 
 .. bro:attr:: &synchronized
@@ -957,8 +971,9 @@ scripting language supports the following built-in attributes.
 
 .. bro:attr:: &priority
 
-    Specifies the execution priority of an event handler. Higher values
+    Specifies the execution priority (as a signed integer) of a hook or
-    are executed before lower ones. The default value is 0.
+    event handler. Higher values are executed before lower ones. The
+    default value is 0.
 
 .. bro:attr:: &group
 

magic

@@ -1 +1 @@
-Subproject commit e87fe13a7b776182ffc8c75076d42702f5c28fed
+Subproject commit 99c6b89230e2b9b0e781c42b0b9412d2ab4e14b2

scripts/base/frameworks/files/main.bro

@@ -56,7 +56,7 @@ export {
 		## local file path which was read, or some other input source.
 		source: string &log &optional;
 
-		## A value to represent the depth of this file in relation 
+		## A value to represent the depth of this file in relation
 		## to its source.  In SMTP, it is the depth of the MIME
 		## attachment on the message.  In HTTP, it is the depth of the
 		## request within the TCP connection.
@@ -72,7 +72,7 @@ export {
 		mime_type: string &log &optional;
 
 		## A filename for the file if one is available from the source
-		## for the file.  These will frequently come from 
+		## for the file.  These will frequently come from
 		## "Content-Disposition" headers in network protocols.
 		filename: string &log &optional;
 
@@ -148,9 +148,18 @@ export {
 	## Returns: true if the analyzer will be added, or false if analysis
 	##          for the file isn't currently active or the *args*
 	##          were invalid for the analyzer type.
-	global add_analyzer: function(f: fa_file, 
+	global add_analyzer: function(f: fa_file,
-	                              tag: Files::Tag, 
+				      tag: Files::Tag,
-	                              args: AnalyzerArgs &default=AnalyzerArgs()): bool;
+				      args: AnalyzerArgs &default=AnalyzerArgs()): bool;
+
+	## Adds all analyzers associated with a give MIME type to the analysis of
+	## a  file.  Note that analyzers added via MIME types cannot take further
+	## arguments.
+	##
+	## f: the file.
+	##
+	## mtype: the MIME type; it will be compared case-insensitive.
+	global add_analyzers_for_mime_type: function(f: fa_file, mtype: string);
 
 	## Removes an analyzer from the analysis of a given file.
 	##
@@ -195,7 +204,7 @@ export {
 		## A callback to generate a file handle on demand when
 		## one is needed by the core.
 		get_file_handle: function(c: connection, is_orig: bool): string;
-		
+
 		## A callback to "describe" a file.  In the case of an HTTP
 		## transfer the most obvious description would be the URL.
 		## It's like an extremely compressed version of the normal log.
@@ -206,7 +215,7 @@ export {
 	## Register callbacks for protocols that work with the Files framework.
 	## The callbacks must uniquely identify a file and each protocol can 
 	## only have a single callback registered for it.
-	## 
+	##
 	## tag: Tag for the protocol analyzer having a callback being registered.
 	##
 	## reg: A :bro:see:`Files::ProtoRegistration` record.
@@ -258,13 +267,13 @@ function set_info(f: fa_file)
 		f$info$source = f$source;
 	f$info$duration = f$last_active - f$info$ts;
 	f$info$seen_bytes = f$seen_bytes;
-	if ( f?$total_bytes ) 
+	if ( f?$total_bytes )
 		f$info$total_bytes = f$total_bytes;
 	f$info$missing_bytes = f$missing_bytes;
 	f$info$overflow_bytes = f$overflow_bytes;
 	if ( f?$is_orig )
 		f$info$is_orig = f$is_orig;
-	if ( f?$mime_type ) 
+	if ( f?$mime_type )
 		f$info$mime_type = f$mime_type;
 	}
 
@@ -288,6 +297,15 @@ function add_analyzer(f: fa_file, tag: Files::Tag, args: AnalyzerArgs): bool
 	return T;
 	}
 
+function add_analyzers_for_mime_type(f: fa_file, mtype: string)
+	{
+	local dummy_args: AnalyzerArgs;
+	local analyzers = __add_analyzers_for_mime_type(f$id, mtype, dummy_args);
+
+	for ( tag in analyzers )
+		add f$info$analyzers[Files::analyzer_name(tag)];
+	}
+
 function register_analyzer_add_callback(tag: Files::Tag, callback: function(f: fa_file, args: AnalyzerArgs))
 	{
 	analyzer_add_callbacks[tag] = callback;
@@ -311,6 +329,9 @@ function analyzer_name(tag: Files::Tag): string
 event file_new(f: fa_file) &priority=10
 	{
 	set_info(f);
+
+	if ( f?$mime_type )
+		add_analyzers_for_mime_type(f, f$mime_type);
 	}
 
 event file_over_new_connection(f: fa_file, c: connection, is_orig: bool) &priority=10

scripts/base/frameworks/notice/cluster.bro

@@ -23,7 +23,8 @@ redef Cluster::worker2manager_events += /Notice::cluster_notice/;
 @if ( Cluster::local_node_type() != Cluster::MANAGER )
 event Notice::begin_suppression(n: Notice::Info)
 	{
-	suppressing[n$note, n$identifier] = n;
+	local suppress_until = n$ts + n$suppress_for;
+	suppressing[n$note, n$identifier] = suppress_until;
 	}
 @endif
 

scripts/base/frameworks/notice/main.bro

@@ -242,12 +242,6 @@ export {
 	##    being suppressed.
 	global suppressed: event(n: Notice::Info);
 
-	## This event is generated when a notice stops being suppressed.
-	##
-	## n: The record containing notice data regarding the notice type
-	##    that was being suppressed.
-	global end_suppression: event(n: Notice::Info);
-
 	## Call this function to send a notice in an email.  It is already used
 	## by default with the built in :bro:enum:`Notice::ACTION_EMAIL` and
 	## :bro:enum:`Notice::ACTION_PAGE` actions.
@@ -285,27 +279,22 @@ export {
 }
 
 # This is used as a hack to implement per-item expiration intervals.
-function per_notice_suppression_interval(t: table[Notice::Type, string] of Notice::Info, idx: any): interval
+function per_notice_suppression_interval(t: table[Notice::Type, string] of time, idx: any): interval
 	{
 	local n: Notice::Type;
 	local s: string;
 	[n,s] = idx;
 
-	local suppress_time = t[n,s]$suppress_for - (network_time() - t[n,s]$ts);
+	local suppress_time = t[n,s] - network_time();
 	if ( suppress_time < 0secs )
 		suppress_time = 0secs;
 
-	# If there is no more suppression time left, the notice needs to be sent
-	# to the end_suppression event.
-	if ( suppress_time == 0secs )
-		event Notice::end_suppression(t[n,s]);
-
 	return suppress_time;
 	}
 
 # This is the internally maintained notice suppression table.  It's
 # indexed on the Notice::Type and the $identifier field from the notice.
-global suppressing: table[Type, string] of Notice::Info = {}
+global suppressing: table[Type, string] of time = {}
 		&create_expire=0secs
 		&expire_func=per_notice_suppression_interval;
 
@@ -400,11 +389,22 @@ function email_notice_to(n: Notice::Info, dest: string, extend: bool)
 
 	# First off, finish the headers and include the human readable messages
 	# then leave a blank line after the message.
-	email_text = string_cat(email_text, "\nMessage: ", n$msg);
+	email_text = string_cat(email_text, "\nMessage: ", n$msg, "\n");
-	if ( n?$sub )
-		email_text = string_cat(email_text, "\nSub-message: ", n$sub);
 
-	email_text = string_cat(email_text, "\n\n");
+	if ( n?$sub )
+		email_text = string_cat(email_text, "Sub-message: ", n$sub, "\n");
+
+	email_text = string_cat(email_text, "\n");
+
+	# Add information about the file if it exists.
+	if ( n?$file_desc )
+		email_text = string_cat(email_text, "File Description: ", n$file_desc, "\n");
+
+	if ( n?$file_mime_type )
+		email_text = string_cat(email_text, "File MIME Type: ", n$file_mime_type, "\n");
+
+	if ( n?$file_desc || n?$file_mime_type )
+		email_text = string_cat(email_text, "\n");
 
 	# Next, add information about the connection if it exists.
 	if ( n?$id )
@@ -467,7 +467,8 @@ hook Notice::notice(n: Notice::Info) &priority=-5
 	     [n$note, n$identifier] !in suppressing &&
 	     n$suppress_for != 0secs )
 		{
-		suppressing[n$note, n$identifier] = n;
+		local suppress_until = n$ts + n$suppress_for;
+		suppressing[n$note, n$identifier] = suppress_until;
 		event Notice::begin_suppression(n);
 		}
 	}

scripts/base/init-bare.bro

@@ -60,6 +60,13 @@ type addr_vec: vector of addr;
 ##    directly and then remove this alias.
 type table_string_of_string: table[string] of string;
 
+## A set of file analyzer tags.
+##
+## .. todo:: We need this type definition only for declaring builtin functions
+##    via ``bifcl``. We should extend ``bifcl`` to understand composite types
+##    directly and then remove this alias.
+type files_tag_set: set[Files::Tag];
+
 ## A connection's transport-layer protocol. Note that Bro uses the term
 ## "connection" broadly, using flow semantics for ICMP and UDP.
 type transport_proto: enum {
@@ -3057,6 +3064,9 @@ export {
 	## Toggle whether to do GTPv1 decapsulation.
 	const enable_gtpv1 = T &redef;
 
+	## Toggle whether to do GRE decapsulation.
+	const enable_gre = T &redef;
+
 	## With this option set, the Teredo analysis will first check to see if
 	## other protocol analyzers have confirmed that they think they're
 	## parsing the right protocol and only continue with Teredo tunnel
@@ -3082,7 +3092,8 @@ export {
 	## may work better.
 	const delay_gtp_confirmation = F &redef;
 
-	## How often to cleanup internal state for inactive IP tunnels.
+	## How often to cleanup internal state for inactive IP tunnels
+	## (includes GRE tunnels).
 	const ip_tunnel_timeout = 24hrs &redef;
 } # end export
 module GLOBAL;

scripts/policy/protocols/ssl/validate-certs.bro

@@ -40,6 +40,7 @@ event ssl_established(c: connection) &priority=3
 		{
 		local result = x509_verify(c$ssl$cert, c$ssl$cert_chain, root_certs);
 		c$ssl$validation_status = x509_err2str(result);
+		recently_validated_certs[c$ssl$cert_hash] = c$ssl$validation_status;
 		}
 		
 	if ( c$ssl$validation_status != "ok" )

src/3rdparty

@@ -1 +1 @@
-Subproject commit 12b5cb446c8128bb22e5cbd7baa7d53669539487
+Subproject commit 42a4c9694a2b2677b050fbb7cbae26bc5ec4605a

src/Attr.cc

@@ -317,8 +317,9 @@ void Attributes::CheckAttr(Attr* a)
 				break;
 
 			// Table defaults may be promotable.
-			if ( (ytype->Tag() == TYPE_RECORD && atype->Tag() == TYPE_RECORD &&
+			if ( ytype && ytype->Tag() == TYPE_RECORD &&
-			      record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType())) )
+			     atype->Tag() == TYPE_RECORD &&
+			     record_promotion_compatible(atype->AsRecordType(), ytype->AsRecordType()) )
 				// Ok.
 				break;
 

src/BroDoc.cc

@@ -0,0 +1,671 @@
+#include <cstdio>
+#include <cstdarg>
+#include <string>
+#include <list>
+#include <algorithm>
+#include <libgen.h>
+
+#include "BroDoc.h"
+#include "BroDocObj.h"
+#include "util.h"
+#include "plugin/Manager.h"
+#include "analyzer/Manager.h"
+#include "analyzer/Component.h"
+#include "file_analysis/Manager.h"
+
+BroDoc::BroDoc(const std::string& rel, const std::string& abs)
+	{
+	size_t f_pos = abs.find_last_of('/');
+
+	if ( std::string::npos == f_pos )
+		source_filename = abs;
+	else
+		source_filename = abs.substr(f_pos + 1);
+
+	if ( rel[0] == '/' || rel[0] == '.' )
+		{
+		// The Bro script isn't being loaded via BROPATH, so just use basename
+		// as the document title.
+		doc_title = source_filename;
+		}
+	else
+		{
+		// Keep the relative directory as part of the document title.
+		if ( rel.size() == 0 || rel[rel.size() - 1] == '/' )
+			doc_title = rel + source_filename;
+		else
+			doc_title = rel + "/" + source_filename;
+		}
+
+	downloadable_filename = source_filename;
+
+#if 0
+	size_t ext_pos = downloadable_filename.find(".bif.bro");
+	if ( std::string::npos != ext_pos )
+		downloadable_filename.erase(ext_pos + 4);
+#endif
+
+	reST_filename = doc_title;
+	size_t ext_pos = reST_filename.find(".bro");
+
+	if ( std::string::npos == ext_pos )
+		reST_filename += ".rst";
+	else
+		reST_filename.replace(ext_pos, 4, ".rst");
+
+	reST_filename = doc_title.substr(0, ext_pos);
+	reST_filename += ".rst";
+
+	// Instead of re-creating the directory hierarchy based on related
+	// loads, just replace the directory separatories such that the reST
+	// output will all be placed in a flat directory (the working dir).
+	std::for_each(reST_filename.begin(), reST_filename.end(), replace_slash());
+
+	reST_file = fopen(reST_filename.c_str(), "w");
+
+	if ( ! reST_file )
+		fprintf(stderr, "Failed to open %s\n", reST_filename.c_str());
+
+#ifdef DOCDEBUG
+	fprintf(stdout, "Documenting absolute source: %s\n", abs.c_str());
+	fprintf(stdout, "\trelative dir: %s\n", rel.c_str());
+	fprintf(stdout, "\tdoc title: %s\n", doc_title.c_str());
+	fprintf(stdout, "\tbro file: %s\n", source_filename.c_str());
+	fprintf(stdout, "\trst file: %s\n", reST_filename.c_str());
+#endif
+	}
+
+BroDoc::~BroDoc()
+	{
+	if ( reST_file && fclose( reST_file ) )
+		fprintf(stderr, "Failed to close %s\n", reST_filename.c_str());
+
+	FreeBroDocObjPtrList(all);
+	}
+
+void BroDoc::AddImport(const std::string& s)
+	{
+	std::string lname(s);
+	// First strip any .bro extension.
+	size_t ext_pos = lname.find(".bro");
+	if ( ext_pos != std::string::npos )
+		lname = lname.substr(0, ext_pos);
+
+	const char* full_filename = NULL;
+	const char* subpath = NULL;
+
+	FILE* f = search_for_file(lname.c_str(), "bro", &full_filename, true,
+	                          &subpath);
+
+	if ( f && full_filename && subpath )
+		{
+		char* tmp = copy_string(full_filename);
+		char* filename = basename(tmp);
+		extern char* PACKAGE_LOADER;
+
+		if ( streq(filename, PACKAGE_LOADER) )
+			{
+			// link to the package's index
+			string pkg(subpath);
+			pkg += "/index";
+			imports.push_back(pkg);
+			}
+		else
+			{
+			if ( subpath[0] == '/' || subpath[0] == '.' )
+				{
+				// it's not a subpath of scripts/, so just add the name of it
+				// as it's given in the @load directive
+				imports.push_back(lname);
+				}
+			else
+				{
+				// combine the base file name of script in the @load directive
+				// with the subpath of BROPATH's scripts/ directory
+				string fname(subpath);
+				char* othertmp = copy_string(lname.c_str());
+				fname.append("/").append(basename(othertmp));
+				imports.push_back(fname);
+				delete [] othertmp;
+				}
+			}
+
+		delete [] tmp;
+		}
+
+	else
+		fprintf(stderr, "Failed to document '@load %s' in file: %s\n",
+		        s.c_str(), reST_filename.c_str());
+
+	if ( f )
+		fclose(f);
+
+	delete [] full_filename;
+	delete [] subpath;
+	}
+
+void BroDoc::SetPacketFilter(const std::string& s)
+	{
+	packet_filter = s;
+	size_t pos1 = s.find("{\n");
+	size_t pos2 = s.find("}");
+
+	if ( pos1 != std::string::npos && pos2 != std::string::npos )
+		packet_filter = s.substr(pos1 + 2, pos2 - 2);
+
+	bool has_non_whitespace = false;
+
+	for ( std::string::const_iterator it = packet_filter.begin();
+		it != packet_filter.end(); ++it )
+		{
+		if ( *it != ' ' && *it != '\t' && *it != '\n' && *it != '\r' )
+			{
+			has_non_whitespace = true;
+			break;
+			}
+		}
+
+	if ( ! has_non_whitespace )
+		packet_filter.clear();
+	}
+
+void BroDoc::WriteDocFile() const
+	{
+	WriteToDoc(reST_file, ".. Automatically generated.  Do not edit.\n\n");
+
+	WriteToDoc(reST_file, ":tocdepth: 3\n\n");
+
+	WriteSectionHeading(reST_file, doc_title.c_str(), '=');
+
+	WriteStringList(reST_file, ".. bro:namespace:: %s\n", modules);
+
+	WriteToDoc(reST_file, "\n");
+
+	// WriteSectionHeading(reST_file, "Overview", '-');
+	WriteStringList(reST_file, "%s\n", summary);
+
+	WriteToDoc(reST_file, "\n");
+
+	if ( ! modules.empty() )
+		{
+		WriteToDoc(reST_file, ":Namespace%s: ", (modules.size() > 1 ? "s" : ""));
+		// WriteStringList(reST_file, ":bro:namespace:`%s`", modules);
+		WriteStringList(reST_file, "``%s``, ", "``%s``", modules);
+		WriteToDoc(reST_file, "\n");
+		}
+
+	if ( ! imports.empty() )
+		{
+		WriteToDoc(reST_file, ":Imports: ");
+		std::list<std::string>::const_iterator it;
+		for ( it = imports.begin(); it != imports.end(); ++it )
+			{
+			if ( it != imports.begin() )
+				WriteToDoc(reST_file, ", ");
+
+			string pretty(*it);
+			size_t pos = pretty.find("/index");
+			if ( pos != std::string::npos && pos + 6 == pretty.size() )
+				pretty = pretty.substr(0, pos);
+			WriteToDoc(reST_file, ":doc:`%s </scripts/%s>`", pretty.c_str(), it->c_str());
+			}
+		WriteToDoc(reST_file, "\n");
+		}
+
+	WriteToDoc(reST_file, ":Source File: :download:`%s`\n",
+		downloadable_filename.c_str());
+
+	WriteToDoc(reST_file, "\n");
+
+	WriteInterface("Summary", '~', '#', true, true);
+
+	if ( ! notices.empty() )
+		WriteBroDocObjList(reST_file, notices, "Notices", '#');
+
+	if ( port_analysis.size() || packet_filter.size() )
+		WriteSectionHeading(reST_file, "Configuration Changes", '#');
+
+	if ( ! port_analysis.empty() )
+		{
+		WriteSectionHeading(reST_file, "Port Analysis", '^');
+		WriteToDoc(reST_file, "Loading this script makes the following changes to "
+		           ":bro:see:`dpd_config`.\n\n");
+		WriteStringList(reST_file, "%s, ", "%s", port_analysis);
+		}
+
+	if ( ! packet_filter.empty() )
+		{
+		WriteSectionHeading(reST_file, "Packet Filter", '^');
+		WriteToDoc(reST_file, "Loading this script makes the following changes to "
+		           ":bro:see:`capture_filters`.\n\n");
+		WriteToDoc(reST_file, "Filters added::\n\n");
+		WriteToDoc(reST_file, "%s\n", packet_filter.c_str());
+		}
+
+	WriteInterface("Detailed Interface", '~', '#', true, false);
+
+#if 0   // Disabled for now.
+	BroDocObjList::const_iterator it;
+	bool hasPrivateIdentifiers = false;
+
+	for ( it = all.begin(); it != all.end(); ++it )
+		{
+		if ( ! IsPublicAPI(*it) )
+			{
+			hasPrivateIdentifiers = true;
+			break;
+			}
+		}
+
+	if ( hasPrivateIdentifiers )
+		WriteInterface("Private Interface", '~', '#', false, false);
+#endif
+	}
+
+void BroDoc::WriteInterface(const char* heading, char underline,
+			char sub, bool isPublic, bool isShort) const
+	{
+	WriteSectionHeading(reST_file, heading, underline);
+	WriteBroDocObjList(reST_file, options, isPublic, "Options", sub, isShort);
+	WriteBroDocObjList(reST_file, constants, isPublic, "Constants", sub, isShort);
+	WriteBroDocObjList(reST_file, state_vars, isPublic, "State Variables", sub, isShort);
+	WriteBroDocObjList(reST_file, types, isPublic, "Types", sub, isShort);
+	WriteBroDocObjList(reST_file, events, isPublic, "Events", sub, isShort);
+	WriteBroDocObjList(reST_file, hooks, isPublic, "Hooks", sub, isShort);
+	WriteBroDocObjList(reST_file, functions, isPublic, "Functions", sub, isShort);
+	WriteBroDocObjList(reST_file, redefs, isPublic, "Redefinitions", sub, isShort);
+	}
+
+void BroDoc::WriteStringList(FILE* f, const char* format, const char* last_format,
+			const std::list<std::string>& l)
+	{
+	if ( l.empty() )
+		{
+		WriteToDoc(f, "\n");
+		return;
+		}
+
+	std::list<std::string>::const_iterator it;
+	std::list<std::string>::const_iterator last = l.end();
+	last--;
+
+	for ( it = l.begin(); it != last; ++it )
+		WriteToDoc(f, format, it->c_str());
+
+	WriteToDoc(f, last_format, last->c_str());
+	}
+
+void BroDoc::WriteBroDocObjTable(FILE* f, const BroDocObjList& l)
+	{
+	int max_id_col = 0;
+	int max_com_col = 0;
+	BroDocObjList::const_iterator it;
+
+	for ( it = l.begin(); it != l.end(); ++it )
+		{
+		int c = (*it)->ColumnSize();
+
+		if ( c > max_id_col )
+			max_id_col = c;
+
+		c = (*it)->LongestShortDescLen();
+
+		if ( c > max_com_col )
+			max_com_col = c;
+		}
+
+	// Start table.
+	WriteRepeatedChar(f, '=', max_id_col);
+	WriteToDoc(f, " ");
+
+	if ( max_com_col == 0 )
+		WriteToDoc(f, "=");
+	else
+		WriteRepeatedChar(f, '=', max_com_col);
+
+	WriteToDoc(f, "\n");
+
+	for ( it = l.begin(); it != l.end(); ++it )
+		{
+		if ( it != l.begin() )
+			WriteToDoc(f, "\n\n");
+		(*it)->WriteReSTCompact(f, max_id_col);
+		}
+
+	// End table.
+	WriteToDoc(f, "\n");
+	WriteRepeatedChar(f, '=', max_id_col);
+	WriteToDoc(f, " ");
+
+	if ( max_com_col == 0 )
+		WriteToDoc(f, "=");
+	else
+		WriteRepeatedChar(f, '=', max_com_col);
+
+	WriteToDoc(f, "\n\n");
+	}
+
+void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, bool wantPublic,
+			const char* heading, char underline, bool isShort)
+	{
+	if ( l.empty() )
+		return;
+
+	BroDocObjList::const_iterator it;
+	bool (*f_ptr)(const BroDocObj* o) = 0;
+
+	if ( wantPublic )
+		f_ptr = IsPublicAPI;
+	else
+		f_ptr = IsPrivateAPI;
+
+	it = std::find_if(l.begin(), l.end(), f_ptr);
+
+	if ( it == l.end() )
+		return;
+
+	WriteSectionHeading(f, heading, underline);
+
+	BroDocObjList filtered_list;
+
+	while ( it != l.end() )
+		{
+		filtered_list.push_back(*it);
+		it = find_if(++it, l.end(), f_ptr);
+		}
+
+	if ( isShort )
+		WriteBroDocObjTable(f, filtered_list);
+	else
+		WriteBroDocObjList(f, filtered_list);
+	}
+
+void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, bool wantPublic,
+			const char* heading, char underline, bool isShort)
+	{
+	BroDocObjMap::const_iterator it;
+	BroDocObjList l;
+
+	for ( it = m.begin(); it != m.end(); ++it )
+		l.push_back(it->second);
+
+	WriteBroDocObjList(f, l, wantPublic, heading, underline, isShort);
+	}
+
+void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l, const char* heading,
+			char underline)
+	{
+	WriteSectionHeading(f, heading, underline);
+	WriteBroDocObjList(f, l);
+	}
+
+void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjList& l)
+	{
+	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
+		(*it)->WriteReST(f);
+	}
+
+void BroDoc::WriteBroDocObjList(FILE* f, const BroDocObjMap& m, const char* heading,
+			char underline)
+	{
+	BroDocObjMap::const_iterator it;
+	BroDocObjList l;
+
+	for ( it = m.begin(); it != m.end(); ++it )
+		l.push_back(it->second);
+
+	WriteBroDocObjList(f, l, heading, underline);
+	}
+
+void BroDoc::WriteToDoc(FILE* f, const char* format, ...)
+	{
+	va_list argp;
+	va_start(argp, format);
+	vfprintf(f, format, argp);
+	va_end(argp);
+	}
+
+void BroDoc::WriteSectionHeading(FILE* f, const char* heading, char underline)
+	{
+	WriteToDoc(f, "%s\n", heading);
+	WriteRepeatedChar(f, underline, strlen(heading));
+	WriteToDoc(f, "\n");
+	}
+
+void BroDoc::WriteRepeatedChar(FILE* f, char c, size_t n)
+	{
+	for ( size_t i = 0; i < n; ++i )
+		WriteToDoc(f, "%c", c);
+	}
+
+void BroDoc::FreeBroDocObjPtrList(BroDocObjList& l)
+	{
+	for ( BroDocObjList::const_iterator it = l.begin(); it != l.end(); ++it )
+		delete *it;
+
+	l.clear();
+	}
+
+void BroDoc::AddFunction(BroDocObj* o)
+	{
+	BroDocObjMap::const_iterator it = functions.find(o->Name());
+	if ( it == functions.end() )
+		{
+		functions[o->Name()] = o;
+		all.push_back(o);
+		}
+	else
+		functions[o->Name()]->Combine(o);
+	}
+
+static void WritePluginSectionHeading(FILE* f, const plugin::Plugin* p)
+	{
+	string name = p->Name();
+
+	fprintf(f, "%s\n", name.c_str());
+	for ( size_t i = 0; i < name.size(); ++i )
+		fprintf(f, "-");
+	fprintf(f, "\n\n");
+
+	fprintf(f, "%s\n\n", p->Description().c_str());
+	}
+
+static void WriteAnalyzerComponent(FILE* f, const analyzer::Component* c)
+	{
+	EnumType* atag = analyzer_mgr->GetTagEnumType();
+	string tag = fmt("ANALYZER_%s", c->CanonicalName().c_str());
+
+	if ( atag->Lookup("Analyzer", tag.c_str()) < 0 )
+		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
+
+	fprintf(f, ":bro:enum:`Analyzer::%s`\n\n", tag.c_str());
+	}
+
+static void WriteAnalyzerComponent(FILE* f, const file_analysis::Component* c)
+	{
+	EnumType* atag = file_mgr->GetTagEnumType();
+	string tag = fmt("ANALYZER_%s", c->CanonicalName().c_str());
+
+	if ( atag->Lookup("Files", tag.c_str()) < 0 )
+		reporter->InternalError("missing analyzer tag for %s", tag.c_str());
+
+	fprintf(f, ":bro:enum:`Files::%s`\n\n", tag.c_str());
+	}
+
+static void WritePluginComponents(FILE* f, const plugin::Plugin* p)
+	{
+	plugin::Plugin::component_list components = p->Components();
+	plugin::Plugin::component_list::const_iterator it;
+
+	fprintf(f, "Components\n");
+	fprintf(f, "++++++++++\n\n");
+
+	for ( it = components.begin(); it != components.end(); ++it )
+		{
+		switch ( (*it)->Type() ) {
+		case plugin::component::ANALYZER:
+			{
+			const analyzer::Component* c =
+			        dynamic_cast<const analyzer::Component*>(*it);
+
+			if ( c )
+				WriteAnalyzerComponent(f, c);
+			else
+				reporter->InternalError("component type mismatch");
+			}
+			break;
+
+		case plugin::component::FILE_ANALYZER:
+			{
+			const file_analysis::Component* c =
+			        dynamic_cast<const file_analysis::Component*>(*it);
+
+			if ( c )
+				WriteAnalyzerComponent(f, c);
+			else
+				reporter->InternalError("component type mismatch");
+			}
+			break;
+
+		case plugin::component::READER:
+			reporter->InternalError("docs for READER component unimplemented");
+
+		case plugin::component::WRITER:
+			reporter->InternalError("docs for WRITER component unimplemented");
+
+		default:
+			reporter->InternalError("docs for unknown component unimplemented");
+		}
+		}
+	}
+
+static void WritePluginBifItems(FILE* f, const plugin::Plugin* p,
+                                plugin::BifItem::Type t, const string& heading)
+	{
+	plugin::Plugin::bif_item_list bifitems = p->BifItems();
+	plugin::Plugin::bif_item_list::iterator it = bifitems.begin();
+
+	while ( it != bifitems.end() )
+		{
+		if ( it->GetType() != t )
+			it = bifitems.erase(it);
+		else
+			++it;
+		}
+
+	if ( bifitems.empty() )
+		return;
+
+	fprintf(f, "%s\n", heading.c_str());
+	for ( size_t i = 0; i < heading.size(); ++i )
+		fprintf(f, "+");
+	fprintf(f, "\n\n");
+
+	for ( it = bifitems.begin(); it != bifitems.end(); ++it )
+		{
+		BroDocObj* o = doc_ids[it->GetID()];
+
+		if ( o )
+			o->WriteReST(f);
+		else
+			reporter->Warning("No docs for ID: %s\n", it->GetID().c_str());
+		}
+	}
+
+static void WriteAnalyzerTagDefn(FILE* f, EnumType* e, const string& module)
+	{
+	string tag_id= module + "::Tag";
+	e = new CommentedEnumType(e);
+	e->SetTypeID(copy_string(tag_id.c_str()));
+
+	ID* dummy_id = new ID(tag_id.c_str(), SCOPE_GLOBAL, true);
+	dummy_id->SetType(e);
+	dummy_id->MakeType();
+
+	list<string>* r = new list<string>();
+	r->push_back("Unique identifiers for analyzers.");
+
+	BroDocObj bdo(dummy_id, r, true);
+
+	bdo.WriteReST(f);
+	}
+
+static bool ComponentsMatch(const plugin::Plugin* p, plugin::component::Type t,
+                            bool match_empty = false)
+	{
+	plugin::Plugin::component_list components = p->Components();
+	plugin::Plugin::component_list::const_iterator it;
+
+	if ( components.empty() )
+		return match_empty;
+
+	for ( it = components.begin(); it != components.end(); ++it )
+		if ( (*it)->Type() != t )
+			return false;
+
+	return true;
+	}
+
+void CreateProtoAnalyzerDoc(const char* filename)
+	{
+	FILE* f = fopen(filename, "w");
+
+	fprintf(f, "Protocol Analyzers\n");
+	fprintf(f, "==================\n\n\n");
+	fprintf(f, ".. contents::\n");
+	fprintf(f, "     :depth: 1\n\n");
+
+	WriteAnalyzerTagDefn(f, analyzer_mgr->GetTagEnumType(), "Analyzer");
+
+	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
+	plugin::Manager::plugin_list::const_iterator it;
+
+	for ( it = plugins.begin(); it != plugins.end(); ++it )
+		{
+		if ( ! ComponentsMatch(*it, plugin::component::ANALYZER, true) )
+			continue;
+
+		WritePluginSectionHeading(f, *it);
+		WritePluginComponents(f, *it);
+		WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT,
+		                    "Options/Constants");
+		WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals");
+		WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types");
+		WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events");
+		WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions");
+		}
+
+	fclose(f);
+	}
+
+void CreateFileAnalyzerDoc(const char* filename)
+	{
+	FILE* f = fopen(filename, "w");
+
+	fprintf(f, "File Analyzers\n");
+	fprintf(f, "==============\n\n");
+	fprintf(f, ".. contents::\n");
+	fprintf(f, "     :depth: 1\n\n");
+
+	WriteAnalyzerTagDefn(f, file_mgr->GetTagEnumType(), "Files");
+
+	plugin::Manager::plugin_list plugins = plugin_mgr->Plugins();
+	plugin::Manager::plugin_list::const_iterator it;
+
+	for ( it = plugins.begin(); it != plugins.end(); ++it )
+		{
+		if ( ! ComponentsMatch(*it, plugin::component::FILE_ANALYZER) )
+			continue;
+
+		WritePluginSectionHeading(f, *it);
+		WritePluginComponents(f, *it);
+		WritePluginBifItems(f, *it, plugin::BifItem::CONSTANT,
+		                    "Options/Constants");
+		WritePluginBifItems(f, *it, plugin::BifItem::GLOBAL, "Globals");
+		WritePluginBifItems(f, *it, plugin::BifItem::TYPE, "Types");
+		WritePluginBifItems(f, *it, plugin::BifItem::EVENT, "Events");
+		WritePluginBifItems(f, *it, plugin::BifItem::FUNCTION, "Functions");
+		}
+
+	fclose(f);
+	}

src/CMakeLists.txt

@@ -7,7 +7,8 @@ include_directories(BEFORE
 set(bro_ALL_GENERATED_OUTPUTS  CACHE INTERNAL "automatically generated files" FORCE)
 
 # This collects bif inputs that we'll load automatically.
-set(bro_AUTO_BIFS CACHE INTERNAL "BIFs for automatic inclusion" FORCE)
+set(bro_AUTO_BIFS     CACHE INTERNAL "BIFs for automatic inclusion" FORCE)
+set(bro_REGISTER_BIFS CACHE INTERNAL "BIFs for automatic registering" FORCE)
 
 # If TRUE, use CMake's object libraries for sub-directories instead of
 # static libraries. This requires CMake >= 2.8.8.
@@ -166,8 +167,8 @@ if ( NOT bro_HAVE_OBJECT_LIBRARIES )
     foreach (_plugin ${bro_PLUGIN_LIBS})
         string(REGEX REPLACE "plugin-" "" _plugin "${_plugin}")
         string(REGEX REPLACE "-" "_" _plugin "${_plugin}")
-        set(_decl "namespace plugin { namespace ${_plugin} { class Plugin; extern Plugin __plugin; } };")
+        set(_decl "namespace plugin { namespace ${_plugin} { class Plugin; extern Plugin plugin; } };")
-        set(_use "i += (size_t)(&(plugin::${_plugin}::__plugin));")
+        set(_use "i += (size_t)(&(plugin::${_plugin}::plugin));")
         set(__BRO_DECL_PLUGINS "${__BRO_DECL_PLUGINS}${_decl}\n")
         set(__BRO_USE_PLUGINS "${__BRO_USE_PLUGINS}${_use}\n")
     endforeach()
@@ -365,7 +366,6 @@ set(bro_SRCS
     plugin/TaggedComponent.h
     plugin/Manager.cc
     plugin/Plugin.cc
-    plugin/Macros.h
 
     nb_dns.c
     digest.h
@@ -386,17 +386,25 @@ install(TARGETS bro DESTINATION bin)
 set(BRO_EXE bro
     CACHE STRING "Bro executable binary" FORCE)
 
+set(BRO_EXE_PATH ${CMAKE_CURRENT_BINARY_DIR}/bro
+    CACHE STRING "Path to Bro executable binary" FORCE)
+
+# External libmagic project must be built before bro.
+add_dependencies(bro libmagic)
+
 # Target to create all the autogenerated files.
 add_custom_target(generate_outputs_stage1)
 add_dependencies(generate_outputs_stage1 ${bro_ALL_GENERATED_OUTPUTS})
 
 # Target to create the joint includes files that pull in the bif code.
-bro_bif_create_includes(generate_outputs_stage2 ${CMAKE_CURRENT_BINARY_DIR} "${bro_AUTO_BIFS}")
+bro_bif_create_includes(generate_outputs_stage2a ${CMAKE_CURRENT_BINARY_DIR} "${bro_AUTO_BIFS}")
-add_dependencies(generate_outputs_stage2 generate_outputs_stage1)
+bro_bif_create_register(generate_outputs_stage2b ${CMAKE_CURRENT_BINARY_DIR} "${bro_REGISTER_BIFS}")
+add_dependencies(generate_outputs_stage2a generate_outputs_stage1)
+add_dependencies(generate_outputs_stage2b generate_outputs_stage1)
 
 # Global target to trigger creation of autogenerated code.
 add_custom_target(generate_outputs)
-add_dependencies(generate_outputs generate_outputs_stage2)
+add_dependencies(generate_outputs generate_outputs_stage2a generate_outputs_stage2b)
 
 # Build __load__.bro files for standard *.bif.bro.
 bro_bif_create_loader(bif_loader ${CMAKE_BINARY_DIR}/scripts/base/bif)
@@ -411,6 +419,6 @@ add_dependencies(bro bif_loader_plugins)
 # Install *.bif.bro.
 install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)
 
-# Make clean removes the bif directory.
+# Make clean removes the bif and plugin directories.
 set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif)
 

src/DebugLogger.cc

@@ -5,6 +5,7 @@
 
 #include "DebugLogger.h"
 #include "Net.h"
+#include "plugin/Plugin.h"
 
 DebugLogger debug_logger("debug");
 
@@ -74,10 +75,12 @@ void DebugLogger::EnableStreams(const char* s)
 			{
 			if ( strcasecmp("verbose", tok) == 0 )
 				verbose = true;
-			else
+			else if ( strncmp(tok, "plugin-", 7) != 0 )
 				reporter->FatalError("unknown debug stream %s\n", tok);
 			}
 
+		enabled_streams.insert(tok);
+
 		tok = strtok(0, ",");
 		}
 
@@ -106,4 +109,24 @@ void DebugLogger::Log(DebugStream stream, const char* fmt, ...)
 	fflush(file);
 	}
 
+void DebugLogger::Log(const plugin::Plugin& plugin, const char* fmt, ...)
+	{
+	string tok = string("plugin-") + plugin.Name();
+	tok = strreplace(tok, "::", "-");
+
+	if ( enabled_streams.find(tok) == enabled_streams.end() )
+		return;
+
+	fprintf(file, "%17.06f/%17.06f [plugin %s] ",
+			network_time, current_time(true), plugin.Name().c_str());
+
+	va_list ap;
+	va_start(ap, fmt);
+	vfprintf(file, fmt, ap);
+	va_end(ap);
+
+	fputc('\n', file);
+	fflush(file);
+	}
+
 #endif

src/DebugLogger.h

@@ -7,6 +7,8 @@
 #ifdef DEBUG
 
 #include <stdio.h>
+#include <string>
+#include <set>
 
 // To add a new debugging stream, add a constant here as well as
 // an entry to DebugLogger::streams in DebugLogger.cc.
@@ -27,8 +29,8 @@ enum DebugStream {
 	DBG_INPUT,	// Input streams
 	DBG_THREADING,	// Threading system
 	DBG_FILE_ANALYSIS,	// File analysis
-	DBG_PLUGINS,
+	DBG_PLUGINS,	// Plugin system
-	DBG_BROXYGEN,
+	DBG_BROXYGEN,	// Broxygen
 	DBG_PKTIO,	// Packet sources and dumpers.
 
 	NUM_DBGS // Has to be last
@@ -41,6 +43,10 @@ enum DebugStream {
 #define DBG_PUSH(stream) debug_logger.PushIndent(stream)
 #define DBG_POP(stream) debug_logger.PopIndent(stream)
 
+#define PLUGIN_DBG_LOG(plugin, args...) debug_logger.Log(plugin, args)
+
+namespace plugin { class Plugin; }
+
 class DebugLogger {
 public:
 	// Output goes to stderr per default.
@@ -48,6 +54,7 @@ public:
 	~DebugLogger();
 
 	void Log(DebugStream stream, const char* fmt, ...);
+	void Log(const plugin::Plugin& plugin, const char* fmt, ...);
 
 	void PushIndent(DebugStream stream)
 		{ ++streams[int(stream)].indent; }
@@ -78,6 +85,8 @@ private:
 		bool enabled;
 	};
 
+	std::set<std::string> enabled_streams;
+
 	static Stream streams[NUM_DBGS];
 };
 
@@ -88,6 +97,7 @@ extern DebugLogger debug_logger;
 #define DBG_LOG_VERBOSE(args...)
 #define DBG_PUSH(stream)
 #define DBG_POP(stream)
+#define PLUGIN_DBG_LOG(plugin, args...)
 #endif
 
 #endif

src/Event.cc

@@ -6,6 +6,7 @@
 #include "Func.h"
 #include "NetVar.h"
 #include "Trigger.h"
+#include "plugin/Manager.h"
 
 EventMgr mgr;
 
@@ -77,6 +78,8 @@ EventMgr::~EventMgr()
 
 void EventMgr::QueueEvent(Event* event)
 	{
+	PLUGIN_HOOK_VOID(HOOK_QUEUE_EVENT, HookQueueEvent(event));
+
 	if ( ! head )
 		head = tail = event;
 	else
@@ -115,6 +118,8 @@ void EventMgr::Drain()
 
 	SegmentProfiler(segment_logger, "draining-events");
 
+	PLUGIN_HOOK_VOID(HOOK_DRAIN_EVENTS, HookDrainEvents());
+
 	draining = true;
 	while ( head )
 		Dispatch();

src/Event.h

@@ -24,6 +24,8 @@ public:
 	SourceID Source() const		{ return src; }
 	analyzer::ID Analyzer() const	{ return aid; }
 	TimerMgr* Mgr() const		{ return mgr; }
+	EventHandlerPtr Handler() const	{ return handler; }
+	val_list* Args() const	{ return args; }
 
 	void Describe(ODesc* d) const;
 

src/FlowSrc.cc

@@ -48,7 +48,7 @@ void FlowSrc::Process()
 
 	// This is normally done by calling net_packet_dispatch(),
 	// but as we don't have a packet to dispatch ...
-	network_time = next_timestamp;
+    net_update_time(next_timestamp);
 	expire_timers();
 
 	netflow_analyzer->downflow()->set_exporter_ip(exporter_ip);

src/Func.cc

@@ -46,6 +46,7 @@
 #include "Event.h"
 #include "Traverse.h"
 #include "Reporter.h"
+#include "plugin/Manager.h"
 
 extern	RETSIGTYPE sig_handler(int signo);
 
@@ -226,7 +227,7 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_PRE(tc);
 
 	// FIXME: Traverse arguments to builtin functions, too.
-	if ( kind == BRO_FUNC )
+	if ( kind == BRO_FUNC && scope )
 		{
 		tc = scope->Traverse(cb);
 		HANDLE_TC_STMT_PRE(tc);
@@ -281,6 +282,50 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 #ifdef PROFILE_BRO_FUNCTIONS
 	DEBUG_MSG("Function: %s\n", id->Name());
 #endif
+
+	Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
+
+	if ( plugin_result )
+		{
+		// TODO: We should factor this out into its own method.
+		switch ( Flavor() ) {
+		case FUNC_FLAVOR_EVENT:
+			Unref(plugin_result);
+			plugin_result = 0;
+			break;
+
+		case FUNC_FLAVOR_HOOK:
+			if ( plugin_result->Type()->Tag() != TYPE_BOOL )
+				reporter->InternalError("plugin returned non-bool for hook");
+
+			break;
+
+		case FUNC_FLAVOR_FUNCTION:
+			{
+			BroType* yt = FType()->YieldType();
+
+			if ( (! yt) || yt->Tag() == TYPE_VOID )
+				{
+				Unref(plugin_result);
+				plugin_result = 0;
+				}
+
+			else
+				{
+				if ( plugin_result->Type()->Tag() != yt->Tag() )
+					reporter->InternalError("plugin returned wrong type for function call");
+				}
+
+			break;
+			}
+		}
+
+		loop_over_list(*args, i)
+			Unref((*args)[i]);
+
+		return plugin_result;
+		}
+
 	if ( bodies.empty() )
 		{
 		// Can only happen for events and hooks.
@@ -561,6 +606,7 @@ void builtin_error(const char* msg, BroObj* arg)
 #include "strings.bif.func_def"
 
 #include "__all__.bif.cc" // Autogenerated for compiling in the bif_target() code.
+#include "__all__.bif.register.cc" // Autogenerated for compiling in the bif_target() code.
 
 void init_builtin_funcs()
 	{

src/ID.h

@@ -32,9 +32,11 @@ public:
 
 	void SetType(BroType* t)	{ Unref(type); type = t; }
 	BroType* Type()			{ return type; }
+	const BroType* Type() const	{ return type; }
 
 	void MakeType()			{ is_type = 1; }
 	BroType* AsType()		{ return is_type ? Type() : 0; }
+	const BroType* AsType() const	{ return is_type ? Type() : 0; }
 
 	// If weak_ref is false, the Val is assumed to be already ref'ed
 	// and will be deref'ed when the ID is deleted.

src/Net.cc

@@ -33,6 +33,7 @@
 #include "iosource/Manager.h"
 #include "iosource/pktsrc/PktSrc.h"
 #include "iosource/pktsrc/PktDumper.h"
+#include "plugin/Manager.h"
 
 extern "C" {
 #include "setsignal.h"
@@ -149,13 +150,17 @@ RETSIGTYPE watchdog(int /* signo */)
 	return RETSIGVAL;
 	}
 
+void net_update_time(double new_network_time)
+	{
+	network_time = new_network_time;
+	PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time));
+	}
+
 void net_init(name_list& interfaces, name_list& readfiles,
 	      name_list& netflows, name_list& flowfiles,
 	        const char* writefile, const char* filter,
 			const char* secondary_filter, int do_watchdog)
 	{
-	init_net_var();
-
 	if ( readfiles.length() > 0 || flowfiles.length() > 0 )
 		{
 		reading_live = pseudo_realtime > 0.0;
@@ -304,7 +309,7 @@ void net_packet_dispatch(double t, const struct pcap_pkthdr* hdr,
 			: timer_mgr;
 
 	// network_time never goes back.
-	network_time = tmgr->Time() < t ? t : tmgr->Time();
+	net_update_time(tmgr->Time() < t ? t : tmgr->Time());
 
 	current_pktsrc = src_ps;
 	current_iosrc = src_ps;
@@ -437,7 +442,7 @@ void net_run()
 				{
 				// Take advantage of the lull to get up to
 				// date on timers and events.
-				network_time = ct;
+				net_update_time(ct);
 				expire_timers();
 				usleep(1); // Just yield.
 				}
@@ -459,7 +464,7 @@ void net_run()
 			// date on timers and events.  Because we only
 			// have timers as sources, going to sleep here
 			// doesn't risk blocking on other inputs.
-			network_time = current_time();
+			net_update_time(current_time());
 			expire_timers();
 
 			// Avoid busy-waiting - pause for 100 ms.

src/Net.h

@@ -21,6 +21,7 @@ extern void net_run();
 extern void net_get_final_stats();
 extern void net_finish(int drain_events);
 extern void net_delete();	// Reclaim all memory, etc.
+extern void net_update_time(double new_network_time);
 extern void net_packet_arrival(double t, const struct pcap_pkthdr* hdr,
 			const u_char* pkt, int hdr_size,
 			iosource::PktSrc* src_ps);

src/PktSrc.cc

@@ -0,0 +1,796 @@
+// See the file "COPYING" in the main distribution directory for copyright.
+
+#include <errno.h>
+#include <sys/stat.h>
+
+#include "config.h"
+
+#include "util.h"
+#include "PktSrc.h"
+#include "Hash.h"
+#include "Net.h"
+#include "Sessions.h"
+
+
+// ### This needs auto-confing.
+#ifdef HAVE_PCAP_INT_H
+#include <pcap-int.h>
+#endif
+
+PktSrc::PktSrc()
+	{
+	interface = readfile = 0;
+	data = last_data = 0;
+	memset(&hdr, 0, sizeof(hdr));
+	hdr_size = 0;
+	datalink = 0;
+	netmask = 0xffffff00;
+	pd = 0;
+	idle = false;
+
+	next_sync_point = 0;
+	first_timestamp = current_timestamp = next_timestamp = 0.0;
+	first_wallclock = current_wallclock = 0;
+
+	stats.received = stats.dropped = stats.link = 0;
+	}
+
+PktSrc::~PktSrc()
+	{
+	Close();
+
+	loop_over_list(program_list, i)
+		delete program_list[i];
+
+	BPF_Program* code;
+	IterCookie* cookie = filters.InitForIteration();
+	while ( (code = filters.NextEntry(cookie)) )
+		delete code;
+
+	delete [] interface;
+	delete [] readfile;
+	}
+
+void PktSrc::GetFds(int* read, int* write, int* except)
+	{
+	if ( pseudo_realtime )
+		{
+		// Select would give erroneous results. But we simulate it
+		// by setting idle accordingly.
+		idle = CheckPseudoTime() == 0;
+		return;
+		}
+
+	if ( selectable_fd >= 0 )
+		*read = selectable_fd;
+	}
+
+int PktSrc::ExtractNextPacket()
+	{
+	// Don't return any packets if processing is suspended (except for the
+	// very first packet which we need to set up times).
+	if ( net_is_processing_suspended() && first_timestamp )
+		{
+		idle = true;
+		return 0;
+		}
+
+	data = last_data = pcap_next(pd, &hdr);
+
+	if ( data && (hdr.len == 0 || hdr.caplen == 0) )
+		{
+		sessions->Weird("empty_pcap_header", &hdr, data);
+		return 0;
+		}
+
+	if ( data )
+		next_timestamp = hdr.ts.tv_sec + double(hdr.ts.tv_usec) / 1e6;
+
+	if ( pseudo_realtime )
+		current_wallclock = current_time(true);
+
+	if ( ! first_timestamp )
+		first_timestamp = next_timestamp;
+
+	idle = (data == 0);
+
+	if ( data )
+		++stats.received;
+
+	// Source has gone dry.  If it's a network interface, this just means
+	// it's timed out. If it's a file, though, then the file has been
+	// exhausted.
+	if ( ! data && ! IsLive() )
+		{
+		closed = true;
+
+		if ( pseudo_realtime && using_communication )
+			{
+			if ( remote_trace_sync_interval )
+				remote_serializer->SendFinalSyncPoint();
+			else
+				remote_serializer->Terminate();
+			}
+		}
+
+	return data != 0;
+	}
+
+double PktSrc::NextTimestamp(double* local_network_time)
+	{
+	if ( ! data && ! ExtractNextPacket() )
+		return -1.0;
+
+	if ( pseudo_realtime )
+		{
+		// Delay packet if necessary.
+		double packet_time = CheckPseudoTime();
+		if ( packet_time )
+			return packet_time;
+
+		idle = true;
+		return -1.0;
+		}
+
+	return next_timestamp;
+	}
+
+void PktSrc::ContinueAfterSuspend()
+	{
+	current_wallclock = current_time(true);
+	}
+
+double PktSrc::CurrentPacketWallClock()
+	{
+	// We stop time when we are suspended.
+	if ( net_is_processing_suspended() )
+		current_wallclock = current_time(true);
+
+	return current_wallclock;
+	}
+
+double PktSrc::CheckPseudoTime()
+	{
+	if ( ! data && ! ExtractNextPacket() )
+		return 0;
+
+	if ( ! current_timestamp )
+		return bro_start_time;
+
+	if ( remote_trace_sync_interval )
+		{
+		if ( next_sync_point == 0 || next_timestamp >= next_sync_point )
+			{
+			int n = remote_serializer->SendSyncPoint();
+			next_sync_point = first_timestamp +
+						n * remote_trace_sync_interval;
+			remote_serializer->Log(RemoteSerializer::LogInfo,
+				fmt("stopping at packet %.6f, next sync-point at %.6f",
+					current_timestamp, next_sync_point));
+
+			return 0;
+			}
+		}
+
+	double pseudo_time = next_timestamp - first_timestamp;
+	double ct = (current_time(true) - first_wallclock) * pseudo_realtime;
+
+	return pseudo_time <= ct ? bro_start_time + pseudo_time : 0;
+	}
+
+void PktSrc::Process()
+	{
+	if ( ! data && ! ExtractNextPacket() )
+		return;
+
+	current_timestamp = next_timestamp;
+
+	int pkt_hdr_size = hdr_size;
+
+	// Unfortunately some packets on the link might have MPLS labels
+	// while others don't. That means we need to ask the link-layer if
+	// labels are in place.
+	bool have_mpls = false;
+
+	int protocol = 0;
+
+	switch ( datalink ) {
+	case DLT_NULL:
+		{
+		protocol = (data[3] << 24) + (data[2] << 16) + (data[1] << 8) + data[0];
+
+		// From the Wireshark Wiki: "AF_INET6, unfortunately, has
+		// different values in {NetBSD,OpenBSD,BSD/OS},
+		// {FreeBSD,DragonFlyBSD}, and {Darwin/Mac OS X}, so an IPv6
+		// packet might have a link-layer header with 24, 28, or 30
+		// as the AF_ value." As we may be reading traces captured on
+		// platforms other than what we're running on, we accept them
+		// all here.
+		if ( protocol != AF_INET
+		     && protocol != AF_INET6
+		     && protocol != 24
+		     && protocol != 28
+		     && protocol != 30 )
+			{
+			sessions->Weird("non_ip_packet_in_null_transport", &hdr, data);
+			data = 0;
+			return;
+			}
+
+		break;
+		}
+
+	case DLT_EN10MB:
+		{
+		// Get protocol being carried from the ethernet frame.
+		protocol = (data[12] << 8) + data[13];
+
+		switch ( protocol )
+			{
+			// MPLS carried over the ethernet frame.
+			case 0x8847:
+				have_mpls = true;
+				break;
+
+			// VLAN carried over the ethernet frame.
+			case 0x8100:
+				data += get_link_header_size(datalink);
+				data += 4; // Skip the vlan header
+				pkt_hdr_size = 0;
+
+				// Check for 802.1ah (Q-in-Q) containing IP.
+				// Only do a second layer of vlan tag
+				// stripping because there is no
+				// specification that allows for deeper
+				// nesting.
+				if ( ((data[2] << 8) + data[3]) == 0x0800 )
+					data += 4;
+
+				break;
+
+			// PPPoE carried over the ethernet frame.
+			case 0x8864:
+				data += get_link_header_size(datalink);
+				protocol = (data[6] << 8) + data[7];
+				data += 8; // Skip the PPPoE session and PPP header
+				pkt_hdr_size = 0;
+
+				if ( protocol != 0x0021 && protocol != 0x0057 )
+					{
+					// Neither IPv4 nor IPv6.
+					sessions->Weird("non_ip_packet_in_pppoe_encapsulation", &hdr, data);
+					data = 0;
+					return;
+					}
+				break;
+			}
+
+		break;
+		}
+
+	case DLT_PPP_SERIAL:
+		{
+		// Get PPP protocol.
+		protocol = (data[2] << 8) + data[3];
+
+		if ( protocol == 0x0281 )
+			// MPLS Unicast
+			have_mpls = true;
+
+		else if ( protocol != 0x0021 && protocol != 0x0057 )
+			{
+			// Neither IPv4 nor IPv6.
+			sessions->Weird("non_ip_packet_in_ppp_encapsulation", &hdr, data);
+			data = 0;
+			return;
+			}
+		break;
+		}
+	}
+
+	if ( have_mpls )
+		{
+		// Remove the data link layer
+		data += get_link_header_size(datalink);
+
+		// Denote a header size of zero before the IP header
+		pkt_hdr_size = 0;
+
+		// Skip the MPLS label stack.
+		bool end_of_stack = false;
+
+		while ( ! end_of_stack )
+			{
+			end_of_stack = *(data + 2) & 0x01;
+			data += 4;
+			}
+		}
+
+	if ( pseudo_realtime )
+		{
+		current_pseudo = CheckPseudoTime();
+		net_packet_arrival(current_pseudo, &hdr, data, pkt_hdr_size, this);
+		if ( ! first_wallclock )
+			first_wallclock = current_time(true);
+		}
+
+	else
+		net_packet_arrival(current_timestamp, &hdr, data, pkt_hdr_size, this);
+
+	data = 0;
+	}
+
+bool PktSrc::GetCurrentPacket(const struct pcap_pkthdr** arg_hdr,
+				const u_char** arg_pkt)
+	{
+	if ( ! last_data )
+		return false;
+
+	*arg_hdr = &hdr;
+	*arg_pkt = last_data;
+	return true;
+	}
+
+int PktSrc::PrecompileFilter(int index, const char* filter)
+	{
+	// Compile filter.
+	BPF_Program* code = new BPF_Program();
+
+	if ( ! code->Compile(pd, filter, netmask, errbuf, sizeof(errbuf)) )
+		{
+		delete code;
+		return 0;
+		}
+
+	// Store it in hash.
+	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
+	BPF_Program* oldcode = filters.Lookup(hash);
+	if ( oldcode )
+		delete oldcode;
+
+	filters.Insert(hash, code);
+	delete hash;
+
+	return 1;
+	}
+
+int PktSrc::SetFilter(int index)
+	{
+	// We don't want load-level filters for the secondary path.
+	if ( filter_type == TYPE_FILTER_SECONDARY && index > 0 )
+		return 1;
+
+	HashKey* hash = new HashKey(HashKey(bro_int_t(index)));
+	BPF_Program* code = filters.Lookup(hash);
+	delete hash;
+
+	if ( ! code )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			      "No precompiled pcap filter for index %d",
+			      index);
+		return 0;
+		}
+
+	if ( pcap_setfilter(pd, code->GetProgram()) < 0 )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			      "pcap_setfilter(%d): %s",
+			      index, pcap_geterr(pd));
+		return 0;
+		}
+
+#ifndef HAVE_LINUX
+	// Linux doesn't clear counters when resetting filter.
+	stats.received = stats.dropped = stats.link = 0;
+#endif
+
+	return 1;
+	}
+
+void PktSrc::SetHdrSize()
+	{
+	int dl = pcap_datalink(pd);
+	hdr_size = get_link_header_size(dl);
+
+	if ( hdr_size < 0 )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			 "unknown data link type 0x%x", dl);
+		Close();
+		}
+
+	datalink = dl;
+	}
+
+void PktSrc::Close()
+	{
+	if ( pd )
+		{
+		pcap_close(pd);
+		pd = 0;
+		closed = true;
+		}
+	}
+
+void PktSrc::AddSecondaryTablePrograms()
+	{
+	BPF_Program* program;
+
+	loop_over_list(secondary_path->EventTable(), i)
+		{
+		SecondaryEvent* se = secondary_path->EventTable()[i];
+		program = new BPF_Program();
+
+		if ( ! program->Compile(snaplen, datalink, se->Filter(),
+					netmask, errbuf, sizeof(errbuf)) )
+			{
+			delete program;
+			Close();
+			return;
+			}
+
+		SecondaryProgram* sp = new SecondaryProgram(program, se);
+		program_list.append(sp);
+		}
+	}
+
+void PktSrc::Statistics(Stats* s)
+	{
+	if ( reading_traces )
+		s->received = s->dropped = s->link = 0;
+
+	else
+		{
+		struct pcap_stat pstat;
+		if ( pcap_stats(pd, &pstat) < 0 )
+			{
+			reporter->Error("problem getting packet filter statistics: %s",
+					ErrorMsg());
+			s->received = s->dropped = s->link = 0;
+			}
+
+		else
+			{
+			s->dropped = pstat.ps_drop;
+			s->link = pstat.ps_recv;
+			}
+		}
+
+	s->received = stats.received;
+
+	if ( pseudo_realtime )
+		s->dropped = 0;
+
+	stats.dropped = s->dropped;
+	}
+
+PktInterfaceSrc::PktInterfaceSrc(const char* arg_interface, const char* filter,
+					PktSrc_Filter_Type ft)
+: PktSrc()
+	{
+	char tmp_errbuf[PCAP_ERRBUF_SIZE];
+	filter_type = ft;
+
+	// Determine interface if not specified.
+	if ( ! arg_interface && ! (arg_interface = pcap_lookupdev(tmp_errbuf)) )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			 "pcap_lookupdev: %s", tmp_errbuf);
+		return;
+		}
+
+	interface = copy_string(arg_interface);
+
+	// Determine network and netmask.
+	uint32 net;
+	if ( pcap_lookupnet(interface, &net, &netmask, tmp_errbuf) < 0 )
+		{
+		// ### The lookup can fail if no address is assigned to
+		// the interface; and libpcap doesn't have any useful notion
+		// of error codes, just error strings - how bogus - so we
+		// just kludge around the error :-(.
+		// sprintf(errbuf, "pcap_lookupnet %s", tmp_errbuf);
+		// return;
+		net = 0;
+		netmask = 0xffffff00;
+		}
+
+	// We use the smallest time-out possible to return almost immediately if
+	// no packets are available. (We can't use set_nonblocking() as it's
+	// broken on FreeBSD: even when select() indicates that we can read
+	// something, we may get nothing if the store buffer hasn't filled up
+	// yet.)
+	pd = pcap_open_live(interface, snaplen, 1, 1, tmp_errbuf);
+
+	if ( ! pd )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			 "pcap_open_live: %s", tmp_errbuf);
+		closed = true;
+		return;
+		}
+
+	// ### This needs autoconf'ing.
+#ifdef HAVE_PCAP_INT_H
+	reporter->Info("pcap bufsize = %d\n", ((struct pcap *) pd)->bufsize);
+#endif
+
+#ifdef HAVE_LINUX
+	if ( pcap_setnonblock(pd, 1, tmp_errbuf) < 0 )
+		{
+		safe_snprintf(errbuf, sizeof(errbuf),
+			 "pcap_setnonblock: %s", tmp_errbuf);
+		pcap_close(pd);
+		closed = true;
+		return;
+		}
+#endif
+	selectable_fd = pcap_fileno(pd);
+
+	if ( PrecompileFilter(0, filter) && SetFilter(0) )
+		{
+		SetHdrSize();
+
+		if ( closed )
+			// Couldn't get header size.
+			return;
+
+		reporter->Info("listening on %s, capture length %d bytes\n", interface, snaplen);
+		}
+	else
+		closed = true;
+	}
+
+
+PktFileSrc::PktFileSrc(const char* arg_readfile, const char* filter,
+			PktSrc_Filter_Type ft)
+: PktSrc()
+	{
+	readfile = copy_string(arg_readfile);
+
+	filter_type = ft;
+
+	pd = pcap_open_offline((char*) readfile, errbuf);
+
+	if ( pd && PrecompileFilter(0, filter) && SetFilter(0) )
+		{
+		SetHdrSize();
+
+		if ( closed )
+			// Unknown link layer type.
+			return;
+
+		// We don't put file sources into non-blocking mode as
+		// otherwise we would not be able to identify the EOF.
+
+		selectable_fd = fileno(pcap_file(pd));
+
+		if ( selectable_fd < 0 )
+			reporter->InternalError("OS does not support selectable pcap fd");
+		}
+	else
+		closed = true;
+	}
+
+
+SecondaryPath::SecondaryPath()
+	{
+	filter = 0;
+
+	// Glue together the secondary filter, if exists.
+	Val* secondary_fv = internal_val("secondary_filters");
+	if ( secondary_fv->AsTableVal()->Size() == 0 )
+		return;
+
+	int did_first = 0;
+	const TableEntryValPDict* v = secondary_fv->AsTable();
+	IterCookie* c = v->InitForIteration();
+	TableEntryVal* tv;
+	HashKey* h;
+
+	while ( (tv = v->NextEntry(h, c)) )
+		{
+		// Get the index values.
+		ListVal* index =
+			secondary_fv->AsTableVal()->RecoverIndex(h);
+
+		const char* str =
+			index->Index(0)->Ref()->AsString()->CheckString();
+
+		if ( ++did_first == 1 )
+			{
+			filter = copy_string(str);
+			}
+		else
+			{
+			if ( strlen(filter) > 0 )
+				{
+				char* tmp_f = new char[strlen(str) + strlen(filter) + 32];
+				if ( strlen(str) == 0 )
+					sprintf(tmp_f, "%s", filter);
+				else
+					sprintf(tmp_f, "(%s) or (%s)", filter, str);
+				delete [] filter;
+				filter = tmp_f;
+				}
+			}
+
+		// Build secondary_path event table item and link it.
+		SecondaryEvent* se =
+			new SecondaryEvent(index->Index(0)->Ref()->AsString()->CheckString(),
+				tv->Value()->AsFunc() );
+
+		event_list.append(se);
+
+		delete h;
+		Unref(index);
+		}
+	}
+
+SecondaryPath::~SecondaryPath()
+	{
+	loop_over_list(event_list, i)
+		delete event_list[i];
+
+	delete [] filter;
+	}
+
+
+SecondaryProgram::~SecondaryProgram()
+	{
+	delete program;
+	}
+
+PktDumper::PktDumper(const char* arg_filename, bool arg_append)
+	{
+	filename[0] = '\0';
+	is_error = false;
+	append = arg_append;
+	dumper = 0;
+	open_time = 0.0;
+
+	// We need a pcap_t with a reasonable link-layer type. We try to get it
+	// from the packet sources. If not available, we fall back to Ethernet.
+	// FIXME: Perhaps we should make this configurable?
+	int linktype = -1;
+
+	if ( pkt_srcs.length() )
+		linktype = pkt_srcs[0]->LinkType();
+
+	if ( linktype < 0 )
+		linktype = DLT_EN10MB;
+
+	pd = pcap_open_dead(linktype, snaplen);
+	if ( ! pd )
+		{
+		Error("error for pcap_open_dead");
+		return;
+		}
+
+	if ( arg_filename )
+		Open(arg_filename);
+	}
+
+bool PktDumper::Open(const char* arg_filename)
+	{
+	if ( ! arg_filename && ! *filename )
+		{
+		Error("no filename given");
+		return false;
+		}
+
+	if ( arg_filename )
+		{
+		if ( dumper && streq(arg_filename, filename) )
+			// Already open.
+			return true;
+
+		safe_strncpy(filename, arg_filename, FNBUF_LEN);
+		}
+
+	if ( dumper )
+		Close();
+
+	struct stat s;
+	int exists = 0;
+
+	if ( append )
+		{
+		// See if output file already exists (and is non-empty).
+		exists = stat(filename, &s); ;
+
+		if ( exists < 0 && errno != ENOENT )
+			{
+			Error(fmt("can't stat file %s: %s", filename, strerror(errno)));
+			return false;
+			}
+		}
+
+	if ( ! append || exists < 0 || s.st_size == 0 )
+		{
+		// Open new file.
+		dumper =  pcap_dump_open(pd, filename);
+		if ( ! dumper )
+			{
+			Error(pcap_geterr(pd));
+			return false;
+			}
+		}
+
+	else
+		{
+		// Old file and we need to append, which, unfortunately,
+		// is not supported by libpcap. So, we have to hack a
+		// little bit, knowing that pcap_dumpter_t is, in fact,
+		// a FILE ... :-(
+		dumper = (pcap_dumper_t*) fopen(filename, "a");
+		if ( ! dumper )
+			{
+			Error(fmt("can't open dump %s: %s", filename, strerror(errno)));
+			return false;
+			}
+		}
+
+	open_time = network_time;
+	is_error = false;
+	return true;
+	}
+
+bool PktDumper::Close()
+	{
+	if ( dumper )
+		{
+		pcap_dump_close(dumper);
+		dumper = 0;
+		is_error = false;
+		}
+
+	return true;
+	}
+
+bool PktDumper::Dump(const struct pcap_pkthdr* hdr, const u_char* pkt)
+	{
+	if ( ! dumper )
+		return false;
+
+	if ( ! open_time )
+		open_time = network_time;
+
+	pcap_dump((u_char*) dumper, hdr, pkt);
+
+	return true;
+	}
+
+void PktDumper::Error(const char* errstr)
+	{
+	safe_strncpy(errbuf, errstr, sizeof(errbuf));
+	is_error = true;
+	}
+
+int get_link_header_size(int dl)
+	{
+	switch ( dl ) {
+	case DLT_NULL:
+		return 4;
+
+	case DLT_EN10MB:
+		return 14;
+
+	case DLT_FDDI:
+		return 13 + 8;	// fddi_header + LLC
+
+#ifdef DLT_LINUX_SLL
+	case DLT_LINUX_SLL:
+		return 16;
+#endif
+
+	case DLT_PPP_SERIAL:	// PPP_SERIAL
+		return 4;
+
+	case DLT_RAW:
+		return 0;
+	}
+
+	return -1;
+	}

src/RemoteSerializer.cc

@@ -1455,7 +1455,7 @@ void RemoteSerializer::Process()
 		// FIXME: The following chunk of code is copied from
 		// net_packet_dispatch().  We should change that function
 		// to accept an IOSource instead of the PktSrc.
-		network_time = p->time;
+		net_update_time(p->time);
 
 		SegmentProfiler(segment_logger, "expiring-timers");
 		TimerMgr* tmgr = sessions->LookupTimerMgr(GetCurrentTag());

src/RuleAction.cc

@@ -60,11 +60,11 @@ RuleActionAnalyzer::RuleActionAnalyzer(const char* arg_analyzer)
 void RuleActionAnalyzer::PrintDebug()
 	{
 	if ( ! child_analyzer )
-		fprintf(stderr, "|%s|\n", analyzer_mgr->GetComponentName(analyzer));
+		fprintf(stderr, "|%s|\n", analyzer_mgr->GetComponentName(analyzer).c_str());
 	else
 		fprintf(stderr, "|%s:%s|\n",
-			analyzer_mgr->GetComponentName(analyzer),
+			analyzer_mgr->GetComponentName(analyzer).c_str(),
-			analyzer_mgr->GetComponentName(child_analyzer));
+			analyzer_mgr->GetComponentName(child_analyzer).c_str());
 	}
 
 

src/Sessions.cc

@@ -382,6 +382,31 @@ int NetSessions::CheckConnectionTag(Connection* conn)
 	return 1;
 	}
 
+static unsigned int gre_header_len(uint16 flags)
+	{
+	unsigned int len = 4;  // Always has 2 byte flags and 2 byte protocol type.
+
+	if ( flags & 0x8000 )
+		// Checksum/Reserved1 present.
+		len += 4;
+
+	// Not considering routing presence bit since it's deprecated ...
+
+	if ( flags & 0x2000 )
+		// Key present.
+		len += 4;
+
+	if ( flags & 0x1000 )
+		// Sequence present.
+		len += 4;
+
+	if ( flags & 0x0080 )
+		// Acknowledgement present.
+		len += 4;
+
+	return len;
+	}
+
 void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 				const IP_Hdr* ip_hdr, const u_char* const pkt,
 				int hdr_size, const EncapsulationStack* encapsulation)
@@ -390,6 +415,15 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	const struct ip* ip4 = ip_hdr->IP4_Hdr();
 
 	uint32 len = ip_hdr->TotalLen();
+	if ( len == 0 )
+		{
+		// TCP segmentation offloading can zero out the ip_len field.
+		Weird("ip_hdr_len_zero", hdr, pkt, encapsulation);
+
+		// Cope with the zero'd out ip_len field by using the caplen.
+		len = hdr->caplen - hdr_size;
+		}
+
 	if ( hdr->len < len + hdr_size )
 		{
 		Weird("truncated_IP", hdr, pkt, encapsulation);
@@ -443,6 +477,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			}
 		}
 
+	FragReassemblerTracker frt(this, f);
+
 	len -= ip_hdr_len;	// remove IP header
 	caplen -= ip_hdr_len;
 
@@ -457,7 +493,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 			vl->append(ip_hdr->BuildPktHdrVal());
 			mgr.QueueEvent(esp_packet, vl);
 			}
-		Remove(f);
+
 		// Can't do more since upper-layer payloads are going to be encrypted.
 		return;
 		}
@@ -472,7 +508,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( ! ignore_checksums && mobility_header_checksum(ip_hdr) != 0xffff )
 			{
 			Weird("bad_MH_checksum", hdr, pkt, encapsulation);
-			Remove(f);
 			return;
 			}
 
@@ -486,7 +521,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( ip_hdr->NextProto() != IPPROTO_NONE )
 			Weird("mobility_piggyback", hdr, pkt, encapsulation);
 
-		Remove(f);
 		return;
 		}
 #endif
@@ -494,10 +528,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	int proto = ip_hdr->NextProto();
 
 	if ( CheckHeaderTrunc(proto, len, caplen, hdr, pkt, encapsulation) )
-		{
-		Remove(f);
 		return;
-		}
 
 	const u_char* data = ip_hdr->Payload();
 
@@ -559,13 +590,100 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		break;
 		}
 
+	case IPPROTO_GRE:
+		{
+		if ( ! BifConst::Tunnel::enable_gre )
+			{
+			Weird("GRE_tunnel", ip_hdr, encapsulation);
+			return;
+			}
+
+		uint16 flags_ver = ntohs(*((uint16*)(data + 0)));
+		uint16 proto_typ = ntohs(*((uint16*)(data + 2)));
+		int gre_version = flags_ver & 0x0007;
+
+		if ( gre_version != 0 && gre_version != 1 )
+			{
+			Weird(fmt("unknown_gre_version_%d", gre_version), ip_hdr,
+			      encapsulation);
+			return;
+			}
+
+		if ( gre_version == 0 )
+			{
+			if ( proto_typ != 0x0800 && proto_typ != 0x86dd )
+				{
+				// Not IPv4/IPv6 payload.
+				Weird(fmt("unknown_gre_protocol_%"PRIu16, proto_typ), ip_hdr,
+				      encapsulation);
+				return;
+				}
+
+			proto = (proto_typ == 0x0800) ? IPPROTO_IPV4 : IPPROTO_IPV6;
+			}
+
+		else // gre_version == 1
+			{
+			if ( proto_typ != 0x880b )
+				{
+				// Enhanced GRE payload must be PPP.
+				Weird("egre_protocol_type", ip_hdr, encapsulation);
+				return;
+				}
+			}
+
+		if ( flags_ver & 0x4000 )
+			{
+			// RFC 2784 deprecates the variable length routing field
+			// specified by RFC 1701. It could be parsed here, but easiest
+			// to just skip for now.
+			Weird("gre_routing", ip_hdr, encapsulation);
+			return;
+			}
+
+		if ( flags_ver & 0x0078 )
+			{
+			// Expect last 4 bits of flags are reserved, undefined.
+			Weird("unknown_gre_flags", ip_hdr, encapsulation);
+			return;
+			}
+
+		unsigned int gre_len = gre_header_len(flags_ver);
+		unsigned int ppp_len = gre_version == 1 ? 1 : 0;
+
+		if ( len < gre_len + ppp_len || caplen < gre_len + ppp_len )
+			{
+			Weird("truncated_GRE", ip_hdr, encapsulation);
+			return;
+			}
+
+		if ( gre_version == 1 )
+			{
+			int ppp_proto = *((uint8*)(data + gre_len));
+
+			if ( ppp_proto != 0x0021 && ppp_proto != 0x0057 )
+				{
+				Weird("non_ip_packet_in_egre", ip_hdr, encapsulation);
+				return;
+				}
+
+			proto = (ppp_proto == 0x0021) ? IPPROTO_IPV4 : IPPROTO_IPV6;
+			}
+
+		data += gre_len + ppp_len;
+		len -= gre_len + ppp_len;
+		caplen -= gre_len + ppp_len;
+
+		// Treat GRE tunnel like IP tunnels, fallthrough to logic below now
+		// that GRE header is stripped and only payload packet remains.
+		}
+
 	case IPPROTO_IPV4:
 	case IPPROTO_IPV6:
 		{
 		if ( ! BifConst::Tunnel::enable_ip )
 			{
 			Weird("IP_tunnel", ip_hdr, encapsulation);
-			Remove(f);
 			return;
 			}
 
@@ -573,7 +691,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		     encapsulation->Depth() >= BifConst::Tunnel::max_depth )
 			{
 			Weird("exceeded_tunnel_max_depth", ip_hdr, encapsulation);
-			Remove(f);
 			return;
 			}
 
@@ -590,7 +707,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( result != 0 )
 			{
 			delete inner;
-			Remove(f);
 			return;
 			}
 
@@ -617,7 +733,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		DoNextInnerPacket(t, hdr, inner, encapsulation,
 		                  ip_tunnels[tunnel_idx].first);
 
-		Remove(f);
 		return;
 		}
 
@@ -630,13 +745,11 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		     encapsulation->LastType() == BifEnum::Tunnel::TEREDO ) )
 			Weird("ipv6_no_next", hdr, pkt);
 
-		Remove(f);
 		return;
 		}
 
 	default:
 		Weird(fmt("unknown_protocol_%d", proto), hdr, pkt, encapsulation);
-		Remove(f);
 		return;
 	}
 
@@ -662,7 +775,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		if ( consistent < 0 )
 			{
 			delete h;
-			Remove(f);
 			return;
 			}
 
@@ -686,7 +798,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	if ( ! conn )
 		{
 		delete h;
-		Remove(f);
 		return;
 		}
 
@@ -718,7 +829,6 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 		{
 		// Above we already recorded the fragment in its entirety.
 		f->DeleteTimer();
-		Remove(f);
 		}
 
 	else if ( record_packet )
@@ -819,6 +929,9 @@ bool NetSessions::CheckHeaderTrunc(int proto, uint32 len, uint32 caplen,
 	case IPPROTO_NONE:
 		min_hdr_len = 0;
 		break;
+	case IPPROTO_GRE:
+		min_hdr_len = 4;
+		break;
 	case IPPROTO_ICMP:
 	case IPPROTO_ICMPV6:
 	default:

src/Sessions.h

@@ -286,6 +286,21 @@ protected:
 	NetSessions::IPPair tunnel_idx;
 };
 
+
+class FragReassemblerTracker {
+public:
+	FragReassemblerTracker(NetSessions* s, FragReassembler* f)
+		: net_sessions(s), frag_reassembler(f)
+		{ }
+
+	~FragReassemblerTracker()
+		{ net_sessions->Remove(frag_reassembler); }
+
+private:
+	NetSessions* net_sessions;
+	FragReassembler* frag_reassembler;
+};
+
 // Manager for the currently active sessions.
 extern NetSessions* sessions;
 

src/Type.cc

@@ -449,6 +449,11 @@ BroType* IndexType::YieldType()
 	return yield_type;
 	}
 
+const BroType* IndexType::YieldType() const
+	{
+	return yield_type;
+	}
+
 void IndexType::Describe(ODesc* d) const
 	{
 	BroType::Describe(d);
@@ -742,6 +747,11 @@ BroType* FuncType::YieldType()
 	return yield;
 	}
 
+const BroType* FuncType::YieldType() const
+	{
+	return yield;
+	}
+
 int FuncType::MatchesIndex(ListExpr*& index) const
 	{
 	return check_and_promote_args(index, args) ?
@@ -1472,9 +1482,9 @@ void EnumType::AddNameInternal(const string& module_name, const char* name,
 	names[copy_string(fullname.c_str())] = val;
 	}
 
-bro_int_t EnumType::Lookup(const string& module_name, const char* name)
+bro_int_t EnumType::Lookup(const string& module_name, const char* name) const
 	{
-	NameMap::iterator pos =
+	NameMap::const_iterator pos =
 		names.find(make_full_var_name(module_name.c_str(), name).c_str());
 
 	if ( pos == names.end() )
@@ -1483,9 +1493,9 @@ bro_int_t EnumType::Lookup(const string& module_name, const char* name)
 		return pos->second;
 	}
 
-const char* EnumType::Lookup(bro_int_t value)
+const char* EnumType::Lookup(bro_int_t value) const
 	{
-	for ( NameMap::iterator iter = names.begin();
+	for ( NameMap::const_iterator iter = names.begin();
 	      iter != names.end(); ++iter )
 		if ( iter->second == value )
 			return iter->first;
@@ -1493,6 +1503,16 @@ const char* EnumType::Lookup(bro_int_t value)
 	return 0;
 	}
 
+EnumType::enum_name_list EnumType::Names() const
+	{
+	enum_name_list n;
+	for ( NameMap::const_iterator iter = names.begin();
+	      iter != names.end(); ++iter )
+		n.push_back(std::make_pair(iter->first, iter->second));
+
+	return n;
+	}
+
 void EnumType::DescribeReST(ODesc* d, bool roles_only) const
 	{
 	d->Add(":bro:type:`enum`");

src/Type.h

@@ -321,6 +321,7 @@ public:
 	TypeList* Indices() const		{ return indices; }
 	const type_list* IndexTypes() const	{ return indices->Types(); }
 	BroType* YieldType();
+	const BroType* YieldType() const;
 
 	void Describe(ODesc* d) const;
 	void DescribeReST(ODesc* d, bool roles_only = false) const;
@@ -383,6 +384,7 @@ public:
 
 	RecordType* Args() const	{ return args; }
 	BroType* YieldType();
+	const BroType* YieldType() const;
 	void SetYieldType(BroType* arg_yield)	{ yield = arg_yield; }
 	function_flavor Flavor() const { return flavor; }
 	string FlavorString() const;
@@ -529,6 +531,8 @@ protected:
 
 class EnumType : public BroType {
 public:
+	typedef std::list<std::pair<string, bro_int_t> > enum_name_list;
+
 	EnumType() : BroType(TYPE_ENUM) { counter = 0; }
 	~EnumType();
 
@@ -542,8 +546,12 @@ public:
 	void AddName(const string& module_name, const char* name, bro_int_t val, bool is_export);
 
 	// -1 indicates not found.
-	bro_int_t Lookup(const string& module_name, const char* name);
+	bro_int_t Lookup(const string& module_name, const char* name) const;
-	const char* Lookup(bro_int_t value); // Returns 0 if not found
+	const char* Lookup(bro_int_t value) const; // Returns 0 if not found
+
+	// Returns the list of defined names with their values. The names
+	// will be fully qualified with their module name.
+	enum_name_list Names() const;
 
 	void DescribeReST(ODesc* d, bool roles_only = false) const;
 
@@ -573,6 +581,7 @@ public:
 	VectorType(BroType* t);
 	virtual ~VectorType();
 	BroType* YieldType()	{ return yield_type; }
+	const BroType* YieldType() const	{ return yield_type; }
 
 	int MatchesIndex(ListExpr*& index) const;
 

src/analyzer/Analyzer.cc

@@ -75,7 +75,7 @@ analyzer::ID Analyzer::id_counter = 0;
 const char* Analyzer::GetAnalyzerName() const
 	{
 	assert(tag);
-	return analyzer_mgr->GetComponentName(tag);
+	return analyzer_mgr->GetComponentName(tag).c_str();
 	}
 
 void Analyzer::SetAnalyzerTag(const Tag& arg_tag)
@@ -87,7 +87,7 @@ void Analyzer::SetAnalyzerTag(const Tag& arg_tag)
 bool Analyzer::IsAnalyzer(const char* name)
 	{
 	assert(tag);
-	return strcmp(analyzer_mgr->GetComponentName(tag), name) == 0;
+	return strcmp(analyzer_mgr->GetComponentName(tag).c_str(), name) == 0;
 	}
 
 // Used in debugging output.
@@ -644,12 +644,12 @@ void Analyzer::FlipRoles()
 	resp_supporters = tmp;
 	}
 
-void Analyzer::ProtocolConfirmation()
+void Analyzer::ProtocolConfirmation(Tag arg_tag)
 	{
 	if ( protocol_confirmed )
 		return;
 
-	EnumVal* tval = tag.AsEnumVal();
+	EnumVal* tval = arg_tag ? arg_tag.AsEnumVal() : tag.AsEnumVal();
 	Ref(tval);
 
 	val_list* vl = new val_list;

src/analyzer/Analyzer.h

@@ -97,8 +97,8 @@ public:
 
 	/**
 	 * Constructor. As this version of the constructor does not receive a
-	 * name or tag, setTag() must be called before the instance can be
+	 * name or tag, SetAnalyzerTag() must be called before the instance
-	 * used.
+	 * can be used.
 	 *
 	 * @param conn The connection the analyzer is associated with.
 	 */
@@ -471,8 +471,11 @@ public:
 	 * may turn into \c protocol_confirmed event at the script-layer (but
 	 * only once per analyzer for each connection, even if the method is
 	 * called multiple times).
+	 *
+	 * If tag is given, it overrides the analyzer tag passed to the
+	 * scripting layer; the default is the one of the analyzer itself.
 	 */
-	 virtual void ProtocolConfirmation();
+	virtual void ProtocolConfirmation(Tag tag = Tag());
 
 	/**
 	 * Signals Bro's protocol detection that the analyzer has found a

src/analyzer/Component.cc

@@ -8,40 +8,24 @@
 
 using namespace analyzer;
 
-Component::Component(const char* arg_name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial)
+Component::Component(const std::string& name, factory_callback arg_factory, Tag::subtype_t arg_subtype, bool arg_enabled, bool arg_partial)
-	: plugin::Component(plugin::component::ANALYZER),
+	: plugin::Component(plugin::component::ANALYZER, name),
 	  plugin::TaggedComponent<analyzer::Tag>(arg_subtype)
 	{
-	name = copy_string(arg_name);
+	canon_name = canonify_name(name);
-	canon_name = canonify_name(arg_name);
 	factory = arg_factory;
 	enabled = arg_enabled;
 	partial = arg_partial;
-	}
 
-Component::Component(const Component& other)
+	analyzer_mgr->RegisterComponent(this, "ANALYZER_");
-	: plugin::Component(Type()),
-	  plugin::TaggedComponent<analyzer::Tag>(other)
-	{
-	name = copy_string(other.name);
-	canon_name = copy_string(other.canon_name);
-	factory = other.factory;
-	enabled = other.enabled;
-	partial = other.partial;
 	}
 
 Component::~Component()
 	{
-	delete [] name;
-	delete [] canon_name;
 	}
 
-void Component::Describe(ODesc* d) const
+void Component::DoDescribe(ODesc* d) const
 	{
-	plugin::Component::Describe(d);
-	d->Add(name);
-	d->Add(" (");
-
 	if ( factory )
 		{
 		d->Add("ANALYZER_");
@@ -50,20 +34,4 @@ void Component::Describe(ODesc* d) const
 		}
 
 	d->Add(enabled ? "enabled" : "disabled");
-	d->Add(")");
-	}
-
-Component& Component::operator=(const Component& other)
-	{
-	plugin::TaggedComponent<analyzer::Tag>::operator=(other);
-
-	if ( &other != this )
-		{
-		name = copy_string(other.name);
-		factory = other.factory;
-		enabled = other.enabled;
-		partial = other.partial;
-		}
-
-	return *this;
 	}

src/analyzer/Component.h

@@ -1,7 +1,7 @@
 // See the file "COPYING" in the main distribution directory for copyright.
 
-#ifndef ANALYZER_PLUGIN_COMPONENT_H
+#ifndef ANALYZER_COMPONENT_H
-#define ANALYZER_PLUGIN_COMPONENT_H
+#define ANALYZER_COMPONENT_H
 
 #include "Tag.h"
 #include "plugin/Component.h"
@@ -56,33 +56,20 @@ public:
 	 * connections has generally not seen much testing yet as virtually
 	 * no existing analyzer supports it.
 	 */
-	Component(const char* name, factory_callback factory, Tag::subtype_t subtype = 0, bool enabled = true, bool partial = false);
+	Component(const std::string& name, factory_callback factory, Tag::subtype_t subtype = 0, bool enabled = true, bool partial = false);
-
-	/**
-	 * Copy constructor.
-	 */
-	Component(const Component& other);
 
 	/**
 	 * Destructor.
 	 */
 	~Component();
 
-	/**
-	 * Returns the name of the analyzer. This name is unique across all
-	 * analyzers and used to identify it. The returned name is derived
-	 * from what's passed to the constructor but upper-cased and
-	 * canonified to allow being part of a script-level ID.
-	 */
-	virtual const char* Name() const	{ return name; }
-
 	/**
 	 * Returns a canonocalized version of the analyzer's name.  The
 	 * returned name is derived from what's passed to the constructor but
 	 * upper-cased and transformed to allow being part of a script-level
 	 * ID.
 	 */
-	const char* CanonicalName() const	{ return canon_name; }
+	const std::string& CanonicalName() const	{ return canon_name; }
 
 	/**
 	 * Returns the analyzer's factory function.
@@ -110,17 +97,14 @@ public:
 	 */
 	void SetEnabled(bool arg_enabled)	{ enabled = arg_enabled; }
 
+protected:
 	/**
-	 * Generates a human-readable description of the component's main
+	  * Overriden from plugin::Component.
-	 * parameters. This goes into the output of \c "bro -NN".
+	  */
-	 */
+	virtual void DoDescribe(ODesc* d) const;
-	virtual void Describe(ODesc* d) const;
-
-	Component& operator=(const Component& other);
 
 private:
-	const char* name;	// The analyzer's name.
+	std::string canon_name;	// The analyzer's canonical name.
-	const char* canon_name;	// The analyzer's canonical name.
 	factory_callback factory;	// The analyzer's factory callback.
 	bool partial;	// True if the analyzer supports partial connections.
 	bool enabled;	// True if the analyzer is enabled.

src/analyzer/Manager.cc

@@ -86,11 +86,6 @@ Manager::~Manager()
 
 void Manager::InitPreScript()
 	{
-	std::list<Component*> analyzers = plugin_mgr->Components<Component>();
-
-	for ( std::list<Component*>::const_iterator i = analyzers.begin(); i != analyzers.end(); i++ )
-		RegisterComponent(*i, "ANALYZER_");
-
 	// Cache these tags.
 	analyzer_backdoor = GetComponentTag("BACKDOOR");
 	analyzer_connsize = GetComponentTag("CONNSIZE");
@@ -109,7 +104,8 @@ void Manager::DumpDebug()
 	DBG_LOG(DBG_ANALYZER, "Available analyzers after bro_init():");
 	list<Component*> all_analyzers = GetComponents();
 	for ( list<Component*>::const_iterator i = all_analyzers.begin(); i != all_analyzers.end(); ++i )
-		DBG_LOG(DBG_ANALYZER, "    %s (%s)", (*i)->Name(), IsEnabled((*i)->Tag()) ? "enabled" : "disabled");
+		DBG_LOG(DBG_ANALYZER, "    %s (%s)", (*i)->Name().c_str(),
+			IsEnabled((*i)->Tag()) ? "enabled" : "disabled");
 
 	DBG_LOG(DBG_ANALYZER, "");
 	DBG_LOG(DBG_ANALYZER, "Analyzers by port:");
@@ -148,7 +144,7 @@ bool Manager::EnableAnalyzer(Tag tag)
 	if ( ! p  )
 		return false;
 
-	DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name());
+	DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name().c_str());
 	p->SetEnabled(true);
 
 	return true;
@@ -161,7 +157,7 @@ bool Manager::EnableAnalyzer(EnumVal* val)
 	if ( ! p  )
 		return false;
 
-	DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name());
+	DBG_LOG(DBG_ANALYZER, "Enabling analyzer %s", p->Name().c_str());
 	p->SetEnabled(true);
 
 	return true;
@@ -174,7 +170,7 @@ bool Manager::DisableAnalyzer(Tag tag)
 	if ( ! p  )
 		return false;
 
-	DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name());
+	DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name().c_str());
 	p->SetEnabled(false);
 
 	return true;
@@ -187,7 +183,7 @@ bool Manager::DisableAnalyzer(EnumVal* val)
 	if ( ! p  )
 		return false;
 
-	DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name());
+	DBG_LOG(DBG_ANALYZER, "Disabling analyzer %s", p->Name().c_str());
 	p->SetEnabled(false);
 
 	return true;
@@ -202,6 +198,11 @@ void Manager::DisableAllAnalyzers()
 		(*i)->SetEnabled(false);
 	}
 
+analyzer::Tag Manager::GetAnalyzerTag(const char* name)
+	{
+	return GetComponentTag(name);
+	}
+
 bool Manager::IsEnabled(Tag tag)
 	{
 	if ( ! tag )
@@ -254,7 +255,7 @@ bool Manager::RegisterAnalyzerForPort(Tag tag, TransportProto proto, uint32 port
 		return false;
 
 #ifdef DEBUG
-	const char* name = GetComponentName(tag);
+	const char* name = GetComponentName(tag).c_str();
 	DBG_LOG(DBG_ANALYZER, "Registering analyzer %s for port %" PRIu32 "/%d", name, port, proto);
 #endif
 
@@ -270,7 +271,7 @@ bool Manager::UnregisterAnalyzerForPort(Tag tag, TransportProto proto, uint32 po
 		return true;  // still a "successful" unregistration
 
 #ifdef DEBUG
-	const char* name = GetComponentName(tag);
+	const char* name = GetComponentName(tag).c_str();
 	DBG_LOG(DBG_ANALYZER, "Unregistering analyzer %s for port %" PRIu32 "/%d", name, port, proto);
 #endif
 
@@ -293,7 +294,8 @@ Analyzer* Manager::InstantiateAnalyzer(Tag tag, Connection* conn)
 
 	if ( ! c->Factory() )
 		{
-		reporter->InternalWarning("analyzer %s cannot be instantiated dynamically", GetComponentName(tag));
+		reporter->InternalWarning("analyzer %s cannot be instantiated dynamically",
+					  GetComponentName(tag).c_str());
 		return 0;
 		}
 
@@ -403,7 +405,7 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn)
 			root->AddChildAnalyzer(analyzer, false);
 
 			DBG_ANALYZER_ARGS(conn, "activated %s analyzer as scheduled",
-					  analyzer_mgr->GetComponentName(*i));
+					  analyzer_mgr->GetComponentName(*i).c_str());
 			}
 
 		}
@@ -429,7 +431,7 @@ bool Manager::BuildInitialAnalyzerTree(Connection* conn)
 
 					root->AddChildAnalyzer(analyzer, false);
 					DBG_ANALYZER_ARGS(conn, "activated %s analyzer due to port %d",
-							  analyzer_mgr->GetComponentName(*j), resp_port);
+							  analyzer_mgr->GetComponentName(*j).c_str(), resp_port);
 					}
 				}
 			}
@@ -555,7 +557,7 @@ void Manager::ExpireScheduledAnalyzers()
 			conns.erase(i);
 
 			DBG_LOG(DBG_ANALYZER, "Expiring expected analyzer %s for connection %s",
-				analyzer_mgr->GetComponentName(a->analyzer),
+				analyzer_mgr->GetComponentName(a->analyzer).c_str(),
 				fmt_conn_id(a->conn.orig, 0, a->conn.resp, a->conn.resp_p));
 
 			delete a;

src/analyzer/Manager.h

@@ -133,6 +133,14 @@ public:
 	 */
 	void DisableAllAnalyzers();
 
+	/**
+	 * Returns the tag associated with an analyer name, or the tag
+	 * associated with an error if no such analyzer exists.
+	 *
+	 * @param name The canonical analyzer name to check.
+	 */
+	Tag GetAnalyzerTag(const char* name);
+
 	/**
 	 * Returns true if an analyzer is enabled.
 	 *

src/analyzer/protocol/arp/Plugin.cc

@@ -1,7 +1,21 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
-BRO_PLUGIN_BEGIN(Bro, ARP)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("ARP Parsing Code");
+namespace Bro_ARP {
-	BRO_PLUGIN_BIF_FILE(events);
+
-BRO_PLUGIN_END
+class Plugin : public plugin::Plugin {
+public:
+	plugin::Configuration Configure()
+		{
+		plugin::Configuration config;
+		config.name = "Bro::ARP";
+		config.description = "ARP Parsing";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/ayiya/AYIYA.h

@@ -14,7 +14,7 @@ public:
 	virtual void DeliverPacket(int len, const u_char* data, bool orig,
 					int seq, const IP_Hdr* ip, int caplen);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new AYIYA_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/ayiya/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "AYIYA.h"
 
-BRO_PLUGIN_BEGIN(Bro, AYIYA)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("AYIYA Analyzer");
+namespace Bro_AYIYA {
-	BRO_PLUGIN_ANALYZER("AYIYA", ayiya::AYIYA_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("AYIYA", ::analyzer::ayiya::AYIYA_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::AYIYA";
+		config.description = "AYIYA Analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/backdoor/BackDoor.h

@@ -73,7 +73,7 @@ public:
 	virtual void Done();
 	void StatTimer(double t, int is_expire);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new BackDoor_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/backdoor/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "BackDoor.h"
 
-BRO_PLUGIN_BEGIN(Bro, BackDoor)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("Backdoor Analyzer (deprecated)");
+namespace Bro_BackDoor {
-	BRO_PLUGIN_ANALYZER("BackDoor", backdoor::BackDoor_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("BackDoor", ::analyzer::backdoor::BackDoor_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::BackDoor";
+		config.description = "Backdoor Analyzer deprecated";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/bittorrent/BitTorrent.h

@@ -19,7 +19,7 @@ public:
 	virtual void Undelivered(int seq, int len, bool orig);
 	virtual void EndpointEOF(bool is_orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new BitTorrent_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/bittorrent/BitTorrentTracker.h

@@ -52,7 +52,7 @@ public:
 	virtual void Undelivered(int seq, int len, bool orig);
 	virtual void EndpointEOF(bool is_orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new BitTorrentTracker_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/bittorrent/Plugin.cc

@@ -1,12 +1,27 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "BitTorrent.h"
 #include "BitTorrentTracker.h"
 
-BRO_PLUGIN_BEGIN(Bro, BitTorrent)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("BitTorrent Analyzer");
+namespace Bro_BitTorrent {
-	BRO_PLUGIN_ANALYZER("BitTorrent", bittorrent::BitTorrent_Analyzer);
+
-	BRO_PLUGIN_ANALYZER("BitTorrentTracker", bittorrent::BitTorrentTracker_Analyzer);
+class Plugin : public plugin::Plugin {
-	BRO_PLUGIN_BIF_FILE(events);
+public:
-BRO_PLUGIN_END
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("BitTorrent", ::analyzer::bittorrent::BitTorrent_Analyzer::Instantiate));
+		AddComponent(new ::analyzer::Component("BitTorrentTracker", ::analyzer::bittorrent::BitTorrentTracker_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::BitTorrent";
+		config.description = "BitTorrent Analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/conn-size/ConnSize.h

@@ -21,7 +21,7 @@ public:
 	virtual void UpdateConnVal(RecordVal *conn_val);
 	virtual void FlipRoles();
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new ConnSize_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/conn-size/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "ConnSize.h"
 
-BRO_PLUGIN_BEGIN(Bro, ConnSize)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("Connection size analyzer");
+namespace Bro_ConnSize {
-	BRO_PLUGIN_ANALYZER("ConnSize", conn_size::ConnSize_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("ConnSize", ::analyzer::conn_size::ConnSize_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::ConnSize";
+		config.description = "Connection size analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/dce-rpc/DCE_RPC.h

@@ -178,7 +178,7 @@ public:
 	DCE_RPC_Analyzer(Connection* conn, bool speculative = false);
 	~DCE_RPC_Analyzer();
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new DCE_RPC_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/dce-rpc/Plugin.cc

@@ -1,11 +1,26 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "DCE_RPC.h"
 
-BRO_PLUGIN_BEGIN(Bro, DCE_RPC)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("DCE-RPC analyzer");
+namespace Bro_DCE_RPC {
-	BRO_PLUGIN_ANALYZER("DCE_RPC", dce_rpc::DCE_RPC_Analyzer);
+
-	BRO_PLUGIN_SUPPORT_ANALYZER("Contents_DCE_RPC");
+class Plugin : public plugin::Plugin {
-	BRO_PLUGIN_BIF_FILE(events);
+public:
-BRO_PLUGIN_END
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("DCE_RPC", ::analyzer::dce_rpc::DCE_RPC_Analyzer::Instantiate));
+		AddComponent(new ::analyzer::Component("Contents_DCE_RPC", 0));
+
+		plugin::Configuration config;
+		config.name = "Bro::DCE_RPC";
+		config.description = "DCE-RPC analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/dhcp/DHCP.h

@@ -16,7 +16,7 @@ public:
 	virtual void DeliverPacket(int len, const u_char* data, bool orig,
 					int seq, const IP_Hdr* ip, int caplen);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new DHCP_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/dhcp/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "DHCP.h"
 
-BRO_PLUGIN_BEGIN(Bro, DHCP)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("DHCP analyzer");
+namespace Bro_DHCP {
-	BRO_PLUGIN_ANALYZER("DHCP", dhcp::DHCP_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("DHCP", ::analyzer::dhcp::DHCP_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::DHCP";
+		config.description = "DHCP analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/dnp3/DNP3.h

@@ -17,7 +17,7 @@ public:
 	virtual void Undelivered(int seq, int len, bool orig);
 	virtual void EndpointEOF(bool is_orig);
 
-	static Analyzer* InstantiateAnalyzer(Connection* conn)
+	static Analyzer* Instantiate(Connection* conn)
 		{ return new DNP3_Analyzer(conn); }
 
 private:

src/analyzer/protocol/dnp3/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "DNP3.h"
 
-BRO_PLUGIN_BEGIN(Bro, DNP3)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("DNP3 analyzer");
+namespace Bro_DNP3 {
-	BRO_PLUGIN_ANALYZER("DNP3", dnp3::DNP3_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("DNP3", ::analyzer::dnp3::DNP3_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::DNP3";
+		config.description = "DNP3 analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/dnp3/events.bif

@@ -32,6 +32,8 @@ event dnp3_application_response_header%(c: connection, is_orig: bool, fc: count,
 ##
 ## qua_field: qualifier field.
 ##
+## number: TODO.
+##
 ## rf_low: the structure of the range field depends on the qualified field.
 ##         In some cases, the range field contains only one logic part, e.g.,
 ##         number of objects, so only *rf_low* contains useful values.

src/analyzer/protocol/dns/DNS.h

@@ -267,7 +267,7 @@ public:
 
 	void ExpireTimer(double t);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new DNS_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/dns/Plugin.cc

@@ -1,11 +1,26 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "DNS.h"
 
-BRO_PLUGIN_BEGIN(Bro, DNS)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("DNS analyzer");
+namespace Bro_DNS {
-	BRO_PLUGIN_ANALYZER("DNS", dns::DNS_Analyzer);
+
-	BRO_PLUGIN_SUPPORT_ANALYZER("Contents_DNS");
+class Plugin : public plugin::Plugin {
-	BRO_PLUGIN_BIF_FILE(events);
+public:
-BRO_PLUGIN_END
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("DNS", ::analyzer::dns::DNS_Analyzer::Instantiate));
+		AddComponent(new ::analyzer::Component("Contents_DNS", 0));
+
+		plugin::Configuration config;
+		config.name = "Bro::DNS";
+		config.description = "DNS analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/file/File.h

@@ -19,7 +19,7 @@ public:
 
 	void Undelivered(int seq, int len, bool orig);
 
-//	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+//	static analyzer::Analyzer* Instantiate(Connection* conn)
 //		{ return new File_Analyzer(conn); }
 
 protected:
@@ -40,7 +40,7 @@ public:
 
 	virtual void Undelivered(int seq, int len, bool orig);
 
-	static Analyzer* InstantiateAnalyzer(Connection* conn)
+	static Analyzer* Instantiate(Connection* conn)
 		{ return new IRC_Data(conn); }
 };
 
@@ -54,7 +54,7 @@ public:
 
 	virtual void Undelivered(int seq, int len, bool orig);
 
-	static Analyzer* InstantiateAnalyzer(Connection* conn)
+	static Analyzer* Instantiate(Connection* conn)
 		{ return new FTP_Data(conn); }
 };
 

src/analyzer/protocol/file/Plugin.cc

@@ -1,11 +1,26 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "./File.h"
 
-BRO_PLUGIN_BEGIN(Bro, File)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("Generic file analyzer");
+namespace Bro_File {
-	BRO_PLUGIN_ANALYZER("FTP_Data", file::FTP_Data);
+
-	BRO_PLUGIN_ANALYZER("IRC_Data", file::IRC_Data);
+class Plugin : public plugin::Plugin {
-	BRO_PLUGIN_BIF_FILE(events);
+public:
-BRO_PLUGIN_END
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("FTP_Data", ::analyzer::file::FTP_Data::Instantiate));
+		AddComponent(new ::analyzer::Component("IRC_Data", ::analyzer::file::IRC_Data::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::File";
+		config.description = "Generic file analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/finger/Finger.h

@@ -17,7 +17,7 @@ public:
 	// Line-based input.
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new Finger_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/finger/Plugin.cc

@@ -1,10 +1,24 @@
+// See the file  in the main distribution directory for copyright.
 
 #include "plugin/Plugin.h"
 
 #include "Finger.h"
 
-BRO_PLUGIN_BEGIN(Bro, Finger)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("Finger analyzer");
+namespace Bro_Finger {
-	BRO_PLUGIN_ANALYZER("Finger", finger::Finger_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("Finger", ::analyzer::finger::Finger_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::Finger";
+		config.description = "Finger analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/ftp/FTP.h

@@ -15,7 +15,7 @@ public:
 	virtual void Done();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{
 		return new FTP_Analyzer(conn);
 		}

src/analyzer/protocol/ftp/Plugin.cc

@@ -1,12 +1,26 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "FTP.h"
 
-BRO_PLUGIN_BEGIN(Bro, FTP)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("FTP analyzer");
+namespace Bro_FTP {
-	BRO_PLUGIN_ANALYZER("FTP", ftp::FTP_Analyzer);
+
-	BRO_PLUGIN_SUPPORT_ANALYZER("FTP_ADAT");
+class Plugin : public plugin::Plugin {
-	BRO_PLUGIN_BIF_FILE(events);
+public:
-	BRO_PLUGIN_BIF_FILE(functions);
+	plugin::Configuration Configure()
-BRO_PLUGIN_END
+		{
+		AddComponent(new ::analyzer::Component("FTP", ::analyzer::ftp::FTP_Analyzer::Instantiate));
+		AddComponent(new ::analyzer::Component("FTP_ADAT", 0));
+
+		plugin::Configuration config;
+		config.name = "Bro::FTP";
+		config.description = "FTP analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/gnutella/Gnutella.h

@@ -42,7 +42,7 @@ public:
 	virtual void Done ();
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new Gnutella_Analyzer(conn); }
 
 private:

src/analyzer/protocol/gnutella/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "Gnutella.h"
 
-BRO_PLUGIN_BEGIN(Bro, Gnutella)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("Gnutella analyzer");
+namespace Bro_Gnutella {
-	BRO_PLUGIN_ANALYZER("Gnutella", gnutella::Gnutella_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("Gnutella", ::analyzer::gnutella::Gnutella_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::Gnutella";
+		config.description = "Gnutella analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/gtpv1/GTPv1.h

@@ -14,7 +14,7 @@ public:
 	virtual void DeliverPacket(int len, const u_char* data, bool orig,
 					int seq, const IP_Hdr* ip, int caplen);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new GTPv1_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/gtpv1/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "GTPv1.h"
 
-BRO_PLUGIN_BEGIN(Bro, GTPv1)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("GTPv1 analyzer");
+namespace Bro_GTPv1 {
-	BRO_PLUGIN_ANALYZER("GTPv1", gtpv1::GTPv1_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("GTPv1", ::analyzer::gtpv1::GTPv1_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::GTPv1";
+		config.description = "GTPv1 analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/http/CMakeLists.txt

@@ -8,4 +8,3 @@ bro_plugin_cc(HTTP.cc Plugin.cc)
 bro_plugin_bif(events.bif)
 bro_plugin_bif(functions.bif)
 bro_plugin_end()
-

src/analyzer/protocol/http/HTTP.h

@@ -9,7 +9,7 @@
 #include "analyzer/protocol/mime/MIME.h"
 #include "binpac_bro.h"
 #include "IPAddr.h"
-#include "events.bif.h"
+#include "analyzer/protocol/http/events.bif.h"
 
 #include "HTTP.h"
 
@@ -183,7 +183,7 @@ public:
 	virtual void ConnectionReset();
 	virtual void PacketWithRST();
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new HTTP_Analyzer(conn); }
 
 	static bool Available()

src/analyzer/protocol/http/Plugin.cc

@@ -1,11 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "HTTP.h"
 
-BRO_PLUGIN_BEGIN(Bro, HTTP)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("HTTP analyzer");
+namespace Bro_HTTP {
-	BRO_PLUGIN_ANALYZER("HTTP", http::HTTP_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-	BRO_PLUGIN_BIF_FILE(functions);
+public:
-BRO_PLUGIN_END
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("HTTP", ::analyzer::http::HTTP_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::HTTP";
+		config.description = "HTTP analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/http/functions.bif

@@ -1,6 +1,6 @@
 
 %%{
-#include "protocol/http/HTTP.h"
+#include "analyzer/protocol/http/HTTP.h"
 %%}
 
 ## Skips the data of the HTTP entity.

src/analyzer/protocol/icmp/ICMP.h

@@ -21,7 +21,7 @@ public:
 
 	virtual void UpdateConnVal(RecordVal *conn_val);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new ICMP_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/icmp/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "ICMP.h"
 
-BRO_PLUGIN_BEGIN(Bro, ICMP)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("ICMP analyzer");
+namespace Bro_ICMP {
-	BRO_PLUGIN_ANALYZER("ICMP", icmp::ICMP_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("ICMP", ::analyzer::icmp::ICMP_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::ICMP";
+		config.description = "ICMP analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/ident/Ident.h

@@ -15,7 +15,7 @@ public:
 
 	virtual void DeliverStream(int length, const u_char* data, bool is_orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new Ident_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/ident/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "Ident.h"
 
-BRO_PLUGIN_BEGIN(Bro, Ident)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("Ident analyzer");
+namespace Bro_Ident {
-	BRO_PLUGIN_ANALYZER("Ident", ident::Ident_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("Ident", ::analyzer::ident::Ident_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::Ident";
+		config.description = "Ident analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/interconn/InterConn.h

@@ -49,7 +49,7 @@ public:
 	virtual void Done();
 	void StatTimer(double t, int is_expire);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{ return new InterConn_Analyzer(conn); }
 
 protected:

src/analyzer/protocol/interconn/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "InterConn.h"
 
-BRO_PLUGIN_BEGIN(Bro, InterConn)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("InterConn analyzer (deprecated)");
+namespace Bro_InterConn {
-	BRO_PLUGIN_ANALYZER("InterConn", interconn::InterConn_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("InterConn", ::analyzer::interconn::InterConn_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::InterConn";
+		config.description = "InterConn analyzer deprecated";
+		return config;
+		}
+} plugin;
+
+}
+}

src/analyzer/protocol/irc/IRC.h

@@ -32,7 +32,7 @@ public:
 	*/
 	virtual void DeliverStream(int len, const u_char* data, bool orig);
 
-	static analyzer::Analyzer* InstantiateAnalyzer(Connection* conn)
+	static analyzer::Analyzer* Instantiate(Connection* conn)
 		{
 		return new IRC_Analyzer(conn);
 		}

src/analyzer/protocol/irc/Plugin.cc

@@ -1,10 +1,25 @@
+// See the file  in the main distribution directory for copyright.
+
 
 #include "plugin/Plugin.h"
 
 #include "IRC.h"
 
-BRO_PLUGIN_BEGIN(Bro, IRC)
+namespace plugin {
-	BRO_PLUGIN_DESCRIPTION("IRC analyzer");
+namespace Bro_IRC {
-	BRO_PLUGIN_ANALYZER("IRC", irc::IRC_Analyzer);
+
-	BRO_PLUGIN_BIF_FILE(events);
+class Plugin : public plugin::Plugin {
-BRO_PLUGIN_END
+public:
+	plugin::Configuration Configure()
+		{
+		AddComponent(new ::analyzer::Component("IRC", ::analyzer::irc::IRC_Analyzer::Instantiate));
+
+		plugin::Configuration config;
+		config.name = "Bro::IRC";
+		config.description = "IRC analyzer";
+		return config;
+		}
+} plugin;
+
+}
+}