Merge remote-tracking branch 'origin/master' into topic/vladg/ssh

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "aux/plugins"]
 	path = aux/plugins
 	url = git://git.bro.org/bro-plugins
+[submodule "aux/broker"]
+	path = aux/broker
+	url = git://git.bro.org/broker

CHANGES

@@ -1,4 +1,190 @@
 
+2.3-541 | 2015-03-13 15:44:08 -0500
+
+  * Make INSTALL a symlink to doc/install/install.rst (Jon siwek)
+
+  * Fix Broxygen coverage. (Jon Siwek)
+
+2.3-539 | 2015-03-13 14:19:27 -0500
+
+  * BIT-1335: Include timestamp in default extracted file names.
+    And add a policy script to extract all files. (Jon Siwek)
+
+  * BIT-1311: Identify GRE tunnels as Tunnel::GRE, not Tunnel::IP.
+    (Jon Siwek)
+
+  * BIT-1309: Add Connection class getter methods for flow labels.
+    (Jon Siwek)
+
+2.3-536 | 2015-03-12 16:16:24 -0500
+
+  * Fix Broker leak tests. (Jon Siwek)
+
+2.3-534 | 2015-03-12 10:59:49 -0500
+
+  * Update NEWS file. (Jon Siwek)
+
+2.3-533 | 2015-03-12 10:18:53 -0500
+
+  * Give broker python bindings default install path within --prefix.
+    (Jon Siwek)
+
+2.3-530 | 2015-03-10 13:22:39 -0500
+
+  * Fix broker data stores in absence of --enable-debug. (Jon Siwek)
+
+2.3-529 | 2015-03-09 13:14:27 -0500
+
+  * Fix format specifier in SSL protocol violation. (Jon Siwek)
+
+2.3-526 | 2015-03-06 12:48:49 -0600
+
+  * Fix build warnings, clarify broker requirements, update submodule.
+    (Jon Siwek)
+
+  * Rename comm/ directories to broker/ (Jon Siwek)
+
+  * Rename broker-related namespaces. (Jon Siwek)
+
+  * Improve remote logging via broker by only sending fields w/ &log.
+    (Jon Siwek)
+
+  * Disable a stream's remote logging via broker if it fails. (Jon Siwek)
+
+  * Improve some broker communication unit tests. (Jon Siwek)
+
+2.3-518 | 2015-03-04 13:13:50 -0800
+
+  * Add bytes_recvd to stats.log recording the number of bytes
+    received, according to packet headers. (Mike Smiley)
+
+2.3-516 | 2015-03-04 12:30:06 -0800
+
+  * Extract most specific Common Name from SSL certificates (Johanna
+    Amann)
+
+  * Send CN and SAN fields of SSL certificates to the Intel framework.
+    (Johanna Amann)
+
+2.3-511 | 2015-03-02 18:07:17 -0800
+
+  * Changes to plugin meta hooks for function calls. (Gilbert Clark)
+
+        - Add frame argument.
+
+        - Change return value to tuple unambigiously whether hook
+          returned a result.
+
+2.3-493 | 2015-03-02 17:17:32 -0800
+
+  * Extend the SSL weak-keys policy file to also alert when
+    encountering SSL connections with old versions as well as unsafe
+    cipher suites. (Johanna Amann)
+
+  * Make the notice suppression handling of other SSL policy files a
+    tad more robust. (Johanna Amann)
+
+2.3-491 | 2015-03-02 17:12:56 -0800
+
+  * Updating docs for recent addition of local_resp. (Robin Sommer)
+
+2.3-489 | 2015-03-02 15:29:30 -0800
+
+  * Integrate Broker, Bro's new communication library. (Jon Siwek)
+
+    See aux/broker/README for more information on Broker, and
+    doc/frameworks/comm.rst for the corresponding Bro script API.
+
+    Broker support is by default off for now; it can be enabled at
+    configure time with --enable-broker. It requires CAF
+    (https://github.com/actor-framework/actor-framework); for now iot
+    needs CAF's "develop" branch. Broker also requires a C++11
+    compiler.
+
+    Broker will become a mandatory dependency in future Bro versions.
+
+  * Add --enable-c++11 configure flag to compile Bro's source code in
+    C++11 mode with a corresponding compiler. (Jon Siwek)
+
+2.3-451 | 2015-02-24 16:37:08 -0800
+
+  * Updating submodule(s).
+
+2.3-448 | 2015-02-23 16:58:10 -0800
+
+  * Updating NEWS. (Robin Sommer)
+
+2.3-447 | 2015-02-23 16:28:30 -0800
+
+  * Fix potential crash in logging framework when deserializing
+    WriterInfo from remote. where config is present. Testcase crashes
+    on unpatched versions of Bro. (Aaron Eppert)
+
+  * Fix wrong value test in WriterBackend. (Aaron Eppert)
+
+2.3-442 | 2015-02-23 13:29:30 -0800
+
+  * Add a "local_resp" field to conn.log, along the lines of the
+    existing "local_orig". (Mike Smiley)
+
+2.3-440 | 2015-02-23 11:39:17 -0600
+
+  * Updating plugin docs to recent changes. (Robin Sommer)
+
+  * Updating plugin tests to recent changes. (Robin Sommer)
+
+  * Making plugin names case-insensitive for some internal comparisions.
+    Makes plugin system more tolerant against spelling inconsistencies
+    are hard to catch otherwise. (Robin Sommer)
+
+  * Explicitly removing some old scripts on install that have moved
+    into plugins to prevent them causing confusion. (Robin Sommer)
+
+  * BIT-1312: Removing setting installation plugin path from
+    bro-path-dev.sh.  Also, adding to existing BRO_PLUGIN_PATH rather
+    than replacing. (Robin Sommer)
+
+  * Creating the installation directory for plugins at install time.
+    (Robin Sommer)
+
+2.3-427 | 2015-02-20 13:49:33 -0800
+
+  * Removing dependency on PCAP_NETMASK_UNKNOWN to compile with
+    libpcap < 1.1.1. (Robin Sommer)
+
+2.3-426 | 2015-02-20 12:45:51 -0800
+
+  * Add 'while' statement to Bro language. Really. (Jon Siwek)
+
+2.3-424 | 2015-02-20 12:39:10 -0800
+
+  * Add the ability to remove surrounding braces from the JSON
+    formatter. (Seth Hall)
+
+2.3-419 | 2015-02-13 09:10:44 -0600
+
+  * BIT-1011: Update the SOCKS analyzer to support user/pass login.
+    (Nicolas Retrain, Seth Hall, Jon Siwek)
+
+    - Add a new field to socks.log: "password".
+    - Two new events: "socks_login_userpass_request" and
+      "socks_login_userpass_reply".
+    - Two new weirds for unsupported SOCKS authentication method or
+      version.
+    - A new test for authenticated socks traffic.
+
+2.3-416 | 2015-02-12 12:18:42 -0600
+
+  * Submodule update - newest sqlite version (Johanna Amann)
+
+  * Fix use of deprecated gperftools headers. (Jon Siwek)
+
+2.3-413 | 2015-02-08 18:23:05 -0800
+
+  * Fixing analyzer tag types for some Files::* functions. (Robin Sommer)
+
+  * Changing load order for plugin scripts. (Robin Sommer)
+
 2.3-411 | 2015-02-05 10:05:48 -0600
 
   * Fix file analysis of files with total size below the bof_buffer size

CMakeLists.txt

@@ -31,12 +31,12 @@ configure_file(bro-path-dev.in ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev)
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.sh
      "export BROPATH=`${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
+     "export BRO_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n"
      "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev.csh
      "setenv BROPATH `${CMAKE_CURRENT_BINARY_DIR}/bro-path-dev`\n"
-     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src:${BRO_PLUGIN_INSTALL_PATH}\"\n"
+     "setenv BRO_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${BRO_PLUGIN_PATH}\n"
      "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
@@ -177,6 +177,17 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 ########################################################################
 ## Recurse on sub-directories
 
+if ( ENABLE_CXX11 )
+    include(RequireCXX11)
+endif ()
+
+if ( ENABLE_BROKER )
+    add_subdirectory(aux/broker)
+    set(brodeps ${brodeps} broker)
+    add_definitions(-DENABLE_BROKER)
+    include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker)
+endif ()
+
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(doc)
@@ -224,6 +235,7 @@ message(
     "\nCXXFLAGS:          ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}"
     "\nCPP:               ${CMAKE_CXX_COMPILER}"
     "\n"
+    "\nBroker:            ${ENABLE_BROKER}"
     "\nBroccoli:          ${INSTALL_BROCCOLI}"
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"

INSTALL

@@ -1,3 +0,0 @@
-
-See doc/install/install.rst for installation instructions.
-

INSTALL

@@ -0,0 +1 @@
+doc/install/install.rst

NEWS

@@ -31,6 +31,36 @@ New Functionality
 - Bro's file analysis now supports reassembly of files that are not
   transferred/seen sequentially.
 
+- Bro's scripting language now has a ``while`` statement::
+
+        while ( i < 5 )
+            print ++i;
+
+  ``next`` and ``break`` can be used inside the loop's body just like
+  with ``for`` loops.
+
+- Bro now integrates Broker, it's new communication library. See
+  aux/broker/README for more information on Broker, and
+  doc/frameworks/comm.rst for the corresponding Bro script API.
+
+  TODO: Extend with some more information on  Broker.
+
+  Broker support is by default off for now; it can be enabled at
+  configure time with --enable-broker. It requires CAF version 0.13+
+  (https://github.com/actor-framework/actor-framework) as well as a
+  C++11 compiler (e.g. GCC 4.8+ or Clang 3.3+).
+
+  Broker will become a mandatory dependency in future Bro versions.
+
+- Add --enable-c++11 configure flag to compile Bro's source code in
+  C++11 mode with a corresponding compiler. Note that 2.4 will be the
+  last version of Bro that compiles without C++11 support.
+
+- The SSL analysis now alert when encountering SSL connections with
+  old protocol versions or unsafe cipher suites.
+
+- [TODO] Add new BroControl features.
+
 Changed Functionality
 ---------------------
 
@@ -43,6 +73,11 @@ Changed Functionality
       have been added which contain the same information.  The
       ``mime_type`` field of ``Files::Info`` also still has this info.
 
+    * The earliest point that new mime type information is available is
+      in the ``file_mime_type`` event which comes after the ``file_new``
+      and ``file_over_new_connection`` events.  Scripts which inspected
+      mime type info within those events will need to be adapted.
+
     * Removed ``Files::add_analyzers_for_mime_type`` function.
 
     * Removed ``offset`` parameter of the ``file_extraction_limit``
@@ -56,6 +91,17 @@ Changed Functionality
 - has_valid_octets: now uses a string_vec parameter instead of
   string_array.
 
+- conn.log gained a new field local_resp that works like local_orig,
+  just for the responder address of the connection.
+
+- GRE tunnels are now identified as ``Tunnel::GRE`` instead of
+  ``Tunnel::IP``.
+
+- The default name for extracted files changed from extract-protocol-id
+  to extract-timestamp-protocol-id.
+
+- [TODO] Add changed BroControl features.
+
 Deprecated Functionality
 ------------------------
 

VERSION

@@ -1 +1 @@
-2.3-411
+2.3-541

aux/binpac

@@ -1 +1 @@
-Subproject commit 77a86591dcf89d7252d3676d3f1199d6c927d073
+Subproject commit ab50e5115bc0d217552a63f15382e45ed608f5fc

aux/bro-aux

@@ -1 +1 @@
-Subproject commit 0b713c027d3efaaca50e5df995c02656175573cd
+Subproject commit 52b273db79298daf5024d2d3d94824e7ab73a782

aux/broccoli

@@ -1 +1 @@
-Subproject commit d43cc790e5b8709b5e032e52ad0e00936494739b
+Subproject commit 45276b39a946d70095c983753cd321ad07dcf285

aux/broctl

@@ -1 +1 @@
-Subproject commit 8c9b87bc73e1ddaa304e3d89028c1e7b95d37a91
+Subproject commit 762d2722290ca0004d0da2b0b96baea6a3a7f3f4

aux/broker

@@ -0,0 +1 @@
+Subproject commit 1a2ab9ee7c80ca905e86a2a11283e7c0477341a9

aux/btest

@@ -1 +1 @@
-Subproject commit 93d4989ed1537e4d143cf09d44077159f869a4b2
+Subproject commit d69df586c91531db0c3abe838b10a429dda4fa87

aux/plugins

@@ -1 +1 @@
-Subproject commit ad600b5bdcd56a2723e323c0f2c8e1708956ca4f
+Subproject commit 71d820e9d8ca753fea8fb34ea3987993b28d79e4

cmake

@@ -1 +1 @@
-Subproject commit 1316c07f7059647b6c4a496ea36e4b83bb5d8f0f
+Subproject commit 2fd35ab6a6245a005828c32f0aa87eb21698c054

configure

@@ -41,6 +41,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --enable-perftools-debug use Google's perftools for debugging
     --enable-jemalloc      link against jemalloc
     --enable-ruby          build ruby bindings for broccoli (deprecated)
+    --enable-c++11         build using the C++11 standard
+    --enable-broker        enable use of the Broker communication library
+                           (requires C++ Actor Framework and C++11)
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
@@ -55,6 +58,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-flex=PATH       path to flex executable
     --with-bison=PATH      path to bison executable
     --with-perl=PATH       path to perl executable
+    --with-libcaf=PATH     path to C++ Actor Framework installation
+                           (a required Broker dependency)
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libGeoIP install root
@@ -67,6 +72,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
+    --with-rocksdb=PATH    path to RocksDB installation
+                           (an optional Broker dependency)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -142,6 +149,10 @@ while [ $# -ne 0 ]; do
             append_cache_entry CMAKE_INSTALL_PREFIX PATH   $optarg
             append_cache_entry BRO_ROOT_DIR         PATH   $optarg
             append_cache_entry PY_MOD_INSTALL_DIR   PATH   $optarg/lib/broctl
+
+            if [ -n "$user_enabled_broker" ]; then
+                append_cache_entry BROKER_PYTHON_HOME   PATH   $prefix
+            fi
             ;;
         --scriptdir=*)
             append_cache_entry BRO_SCRIPT_INSTALL_PATH STRING $optarg
@@ -176,6 +187,15 @@ while [ $# -ne 0 ]; do
         --enable-jemalloc)
             append_cache_entry ENABLE_JEMALLOC     BOOL   true
             ;;
+        --enable-c++11)
+            append_cache_entry ENABLE_CXX11         BOOL   true
+            ;;
+        --enable-broker)
+            append_cache_entry ENABLE_CXX11         BOOL   true
+            append_cache_entry ENABLE_BROKER        BOOL   true
+            append_cache_entry BROKER_PYTHON_HOME   PATH   $prefix
+            user_enabled_broker="true"
+            ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
             ;;
@@ -248,6 +268,12 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-libcaf=*)
+            append_cache_entry LIBCAF_ROOT_DIR      PATH   $optarg
+            ;;
+        --with-rocksdb=*)
+            append_cache_entry ROCKSDB_ROOT_DIR     PATH   $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

doc/components/broker/README.rst

@@ -0,0 +1 @@
+../../../aux/broker/README

doc/components/broker/broker-manual.rst

@@ -0,0 +1 @@
+../../../aux/broker/broker-manual.rst

doc/components/index.rst

@@ -17,6 +17,8 @@ current, independent component releases.
    Broccoli - User Manual <broccoli/broccoli-manual>
    Broccoli Python Bindings <broccoli-python/README>
    Broccoli Ruby Bindings <broccoli-ruby/README>
+   Broker - Bro's (New) Messaging Library (README) <broker/README>
+   Broker - User Manual <broker/broker-manual.rst>
    BroControl - Interactive Bro management shell <broctl/README>
    Bro-Aux - Small auxiliary tools for Bro <bro-aux/README>
    BTest - A unit testing framework <btest/README>

doc/devel/plugins.rst

@@ -3,7 +3,7 @@
 Writing Bro Plugins
 ===================
 
-Bro is internally moving to a plugin structure that enables extending
+Bro internally provides plugin API that enables extending
 the system dynamically, without modifying the core code base. That way
 custom code remains self-contained and can be maintained, compiled,
 and installed independently. Currently, plugins can add the following
@@ -42,18 +42,17 @@ certain structure. To get started, Bro's distribution provides a
 helper script ``aux/bro-aux/plugin-support/init-plugin`` that creates
 a skeleton plugin that can then be customized. Let's use that::
 
-    # mkdir rot13-plugin
+    # init-plugin ./rot13-plugin Demo Rot13
-    # cd rot13-plugin
-    # init-plugin Demo Rot13
 
-As you can see the script takes two arguments. The first is a
+As you can see, the script takes three arguments. The first is a
-namespace the plugin will live in, and the second a descriptive name
+directory inside which the plugin skeleton will be created.  The second
-for the plugin itself. Bro uses the combination of the two to identify
+is the namespace the plugin will live in, and the third is a descriptive
-a plugin. The namespace serves to avoid naming conflicts between
+name for the plugin itself relative to the namespace. Bro uses the
-plugins written by independent developers; pick, e.g., the name of
+combination of namespace and name to identify a plugin. The namespace
-your organisation. The namespace ``Bro`` is reserved for functionality
+serves to avoid naming conflicts between plugins written by independent
-distributed by the Bro Project. In our example, the plugin will be
+developers; pick, e.g., the name of your organisation. The namespace
-called ``Demo::Rot13``.
+``Bro`` is reserved for functionality distributed by the Bro Project. In
+our example, the plugin will be called ``Demo::Rot13``.
 
 The ``init-plugin`` script puts a number of files in place. The full
 layout is described later. For now, all we need is
@@ -61,7 +60,7 @@ layout is described later. For now, all we need is
 there as follows::
 
     # cat src/rot13.bif
-    module CaesarCipher;
+    module Demo;
 
     function rot13%(s: string%) : string
         %{
@@ -82,18 +81,22 @@ The syntax of this file is just like any other ``*.bif`` file; we
 won't go into it here.
 
 Now we can already compile our plugin, we just need to tell the
-configure script put in place by ``init-plugin`` where the Bro source
+configure script that ``init-plugin`` put in place where the Bro
-tree is located (Bro needs to have been built there first)::
+source tree is located (Bro needs to have been built there first)::
 
+    # cd rot13-plugin
     # ./configure --bro-dist=/path/to/bro/dist && make
     [... cmake output ...]
 
-Now our ``rot13-plugin`` directory has everything that it needs
+This builds the plugin in a subdirectory ``build/``. In fact, that
-for Bro to recognize it as a dynamic plugin. Once we point Bro to it,
+subdirectory *becomes* the plugin: when ``make`` finishes, ``build/``
-it will pull it in automatically, as we can check with the ``-N``
+has everything it needs for Bro to recognize it as a dynamic plugin.
+
+Let's try that. Once we point Bro to the ``build/`` directory, it will
+pull in our new plugin automatically, as we can check with the ``-N``
 option::
 
-    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin
+    # export BRO_PLUGIN_PATH=/path/to/rot13-plugin/build
     # bro -N
     [...]
     Plugin: Demo::Rot13 - <Insert brief description of plugin> (dynamic, version 1)
@@ -127,12 +130,12 @@ more verbose option ``-NN``::
     # bro -NN
     [...]
     Plugin: Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1)
-        [Function] CaesarCipher::rot13
+        [Function] Demo::rot13
     [...]
 
 There's our function. Now let's use it::
 
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
     Uryyb
 
 It works. We next install the plugin along with Bro itself, so that it
@@ -141,36 +144,40 @@ environment variable. If we first unset the variable, the function
 will no longer be available::
 
     # unset BRO_PLUGIN_PATH
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
-    error in <command line>, line 1: unknown identifier CaesarCipher::rot13, at or near "CaesarCipher::rot13"
+    error in <command line>, line 1: unknown identifier Demo::rot13, at or near "Demo::rot13"
 
 Once we install it, it works again::
 
     # make install
-    # bro -e 'print CaesarCipher::rot13("Hello")'
+    # bro -e 'print Demo::rot13("Hello")'
     Uryyb
 
 The installed version went into
 ``<bro-install-prefix>/lib/bro/plugins/Demo_Rot13``.
 
-We can distribute the plugin in either source or binary form by using
+One can distribute the plugin independently of Bro for others to use.
-the Makefile's ``sdist`` and ``bdist`` target, respectively. Both
+To distribute in source form, just remove the ``build/`` (``make
-create corrsponding tarballs::
+distclean`` does that) and then tar up the whole ``rot13-plugin/``
+directory. Others then follow the same process as above after
+unpacking. To distribute the plugin in binary form, the build process
+conveniently creates a corresponding tarball in ``build/dist/``. In
+this case, it's called ``Demo_Rot13-0.1.tar.gz``, with the version
+number coming out of the ``VERSION`` file that ``init-plugin`` put
+into place. The binary tarball has everything needed to run the
+plugin, but no further source files. Optionally, one can include
+further files by specifying them in the plugin's ``CMakeLists.txt``
+through the ``bro_plugin_dist_files`` macro; the skeleton does that
+for ``README``, ``VERSION``, ``CHANGES``, and ``COPYING``. To use the
+plugin through the binary tarball, just unpack it and point
+``BRO_PLUGIN_PATH`` there; or copy it into
+``<bro-install-prefix>/lib/bro/plugins/`` directly.
 
-    # make sdist
+Before distributing your plugin, you should edit some of the meta
-    [...]
+files that ``init-plugin`` puts in place. Edit ``README`` and
-    Source distribution in build/sdist/Demo_Rot13.tar.gz
+``VERSION``, and update ``CHANGES`` when you make changes. Also put a
-
+license file in place as ``COPYING``; if BSD is fine, you find a
-    # make bdist
+template in ``COPYING.edit-me``.
-    [...]
-    Binary distribution in build/Demo_Rot13-darwin-x86_64.tar.gz
-
-The source archive will contain everything in the plugin directory
-except any generated files. The binary archive will contain anything
-needed to install and run the plugin, i.e., just what ``make install``
-puts into place as well. As the binary distribution is
-platform-dependent, its name includes the OS and architecture the
-plugin was built on.
 
 Plugin Directory Layout
 =======================
@@ -179,7 +186,7 @@ A plugin's directory needs to follow a set of conventions so that Bro
 (1) recognizes it as a plugin, and (2) knows what to load.  While
 ``init-plugin`` takes care of most of this, the following is the full
 story. We'll use ``<base>`` to represent a plugin's top-level
-directory.
+directory. With the skeleton, ``<base>`` corresponds to ``build/``.
 
 ``<base>/__bro_plugin__``
     A file that marks a directory as containing a Bro plugin. The file
@@ -205,6 +212,8 @@ directory.
     Directory with auto-generated Bro scripts that declare the plugin's
     bif elements. The files here are produced by ``bifcl``.
 
+Any other files in ``<base>`` are ignored by Bro.
+
 By convention, a plugin should put its custom scripts into sub folders
 of ``scripts/``, i.e., ``scripts/<script-namespace>/<script>.bro`` to
 avoid conflicts. As usual, you can then put a ``__load__.bro`` in
@@ -229,15 +238,19 @@ their source directory (after ``make`` and setting Bro's
 install``).
 
 ``make install`` copies over the ``lib`` and ``scripts`` directories,
-as well as the ``__bro_plugin__`` magic file and the ``README`` (which
+as well as the ``__bro_plugin__`` magic file and any further
-you should customize). One can add further CMake ``install`` rules to
+distribution files specified in ``CMakeLists.txt`` (e.g., README,
-install additional files if needed.
+VERSION). You can find a full list of files installed in
+``build/MANIFEST``. Behind the scenes, ``make install`` really just
+copies over the binary tarball in ``build/dist``. 
 
-``init-plugin`` will never overwrite existing files, so it's safe to
+``init-plugin`` will never overwrite existing files. If its target
-rerun in an existing plugin directory; it only put files in place that
+directory already exists, it will be default decline to do anything.
-don't exist yet. That also provides a convenient way to revert a file
+You can run it with ``-u`` instead to update an existing plugin,
-back to what ``init-plugin`` created originally: just delete it and
+however it will never overwrite any existing files; it will only put
-rerun.
+in place files it doesn't find yet. To revert a file back to what
+``init-plugin`` created originally, delete it first and then rerun
+with ``-u``.
 
 Activating a Plugin
 ===================
@@ -355,7 +368,7 @@ let's get that in place::
     % cat .diag
     == File ===============================
     Demo::Rot13 - Caesar cipher rotating a string's characters by 13 places. (dynamic, version 1.0)
-        [Function] CaesarCipher::rot13
+        [Function] Demo::rot13
 
     == Error ===============================
     test-diff: no baseline found.
@@ -375,14 +388,14 @@ Now let's add a custom test that ensures that our bif works
 correctly::
 
     # cd tests
-    # cat >plugin/rot13.bro
+    # cat >rot13/bif-rot13.bro
 
     # @TEST-EXEC: bro %INPUT >output
     # @TEST-EXEC: btest-diff output
 
     event bro_init()
         {
-        print CaesarCipher::rot13("Hello");
+        print Demo::rot13("Hello");
         }
 
 Check the output::
@@ -415,7 +428,7 @@ Debugging Plugins
 =================
 
 If your plugin isn't loading as expected, Bro's debugging facilities
-can help to illuminate what's going on. To enable, recompile Bro
+can help illuminate what's going on. To enable, recompile Bro
 with debugging support (``./configure --enable-debug``), and
 afterwards rebuild your plugin as well. If you then run Bro with ``-B
 plugins``, it will produce a file ``debug.log`` that records details
@@ -435,7 +448,6 @@ replaced with a simple dash. Example: If the plugin is called
 output will be recorded to ``debug.log`` if Bro's compiled in debug
 mode.
 
-
 Documenting Plugins
 ===================
 

doc/frameworks/broker.rst

@@ -0,0 +1,202 @@
+
+.. _brokercomm-framework:
+
+======================================
+Broker-Enabled Communication Framework
+======================================
+
+.. rst-class:: opening
+
+    Bro can now use the `Broker Library
+    <../components/broker/README.html>`_ to exchange information with
+    other Bro processes.  To enable it run Bro's ``configure`` script
+    with the ``--enable-broker`` option.  Note that a C++11 compatible
+    compiler (e.g. GCC 4.8+ or Clang 3.3+) is required as well as the
+    `C++ Actor Framework <http://actor-framework.org/>`_.
+
+.. contents::
+
+Connecting to Peers
+===================
+
+Communication via Broker must first be turned on via
+:bro:see:`BrokerComm::enable`.
+
+Bro can accept incoming connections by calling :bro:see:`BrokerComm::listen`
+and then monitor connection status updates via
+:bro:see:`BrokerComm::incoming_connection_established` and
+:bro:see:`BrokerComm::incoming_connection_broken`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-listener.bro
+
+Bro can initiate outgoing connections by calling :bro:see:`BrokerComm::connect`
+and then monitor connection status updates via
+:bro:see:`BrokerComm::outgoing_connection_established`,
+:bro:see:`BrokerComm::outgoing_connection_broken`, and
+:bro:see:`BrokerComm::outgoing_connection_incompatible`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/connecting-connector.bro
+
+Remote Printing
+===============
+
+To receive remote print messages, first use
+:bro:see:`BrokerComm::subscribe_to_prints` to advertise to peers a topic
+prefix of interest and then create an event handler for
+:bro:see:`BrokerComm::print_handler` to handle any print messages that are
+received.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-listener.bro
+
+To send remote print messages, just call :bro:see:`BrokerComm::print`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/printing-connector.bro
+
+Notice that the subscriber only used the prefix "bro/print/", but is
+able to receive messages with full topics of "bro/print/hi",
+"bro/print/stuff", and "bro/print/bye".  The model here is that the
+publisher of a message checks for all subscribers who advertised
+interest in a prefix of that message's topic and sends it to them.
+
+Message Format
+--------------
+
+For other applications that want to exchange print messages with Bro,
+the Broker message format is simply:
+
+.. code:: c++
+
+    broker::message{std::string{}};
+
+Remote Events
+=============
+
+Receiving remote events is similar to remote prints.  Just use
+:bro:see:`BrokerComm::subscribe_to_events` and possibly define any new events
+along with handlers that peers may want to send.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-listener.bro
+
+To send events, there are two choices.  The first is to use call
+:bro:see:`BrokerComm::event` directly.  The second option is to use
+:bro:see:`BrokerComm::auto_event` to make it so a particular event is
+automatically sent to peers whenever it is called locally via the normal
+event invocation syntax.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/events-connector.bro
+
+Again, the subscription model is prefix-based.
+
+Message Format
+--------------
+
+For other applications that want to exchange event messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{std::string{}, ...};
+
+The first parameter is the name of the event and the remaining ``...``
+are its arguments, which are any of the support Broker data types as
+they correspond to the Bro types for the event named in the first
+parameter of the message.
+
+Remote Logging
+==============
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/testlog.bro
+
+Use :bro:see:`BrokerComm::subscribe_to_logs` to advertise interest in logs
+written by peers.  The topic names that Bro uses are implicitly of the
+form "bro/log/<stream-name>".
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-listener.bro
+
+To send remote logs either use :bro:see:`Log::enable_remote_logging` or
+:bro:see:`BrokerComm::enable_remote_logs`.  The former allows any log stream
+to be sent to peers while the later toggles remote logging for
+particular streams.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/logs-connector.bro
+
+Message Format
+--------------
+
+For other applications that want to exchange logs messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{broker::enum_value{}, broker::record{}};
+
+The enum value corresponds to the stream's :bro:see:`Log::ID` value, and
+the record corresponds to a single entry of that log's columns record,
+in this case a ``Test::INFO`` value.
+
+Tuning Access Control
+=====================
+
+By default, endpoints do not restrict the message topics that it sends
+to peers and do not restrict what message topics and data store
+identifiers get advertised to peers.  These are the default
+:bro:see:`BrokerComm::EndpointFlags` supplied to :bro:see:`BrokerComm::enable`.
+
+If not using the ``auto_publish`` flag, one can use the
+:bro:see:`BrokerComm::publish_topic` and :bro:see:`BrokerComm::unpublish_topic`
+functions to manipulate the set of message topics (must match exactly)
+that are allowed to be sent to peer endpoints.  These settings take
+precedence over the per-message ``peers`` flag supplied to functions
+that take a :bro:see:`BrokerComm::SendFlags` such as :bro:see:`BrokerComm::print`,
+:bro:see:`BrokerComm::event`, :bro:see:`BrokerComm::auto_event` or
+:bro:see:`BrokerComm::enable_remote_logs`.
+
+If not using the ``auto_advertise`` flag, one can use the
+:bro:see:`BrokerComm::advertise_topic` and :bro:see:`BrokerComm::unadvertise_topic`
+to manupulate the set of topic prefixes that are allowed to be
+advertised to peers.  If an endpoint does not advertise a topic prefix,
+the only way a peers can send messages to it is via the ``unsolicited``
+flag of :bro:see:`BrokerComm::SendFlags`  and choosing a topic with a matching
+prefix (i.e. full topic may be longer than receivers prefix, just the
+prefix needs to match).
+
+Distributed Data Stores
+=======================
+
+There are three flavors of key-value data store interfaces: master,
+clone, and frontend.
+
+A frontend is the common interface to query and modify data stores.
+That is, a clone is a specific type of frontend and a master is also a
+specific type of frontend, but a standalone frontend can also exist to
+e.g. query and modify the contents of a remote master store without
+actually "owning" any of the contents itself.
+
+A master data store can be be cloned from remote peers which may then
+perform lightweight, local queries against the clone, which
+automatically stays synchronized with the master store.  Clones cannot
+modify their content directly, instead they send modifications to the
+centralized master store which applies them and then broadcasts them to
+all clones.
+
+Master and clone stores get to choose what type of storage backend to
+use.  E.g. In-memory versus SQLite for persistence.  Note that if clones
+are used, data store sizes should still be able to fit within memory
+regardless of the storage backend as a single snapshot of the master
+store is sent in a single chunk to initialize the clone.
+
+Data stores also support expiration on a per-key basis either using an
+absolute point in time or a relative amount of time since the entry's
+last modification time.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-listener.bro
+
+.. btest-include:: ${DOC_ROOT}/frameworks/broker/stores-connector.bro
+
+In the above example, if a local copy of the store contents isn't
+needed, just replace the :bro:see:`BrokerStore::create_clone` call with
+:bro:see:`BrokerStore::create_frontend`.  Queries will then be made against
+the remote master store instead of the local clone.
+
+Note that all queries are made within Bro's asynchrounous ``when``
+statements and must specify a timeout block.

doc/frameworks/broker/connecting-connector.bro

@@ -0,0 +1,19 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+		  peer_address, peer_port, peer_name;
+	terminate();
+	}

doc/frameworks/broker/connecting-listener.bro

@@ -0,0 +1,21 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event BrokerComm::incoming_connection_broken(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_broken", peer_name;
+	terminate();
+	}

doc/frameworks/broker/events-connector.bro

@@ -0,0 +1,31 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	BrokerComm::auto_event("bro/event/my_auto_event", my_auto_event);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "hi", 0));
+	event my_auto_event("stuff", 88);
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "...", 1));
+	event my_auto_event("more stuff", 51);
+	BrokerComm::event("bro/event/my_event", BrokerComm::event_args(my_event, "bye", 2));
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/events-listener.bro

@@ -0,0 +1,37 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+global msg_count = 0;
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_events("bro/event/");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event my_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}
+
+event my_auto_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_auto_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}

doc/frameworks/broker/logs-connector.bro

@@ -0,0 +1,40 @@
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+redef Log::enable_local_logging = F;
+redef Log::enable_remote_logging = F;
+global n = 0;
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::enable_remote_logs(Test::LOG);
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event do_write()
+	{
+	if ( n == 6 )
+		return;
+
+	Log::write(Test::LOG, [$msg = "ping", $num = n]);
+	++n;
+	event do_write();
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	event do_write();
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/logs-listener.bro

@@ -0,0 +1,25 @@
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_logs("bro/log/Test::LOG");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event Test::log_test(rec: Test::Info)
+	{
+	print "wrote log", rec;
+
+	if ( rec$num == 5 )
+		terminate();
+	}

doc/frameworks/broker/printing-connector.bro

@@ -0,0 +1,26 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "BrokerComm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	BrokerComm::print("bro/print/hi", "hello");
+	BrokerComm::print("bro/print/stuff", "...");
+	BrokerComm::print("bro/print/bye", "goodbye");
+	}
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/broker/printing-listener.bro

@@ -0,0 +1,26 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef BrokerComm::endpoint_name = "listener";
+global msg_count = 0;
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_prints("bro/print/");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}
+
+event BrokerComm::incoming_connection_established(peer_name: string)
+	{
+	print "BrokerComm::incoming_connection_established", peer_name;
+	}
+
+event BrokerComm::print_handler(msg: string)
+	{
+	++msg_count;
+	print "got print message", msg;
+
+	if ( msg_count == 3 )
+		terminate();
+	}

doc/frameworks/broker/stores-connector.bro

@@ -0,0 +1,53 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of BrokerStore::Handle;
+
+function dv(d: BrokerComm::Data): BrokerComm::DataVector
+	{
+	local rval: BrokerComm::DataVector;
+	rval[0] = d;
+	return rval;
+	}
+
+global ready: event();
+
+event BrokerComm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}
+
+event BrokerComm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	local myset: set[string] = {"a", "b", "c"};
+	local myvec: vector of string = {"alpha", "beta", "gamma"};
+	h = BrokerStore::create_master("mystore");
+	BrokerStore::insert(h, BrokerComm::data("one"), BrokerComm::data(110));
+	BrokerStore::insert(h, BrokerComm::data("two"), BrokerComm::data(223));
+	BrokerStore::insert(h, BrokerComm::data("myset"), BrokerComm::data(myset));
+	BrokerStore::insert(h, BrokerComm::data("myvec"), BrokerComm::data(myvec));
+	BrokerStore::increment(h, BrokerComm::data("one"));
+	BrokerStore::decrement(h, BrokerComm::data("two"));
+	BrokerStore::add_to_set(h, BrokerComm::data("myset"), BrokerComm::data("d"));
+	BrokerStore::remove_from_set(h, BrokerComm::data("myset"), BrokerComm::data("b"));
+	BrokerStore::push_left(h, BrokerComm::data("myvec"), dv(BrokerComm::data("delta")));
+	BrokerStore::push_right(h, BrokerComm::data("myvec"), dv(BrokerComm::data("omega")));
+
+	when ( local res = BrokerStore::size(h) )
+		{
+		print "master size", res;
+		event ready();
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::connect("127.0.0.1", broker_port, 1secs);
+	BrokerComm::auto_event("bro/event/ready", ready);
+	}

doc/frameworks/broker/stores-listener.bro

@@ -0,0 +1,43 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of BrokerStore::Handle;
+global expected_key_count = 4;
+global key_count = 0;
+
+function do_lookup(key: string)
+	{
+	when ( local res = BrokerStore::lookup(h, BrokerComm::data(key)) )
+		{
+		++key_count;
+		print "lookup", key, res;
+
+		if ( key_count == expected_key_count )
+			terminate();
+		}
+	timeout 10sec
+		{ print "timeout", key; }
+	}
+
+event ready()
+	{
+	h = BrokerStore::create_clone("mystore");
+
+	when ( local res = BrokerStore::keys(h) )
+		{
+		print "clone keys", res;
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 0)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 1)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 2)));
+		do_lookup(BrokerComm::refine_to_string(BrokerComm::vector_lookup(res$result, 3)));
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	BrokerComm::enable();
+	BrokerComm::subscribe_to_events("bro/event/ready");
+	BrokerComm::listen(broker_port, "127.0.0.1");
+	}

doc/frameworks/broker/testlog.bro

@@ -0,0 +1,19 @@
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		msg: string &log;
+		num: count &log;
+	};
+
+	global log_test: event(rec: Test::Info);
+}
+
+event bro_init() &priority=5
+	{
+	BrokerComm::enable();
+	Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]);
+	}

doc/frameworks/index.rst

@@ -14,4 +14,4 @@ Frameworks
    notice
    signatures
    sumstats
-
+   broker

doc/script-reference/statements.rst

@@ -45,8 +45,11 @@ Statements
 |                            | file                   |
 +----------------------------+------------------------+
 | :bro:keyword:`for`,        | Loop over each         |
-| :bro:keyword:`next`,       | element in a container |
+| :bro:keyword:`while`,      | element in a container |
-| :bro:keyword:`break`       | object                 |
+| :bro:keyword:`next`,       | object (``for``), or   |
+| :bro:keyword:`break`       | as long as a condition |
+|                            | evaluates to true      |
+|                            | (``while``).           |
 +----------------------------+------------------------+
 | :bro:keyword:`if`          | Evaluate boolean       |
 |                            | expression and if true,|
@@ -563,6 +566,36 @@ Here are the statements that the Bro scripting language supports.
     See the :bro:keyword:`return` statement for an explanation of how to
     create an asynchronous function in a Bro script.
 
+.. bro:keyword:: while
+
+    A "while" loop iterates over a body statement as long a given
+    condition remains true.
+
+    A :bro:keyword:`break` statement can be used at any time to immediately
+    terminate the "while" loop, and a :bro:keyword:`next` statement can be
+    used to skip to the next loop iteration.
+
+    Example::
+
+        local i = 0;
+
+        while ( i < 5 )
+            print ++i;
+
+        while ( some_cond() )
+            {
+            local finish_up = F;
+
+            if ( skip_ahead() )
+                next;
+
+            [...]
+
+            if ( finish_up )
+                break;
+
+            [...]
+            }
 
 .. _compound statement:
 

doc/scripting/index.rst

@@ -826,7 +826,7 @@ example of the ``record`` data type in the earlier sections, the
 ``conn.log``, is shown by the excerpt below.
 
 .. btest-include:: ${BRO_SRC_ROOT}/scripts/base/protocols/conn/main.bro
-   :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,92,95,99,102,106,110-111,116
+   :lines: 10-12,16-17,19,21,23,25,28,31,35,38,57,63,69,75,98,101,105,108,112,116-117,122
 
 Looking at the structure of the definition, a new collection of data
 types is being defined as a type called ``Info``.  Since this type

scripts/base/files/extract/main.bro

@@ -53,7 +53,8 @@ function set_limit(f: fa_file, args: Files::AnalyzerArgs, n: count): bool
 function on_add(f: fa_file, args: Files::AnalyzerArgs)
 	{
 	if ( ! args?$extract_filename )
-		args$extract_filename = cat("extract-", f$source, "-", f$id);
+		args$extract_filename = cat("extract-", f$last_active, "-", f$source,
+		                            "-", f$id);
 
 	f$info$extracted = args$extract_filename;
 	args$extract_filename = build_path_compressed(prefix, args$extract_filename);

scripts/base/frameworks/broker/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/broker/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module BrokerComm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: BrokerComm::connect BrokerComm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of BrokerComm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of BrokerComm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type TableItem : record {
+		key: BrokerComm::Data;
+		val: BrokerComm::Data;
+	};
+}
+
+module BrokerStore;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: BrokerStore::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: BrokerComm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/frameworks/files/main.bro

@@ -267,7 +267,7 @@ export {
 	## mts: The set of MIME types, each in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME types were successfully registered.
-	global register_for_mime_types: function(tag: Analyzer::Tag, mts: set[string]) : bool;
+	global register_for_mime_types: function(tag: Files::Tag, mts: set[string]) : bool;
 
 	## Registers a MIME type for an analyzer. If a future file with this type is seen,
 	## the analyzer will be automatically assigned to parsing it. The function *adds*
@@ -278,20 +278,20 @@ export {
 	## mt: The MIME type in the form "foo/bar" (case-insensitive).
 	##
 	## Returns: True if the MIME type was successfully registered.
-	global register_for_mime_type: function(tag: Analyzer::Tag, mt: string) : bool;
+	global register_for_mime_type: function(tag: Files::Tag, mt: string) : bool;
 
 	## Returns a set of all MIME types currently registered for a specific analyzer.
 	##
 	## tag: The tag of the analyzer.
 	##
 	## Returns: The set of MIME types.
-	global registered_mime_types: function(tag: Analyzer::Tag) : set[string];
+	global registered_mime_types: function(tag: Files::Tag) : set[string];
 
 	## Returns a table of all MIME-type-to-analyzer mappings currently registered.
 	##
 	## Returns: A table mapping each analyzer to the set of MIME types
 	##          registered for it.
-	global all_registered_mime_types: function() : table[Analyzer::Tag] of set[string];
+	global all_registered_mime_types: function() : table[Files::Tag] of set[string];
 
 	## Event that can be handled to access the Info record as it is sent on
 	## to the logging framework.
@@ -306,8 +306,8 @@ redef record fa_file += {
 global registered_protocols: table[Analyzer::Tag] of ProtoRegistration = table();
 
 # Store the MIME type to analyzer mappings.
-global mime_types: table[Analyzer::Tag] of set[string];
+global mime_types: table[Files::Tag] of set[string];
-global mime_type_to_analyzers: table[string] of set[Analyzer::Tag];
+global mime_type_to_analyzers: table[string] of set[Files::Tag];
 
 global analyzer_add_callbacks: table[Files::Tag] of function(f: fa_file, args: AnalyzerArgs) = table();
 
@@ -401,7 +401,7 @@ function register_protocol(tag: Analyzer::Tag, reg: ProtoRegistration): bool
 	return result;
 	}
 
-function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) : bool
+function register_for_mime_types(tag: Files::Tag, mime_types: set[string]) : bool
 	{
 	local rc = T;
 
@@ -414,7 +414,7 @@ function register_for_mime_types(tag: Analyzer::Tag, mime_types: set[string]) :
 	return rc;
 	}
 
-function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
+function register_for_mime_type(tag: Files::Tag, mt: string) : bool
 	{
 	if ( tag !in mime_types )
 		{
@@ -431,12 +431,12 @@ function register_for_mime_type(tag: Analyzer::Tag, mt: string) : bool
 	return T;
 	}
 
-function registered_mime_types(tag: Analyzer::Tag) : set[string]
+function registered_mime_types(tag: Files::Tag) : set[string]
 	{
 	return tag in mime_types ? mime_types[tag] : set();
 	}
 
-function all_registered_mime_types(): table[Analyzer::Tag] of set[string]
+function all_registered_mime_types(): table[Files::Tag] of set[string]
 	{
 	return mime_types;
 	}
@@ -451,7 +451,7 @@ function describe(f: fa_file): string
 	return handler$describe(f);
 	}
 
-event get_file_handle(tag: Analyzer::Tag, c: connection, is_orig: bool) &priority=5
+event get_file_handle(tag: Files::Tag, c: connection, is_orig: bool) &priority=5
 	{
 	if ( tag !in registered_protocols )
 		return;

scripts/base/init-bare.bro

@@ -440,6 +440,7 @@ type NetStats: record {
 	## packet capture system, this value may not be available and will then
 	## be always set to zero.
 	pkts_link:    count &default=0;
+	bytes_recvd:  count &default=0;	##< Bytes received by Bro.
 };
 
 ## Statistics about Bro's resource consumption.
@@ -2809,19 +2810,20 @@ export {
 module X509;
 export {
 	type Certificate: record {
-		version: count;	##< Version number.
+		version: count &log;	##< Version number.
-		serial: string;	##< Serial number.
+		serial: string &log;	##< Serial number.
-		subject: string;	##< Subject.
+		subject: string &log;	##< Subject.
-		issuer: string;	##< Issuer.
+		issuer: string &log;	##< Issuer.
-		not_valid_before: time;	##< Timestamp before when certificate is not valid.
+		cn: string &optional; ##< Last (most specific) common name.
-		not_valid_after: time;	##< Timestamp after when certificate is not valid.
+		not_valid_before: time &log;	##< Timestamp before when certificate is not valid.
-		key_alg: string;	##< Name of the key algorithm
+		not_valid_after: time &log;	##< Timestamp after when certificate is not valid.
-		sig_alg: string;	##< Name of the signature algorithm
+		key_alg: string &log;	##< Name of the key algorithm
-		key_type: string &optional;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
+		sig_alg: string &log;	##< Name of the signature algorithm
-		key_length: count &optional;	##< Key length in bits
+		key_type: string &optional &log;	##< Key type, if key parseable by openssl (either rsa, dsa or ec)
-		exponent: string &optional;	##< Exponent, if RSA-certificate
+		key_length: count &optional &log;	##< Key length in bits
-		curve: string &optional;	##< Curve, if EC-certificate
+		exponent: string &optional &log;	##< Exponent, if RSA-certificate
-	} &log;
+		curve: string &optional &log;	##< Curve, if EC-certificate
+	};
 
 	type Extension: record {
 		name: string;	##< Long name of extension. oid if name not known
@@ -3393,6 +3395,7 @@ const bits_per_uid: count = 96 &redef;
 
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
+@load base/frameworks/broker
 @load base/frameworks/logging
 @load base/frameworks/input
 @load base/frameworks/analyzer

scripts/base/protocols/conn/main.bro

@@ -62,6 +62,12 @@ export {
 		## field will be left empty at all times.
 		local_orig:   bool            &log &optional;
 
+		## If the connection is responded to locally, this value will be T.
+		## If it was responded to remotely it will be F.  In the case that
+		## the :bro:id:`Site::local_nets` variable is undefined, this
+		## field will be left empty at all times.
+		local_resp:   bool            &log &optional;
+
 		## Indicates the number of bytes missed in content gaps, which
 		## is representative of packet loss.  A value other than zero
 		## will normally cause protocol analysis to fail but some
@@ -201,7 +207,10 @@ function set_conn(c: connection, eoc: bool)
 		add c$conn$tunnel_parents[c$tunnel[|c$tunnel|-1]$uid];
 	c$conn$proto=get_port_transport_proto(c$id$resp_p);
 	if( |Site::local_nets| > 0 )
+		{
 		c$conn$local_orig=Site::is_local_addr(c$id$orig_h);
+		c$conn$local_resp=Site::is_local_addr(c$id$resp_h);
+		}
 
 	if ( eoc )
 		{

scripts/base/protocols/socks/main.bro

@@ -16,8 +16,10 @@ export {
 		id:          conn_id         &log;
 		## Protocol version of SOCKS.
 		version:     count           &log;
-		## Username for the proxy if extracted from the network.
+		## Username used to request a login to the proxy.
 		user:        string          &log &optional;
+		## Password used to request a login to the proxy.
+		password:    string          &log &optional;
 		## Server status for the attempt at using the proxy.
 		status:      string          &log &optional;
 		## Client requested SOCKS address. Could be an address, a name
@@ -91,3 +93,21 @@ event socks_reply(c: connection, version: count, reply: count, sa: SOCKS::Addres
 	if ( "SOCKS" in c$service )
 		Log::write(SOCKS::LOG, c$socks);
 	}
+
+event socks_login_userpass_request(c: connection, user: string, password: string) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5); 
+
+	c$socks$user = user;
+	c$socks$password = password;
+	}
+
+event socks_login_userpass_reply(c: connection, code: count) &priority=5
+	{
+	# Authentication only possible with the version 5.
+	set_session(c, 5);
+
+	c$socks$status = v5_status[code];
+	}
+

scripts/broxygen/__load__.bro

@@ -5,6 +5,7 @@
 @load frameworks/communication/listen.bro
 @load frameworks/control/controllee.bro
 @load frameworks/control/controller.bro
+@load frameworks/files/extract-all-files.bro
 @load policy/misc/dump-events.bro
 
 @load ./example.bro

scripts/policy/frameworks/files/extract-all-files.bro

@@ -0,0 +1,8 @@
+##! Extract all files to disk.
+
+@load base/files/extract
+
+event file_new(f: fa_file)
+	{
+	Files::add_analyzer(f, Files::ANALYZER_EXTRACT);
+	}

scripts/policy/frameworks/intel/seen/ssl.bro

@@ -10,3 +10,16 @@ event ssl_extension_server_name(c: connection, is_orig: bool, names: string_vec)
 		             $conn=c,
 		             $where=SSL::IN_SERVER_NAME]);
 	}
+
+event ssl_established(c: connection)
+	{
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 ||
+	     ! c$ssl$cert_chain[0]?$x509 )
+		return;
+
+	if ( c$ssl$cert_chain[0]$x509?$certificate && c$ssl$cert_chain[0]$x509$certificate?$cn )
+		Intel::seen([$indicator=c$ssl$cert_chain[0]$x509$certificate$cn,
+			$indicator_type=Intel::DOMAIN,
+			$conn=c,
+			$where=X509::IN_CERT]);
+	}

scripts/policy/frameworks/intel/seen/x509.bro

@@ -2,6 +2,18 @@
 @load base/files/x509
 @load ./where-locations
 
+event x509_ext_subject_alternative_name(f: fa_file, ext: X509::SubjectAlternativeName)
+	{
+	if ( ext?$dns )
+		{
+		for ( i in ext$dns )
+			Intel::seen([$indicator=ext$dns[i],
+				$indicator_type=Intel::DOMAIN,
+				$f=f,
+				$where=X509::IN_CERT]);
+		}
+	}
+
 event x509_certificate(f: fa_file, cert_ref: opaque of x509, cert: X509::Certificate)
 	{
 	if ( /emailAddress=/ in cert$subject )

scripts/policy/misc/stats.bro

@@ -39,6 +39,9 @@ export {
 		## Number of packets seen on the link since the last stats
 		## interval if reading live traffic.
 		pkts_link:     count     &log &optional;
+		## Number of bytes received since the last stats interval if
+		## reading live traffic.
+		bytes_recv:   count     &log &optional;
 	};
 
 	## Event to catch stats as they are written to the logging stream.
@@ -74,6 +77,7 @@ event check_stats(last_ts: time, last_ns: NetStats, last_res: bro_resources)
 		info$pkts_recv = ns$pkts_recvd - last_ns$pkts_recvd;
 		info$pkts_dropped = ns$pkts_dropped  - last_ns$pkts_dropped;
 		info$pkts_link = ns$pkts_link  - last_ns$pkts_link;
+		info$bytes_recv = ns$bytes_recvd  - last_ns$bytes_recvd;
 		}
 
 	Log::write(Stats::LOG, info);

scripts/policy/protocols/ssl/validate-certs.bro

@@ -33,6 +33,7 @@ event ssl_established(c: connection) &priority=3
 		return;
 
 	local chain_id = join_string_vec(c$ssl$cert_chain_fuids, ".");
+	local hash = c$ssl$cert_chain[0]$sha1;
 
 	local chain: vector of opaque of x509 = vector();
 	for ( i in c$ssl$cert_chain )
@@ -57,7 +58,7 @@ event ssl_established(c: connection) &priority=3
 		local message = fmt("SSL certificate validation failed with (%s)", c$ssl$validation_status);
 		NOTICE([$note=Invalid_Server_Cert, $msg=message,
 		        $sub=c$ssl$subject, $conn=c,
-		        $identifier=cat(c$id$resp_h,c$id$resp_p,c$ssl$validation_status)]);
+		        $identifier=cat(c$id$resp_h,c$id$resp_p,hash,c$ssl$validation_status)]);
 		}
 	}
 

scripts/policy/protocols/ssl/validate-ocsp.bro

@@ -34,9 +34,10 @@ event ssl_stapled_ocsp(c: connection, is_orig: bool, response: string) &priority
 
 event ssl_established(c: connection) &priority=3
 	{
-	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 || !c$ssl?$ocsp_response )
+	if ( ! c$ssl?$cert_chain || |c$ssl$cert_chain| == 0 || ! c$ssl$cert_chain[0]?$x509 || !c$ssl?$ocsp_response )
 		return;
 
+	local hash = c$ssl$cert_chain[0]$sha1;
 	local chain: vector of opaque of x509 = vector();
 	for ( i in c$ssl$cert_chain )
 		{

scripts/policy/protocols/ssl/weak-keys.bro

@@ -1,5 +1,5 @@
-##! Generate notices when SSL/TLS connections use certificates or DH parameters
+##! Generate notices when SSL/TLS connections use certificates, DH parameters,
-##! that have potentially unsafe key lengths.
+##! or cipher suites that are deemed to be insecure.
 
 @load base/protocols/ssl
 @load base/frameworks/notice
@@ -11,17 +11,20 @@ export {
 	redef enum Notice::Type += {
 		## Indicates that a server is using a potentially unsafe key.
 		Weak_Key,
+		## Indicates that a server is using a potentially unsafe version
+		Old_Version,
+		## Indicates that a server is using a potentially unsafe cipher
+		Weak_Cipher
 	};
 
-	## The category of hosts you would like to be notified about which have
+	## The category of hosts you would like to be notified about which are using weak
-	## certificates that are going to be expiring soon.  By default, these
+	## keys/ciphers/protocol_versions.  By default, these notices will be suppressed
-	## notices will be suppressed by the notice framework for 1 day after a particular
+	## by the notice framework for 1 day after a particular host has had a notice
-	## certificate has had a notice generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS,
+	## generated. Choices are: LOCAL_HOSTS, REMOTE_HOSTS, ALL_HOSTS, NO_HOSTS
-	## ALL_HOSTS, NO_HOSTS
 	const notify_weak_keys = LOCAL_HOSTS &redef;
 
 	## The minimal key length in bits that is considered to be safe. Any shorter
-	## (non-EC) key lengths will trigger the notice.
+	## (non-EC) key lengths will trigger a notice.
 	const notify_minimal_key_length = 2048 &redef;
 
 	## Warn if the DH key length is smaller than the certificate key length. This is
@@ -29,6 +32,17 @@ export {
 	## certificate key length. However, it is very common and cannot be avoided in some
 	## settings (e.g. with old jave clients).
 	const notify_dh_length_shorter_cert_length = T &redef;
+
+	## Warn if a server negotiates a SSL session with a protocol version smaller than
+	## the specified version. By default, the minimal version is TLSv10 because SSLv2
+	## and v3 have serious security issued.
+	## See https://tools.ietf.org/html/draft-thomson-sslv3-diediedie-00
+	## To disable, set to SSLv20
+	const tls_minimum_version = TLSv10 &redef;
+
+	## Warn if a server negotiates an unsafe cipher suite. By default, we only warn when
+	## encountering old export cipher suites, or RC4 (see RFC7465).
+	const unsafe_ciphers_regex = /(_EXPORT_)|(_RC4_)/ &redef;
 }
 
 # We check key lengths only for DSA or RSA certificates. For others, we do
@@ -43,6 +57,7 @@ event ssl_established(c: connection) &priority=3
 
 	local fuid = c$ssl$cert_chain_fuids[0];
 	local cert = c$ssl$cert_chain[0]$x509$certificate;
+	local hash = c$ssl$cert_chain[0]$sha1;
 
 	if ( !cert?$key_type || !cert?$key_length )
 		return;
@@ -56,7 +71,32 @@ event ssl_established(c: connection) &priority=3
 		NOTICE([$note=Weak_Key,
 			$msg=fmt("Host uses weak certificate with %d bit key", key_length),
 			$conn=c, $suppress_for=1day,
-			$identifier=cat(c$id$resp_h, c$id$resp_h, key_length)
+			$identifier=cat(c$id$resp_h, c$id$resp_h, hash, key_length)
+		]);
+	}
+
+# Check for old SSL versions and weak connection keys
+event ssl_server_hello(c: connection, version: count, possible_ts: time, server_random: string, session_id: string, cipher: count, comp_method: count) &priority=3
+	{
+	if ( ! addr_matches_host(c$id$resp_h, notify_weak_keys) )
+		return;
+
+	if ( version < tls_minimum_version )
+		{
+		local minimum_string = version_strings[tls_minimum_version];
+		local host_string = version_strings[version];
+		NOTICE([$note=Old_Version,
+			$msg=fmt("Host uses protocol version %s which is lower than the safe minimum %s", host_string, minimum_string),
+			$conn=c, $suppress_for=1day,
+			$identifier=cat(c$id$resp_h, c$id$resp_h)
+		]);
+		}
+
+	if ( unsafe_ciphers_regex in c$ssl$cipher )
+		NOTICE([$note=Weak_Cipher,
+			$msg=fmt("Host established connection using unsafe ciper suite %s", c$ssl$cipher),
+			$conn=c, $suppress_for=1day,
+			$identifier=cat(c$id$resp_h, c$id$resp_h, c$ssl$cipher)
 		]);
 	}
 

scripts/test-all-policy.bro

@@ -28,6 +28,7 @@
 @load frameworks/intel/seen/where-locations.bro
 @load frameworks/intel/seen/x509.bro
 @load frameworks/files/detect-MHR.bro
+#@load frameworks/files/extract-all-files.bro
 @load frameworks/files/hash-all-files.bro
 @load frameworks/packet-filter/shunt.bro
 @load frameworks/software/version-changes.bro

src/3rdparty

@@ -1 +1 @@
-Subproject commit 7e15efe9d28d46bfa662fcdd1cbb15ce1db285c9
+Subproject commit f2e34d731ed29bb993fbb065846faa342a8c824f

src/CMakeLists.txt

@@ -161,6 +161,14 @@ add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
 
+if ( ENABLE_BROKER )
+    add_subdirectory(broker)
+else ()
+    # Just to satisfy coverage unit tests until new Broker-based
+    # communication is enabled by default.
+    add_subdirectory(broker-dummy)
+endif ()
+
 set(bro_SUBDIRS
     # Order is important here.
     ${bro_PLUGIN_LIBS}
@@ -408,6 +416,18 @@ add_dependencies(bro bif_loader_plugins)
 # Install *.bif.bro.
 install(DIRECTORY ${CMAKE_BINARY_DIR}/scripts/base/bif DESTINATION ${BRO_SCRIPT_INSTALL_PATH}/base)
 
+# Create plugin directory at install time.
+install(DIRECTORY DESTINATION ${BRO_PLUGIN_INSTALL_PATH})
+
 # Make clean removes the bif directory.
 set_directory_properties(PROPERTIES ADDITIONAL_MAKE_CLEAN_FILES ${CMAKE_BINARY_DIR}/scripts/base/bif)
 
+# Remove some stale files and scripts that previous Bro versions put in
+# place, yet make confuse us now. This makes upgrading easier.
+install(CODE "
+   file(REMOVE_RECURSE
+       ${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/dataseries.bro
+       ${BRO_SCRIPT_INSTALL_PATH}/base/frameworks/logging/writers/elasticsearch.bro
+       ${BRO_SCRIPT_INSTALL_PATH}/policy/tuning/logs-to-elasticsearch.bro
+   )
+")

src/Conn.h

@@ -263,6 +263,9 @@ public:
 
 	void CheckFlowLabel(bool is_orig, uint32 flow_label);
 
+	uint32 GetOrigFlowLabel() { return orig_flow_label; }
+	uint32 GetRespFlowLabel() { return resp_flow_label; }
+
 protected:
 
 	Connection()	{ persistent = 0; }

src/DebugLogger.cc

@@ -19,7 +19,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "logging", 0, false }, {"input", 0, false },
 	{ "threading", 0, false }, { "file_analysis", 0, false },
 	{ "plugins", 0, false }, { "broxygen", 0, false },
-	{ "pktio", 0, false}
+	{ "pktio", 0, false }, { "broker", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -32,6 +32,7 @@ enum DebugStream {
 	DBG_PLUGINS,	// Plugin system
 	DBG_BROXYGEN,	// Broxygen
 	DBG_PKTIO,	// Packet sources and dumpers.
+	DBG_BROKER,	// Broker communication
 
 	NUM_DBGS // Has to be last
 };

src/EventHandler.cc

@@ -5,6 +5,11 @@
 #include "RemoteSerializer.h"
 #include "NetVar.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#include "broker/Data.h"
+#endif
+
 EventHandler::EventHandler(const char* arg_name)
 	{
 	name = copy_string(arg_name);
@@ -26,7 +31,12 @@ EventHandler::operator bool() const
 	{
 	return enabled && ((local && local->HasBodies())
 			   || receivers.length()
-			   || generate_always);
+			   || generate_always
+#ifdef ENABLE_BROKER
+			   || ! auto_remote_send.empty()
+			   // TODO: and require a subscriber interested in a topic or unsolicited flags?
+#endif
+	                   );
 	}
 
 FuncType* EventHandler::FType()
@@ -73,6 +83,46 @@ void EventHandler::Call(val_list* vl, bool no_remote)
 			SerialInfo info(remote_serializer);
 			remote_serializer->SendCall(&info, receivers[i], name, vl);
 			}
+
+#ifdef ENABLE_BROKER
+
+		if ( ! auto_remote_send.empty() )
+			{
+			// TODO: also short-circuit based on interested subscribers/flags?
+			broker::message msg;
+			msg.reserve(vl->length() + 1);
+			msg.emplace_back(Name());
+			bool valid_args = true;
+
+			for ( auto i = 0; i < vl->length(); ++i )
+				{
+				auto opt_data = bro_broker::val_to_data((*vl)[i]);
+
+				if ( opt_data )
+					msg.emplace_back(move(*opt_data));
+				else
+					{
+					valid_args = false;
+					auto_remote_send.clear();
+					reporter->Error("failed auto-remote event '%s', disabled",
+					                Name());
+					break;
+					}
+				}
+
+			if ( valid_args )
+				{
+				for ( auto it = auto_remote_send.begin();
+				      it != auto_remote_send.end(); ++it )
+					{
+					if ( std::next(it) == auto_remote_send.end() )
+						broker_mgr->Event(it->first, move(msg), it->second);
+					else
+						broker_mgr->Event(it->first, msg, it->second);
+					}
+				}
+			}
+#endif
 		}
 
 	if ( local )

src/EventHandler.h

@@ -4,7 +4,8 @@
 #define EVENTHANDLER
 
 #include <assert.h>
-
+#include <map>
+#include <string>
 #include "List.h"
 #include "BroList.h"
 
@@ -28,6 +29,18 @@ public:
 	void AddRemoteHandler(SourceID peer);
 	void RemoveRemoteHandler(SourceID peer);
 
+#ifdef ENABLE_BROKER
+	void AutoRemote(std::string topic, int flags)
+		{
+		auto_remote_send[std::move(topic)] = flags;
+		}
+
+	void AutoRemoteStop(const std::string& topic)
+		{
+		auto_remote_send.erase(topic);
+		}
+#endif
+
 	void Call(val_list* vl, bool no_remote = false);
 
 	// Returns true if there is at least one local or remote handler.
@@ -67,6 +80,10 @@ private:
 	declare(List, SourceID);
 	typedef List(SourceID) receiver_list;
 	receiver_list receivers;
+
+#ifdef ENABLE_BROKER
+	std::map<std::string, int> auto_remote_send;	// topic -> flags
+#endif
 };
 
 // Encapsulates a ptr to an event handler to overload the boolean operator.

src/Func.cc

@@ -54,6 +54,7 @@ const Expr* calling_expr = 0;
 bool did_builtin_init = false;
 
 vector<Func*> Func::unique_ids;
+static const std::pair<bool, Val*> empty_hook_result(false, NULL);
 
 Func::Func() : scope(0), type(0)
 	{
@@ -245,20 +246,31 @@ TraversalCode Func::Traverse(TraversalCallback* cb) const
 	HANDLE_TC_STMT_POST(tc);
 	}
 
-Val* Func::HandlePluginResult(Val* plugin_result, val_list* args, function_flavor flavor) const
+std::pair<bool, Val*> Func::HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const
 	{
-	// Helper function factoring out this code from BroFunc:Call() for better
+	// Helper function factoring out this code from BroFunc:Call() for
-	// readability.
+	// better readability.
+
+	if( ! plugin_result.first )
+		{
+		if( plugin_result.second )
+			reporter->InternalError("plugin set processed flag to false but actually returned a value");
+
+		// The plugin result hasn't been processed yet (read: fall
+		// into ::Call method).
+		return plugin_result;
+		}
 
 	switch ( flavor ) {
 	case FUNC_FLAVOR_EVENT:
-		Unref(plugin_result);
+		if( plugin_result.second )
-		plugin_result = 0;
+			reporter->InternalError("plugin returned non-void result for event %s", this->Name());
+
 		break;
 
 	case FUNC_FLAVOR_HOOK:
-		if ( plugin_result->Type()->Tag() != TYPE_BOOL )
+		if ( plugin_result.second->Type()->Tag() != TYPE_BOOL )
-			reporter->InternalError("plugin returned non-bool for hook");
+			reporter->InternalError("plugin returned non-bool for hook %s", this->Name());
 
 		break;
 
@@ -268,14 +280,14 @@ Val* Func::HandlePluginResult(Val* plugin_result, val_list* args, function_flavo
 
 		if ( (! yt) || yt->Tag() == TYPE_VOID )
 			{
-			Unref(plugin_result);
+			if( plugin_result.second )
-			plugin_result = 0;
+				reporter->InternalError("plugin returned non-void result for void method %s", this->Name());
 			}
 
-		else
+		else if ( plugin_result.second && plugin_result.second->Type()->Tag() != yt->Tag() && yt->Tag() != TYPE_ANY)
 			{
-			if ( plugin_result->Type()->Tag() != yt->Tag() )
+			reporter->InternalError("plugin returned wrong type (got %d, expecting %d) for %s",
-				reporter->InternalError("plugin returned wrong type for function call");
+						plugin_result.second->Type()->Tag(), yt->Tag(), this->Name());
 			}
 
 		break;
@@ -331,10 +343,15 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 	if ( sample_logger )
 		sample_logger->FunctionSeen(this);
 
-	Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
+	std::pair<bool, Val*> plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, parent, args), empty_hook_result);
 
-	if ( plugin_result )
+	plugin_result = HandlePluginResult(plugin_result, args, Flavor());
-		return HandlePluginResult(plugin_result, args, Flavor());
+
+	if( plugin_result.first )
+		{
+		Val *result = plugin_result.second;
+		return result;
+		}
 
 	if ( bodies.empty() )
 		{
@@ -425,11 +442,11 @@ Val* BroFunc::Call(val_list* args, Frame* parent) const
 	// Warn if the function returns something, but we returned from
 	// the function without an explicit return, or without a value.
 	else if ( FType()->YieldType() && FType()->YieldType()->Tag() != TYPE_VOID &&
-	     (flow != FLOW_RETURN /* we fell off the end */ ||
+		 (flow != FLOW_RETURN /* we fell off the end */ ||
-	      ! result /* explicit return with no result */) &&
+		  ! result /* explicit return with no result */) &&
-	     ! f->HasDelayed() )
+		 ! f->HasDelayed() )
 		reporter->Warning("non-void function returns without a value: %s",
-		                  Name());
+				  Name());
 
 	if ( result && g_trace_state.DoTrace() )
 		{
@@ -548,10 +565,15 @@ Val* BuiltinFunc::Call(val_list* args, Frame* parent) const
 	if ( sample_logger )
 		sample_logger->FunctionSeen(this);
 
-	Val* plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, args), 0);
+	std::pair<bool, Val*> plugin_result = PLUGIN_HOOK_WITH_RESULT(HOOK_CALL_FUNCTION, HookCallFunction(this, parent, args), empty_hook_result);
 
-	if ( plugin_result )
+	plugin_result = HandlePluginResult(plugin_result, args, FUNC_FLAVOR_FUNCTION);
-		return HandlePluginResult(plugin_result, args, FUNC_FLAVOR_FUNCTION);
+
+	if ( plugin_result.first )
+		{
+		Val *result = plugin_result.second;
+		return result;
+		}
 
 	if ( g_trace_state.DoTrace() )
 		{

src/Func.h

@@ -3,6 +3,8 @@
 #ifndef func_h
 #define func_h
 
+#include <utility>
+
 #include "BroList.h"
 #include "Obj.h"
 #include "Debug.h"
@@ -71,7 +73,7 @@ protected:
 	Func();
 
 	// Helper function for handling result of plugin hook.
-	Val* HandlePluginResult(Val* plugin_result, val_list* args, function_flavor flavor) const;
+	std::pair<bool, Val*> HandlePluginResult(std::pair<bool, Val*> plugin_result, val_list* args, function_flavor flavor) const;
 
 	DECLARE_ABSTRACT_SERIAL(Func);
 

src/Net.cc

@@ -34,6 +34,10 @@
 #include "iosource/PktDumper.h"
 #include "plugin/Manager.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#endif
+
 extern "C" {
 #include "setsignal.h"
 };
@@ -315,6 +319,11 @@ void net_run()
 			}
 #endif
 		current_iosrc = src;
+		bool communication_enabled = using_communication;
+
+#ifdef ENABLE_BROKER
+		communication_enabled |= broker_mgr->Enabled();
+#endif
 
 		if ( src )
 			src->Process();	// which will call net_packet_dispatch()
@@ -332,7 +341,7 @@ void net_run()
 				}
 			}
 
-		else if ( (have_pending_timers || using_communication) &&
+		else if ( (have_pending_timers || communication_enabled) &&
 			  ! pseudo_realtime )
 			{
 			// Take advantage of the lull to get up to
@@ -347,7 +356,7 @@ void net_run()
 			// us a lot of idle time, but doesn't delay near-term
 			// timers too much.  (Delaying them somewhat is okay,
 			// since Bro timers are not high-precision anyway.)
-			if ( ! using_communication )
+			if ( ! communication_enabled )
 				usleep(100000);
 			else
 				usleep(1000);

src/Reporter.cc

@@ -123,6 +123,19 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...)
 	throw InterpreterException();
 	}
 
+void Reporter::RuntimeError(const Location* location, const char* fmt, ...)
+	{
+	++errors;
+	PushLocation(location);
+	va_list ap;
+	va_start(ap, fmt);
+	FILE* out = errors_to_stderr ? stderr : 0;
+	DoLog("runtime error", reporter_error, out, 0, 0, true, true, "", fmt, ap);
+	va_end(ap);
+	PopLocation();
+	throw InterpreterException();
+	}
+
 void Reporter::InternalError(const char* fmt, ...)
 	{
 	va_list ap;

src/Reporter.h

@@ -73,6 +73,10 @@ public:
 	// function will not return but raise an InterpreterException.
 	void ExprRuntimeError(const Expr* expr, const char* fmt, ...);
 
+	// Report a runtime error in evaluating a Bro script expression. This
+	// function will not return but raise an InterpreterException.
+	void RuntimeError(const Location* location, const char* fmt, ...);
+
 	// Report a traffic weirdness, i.e., an unexpected protocol situation
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name);	// Raises net_weird().

src/SerialTypes.h

@@ -113,6 +113,8 @@ SERIAL_VAL(TOPK_VAL, 20)
 SERIAL_VAL(BLOOMFILTER_VAL, 21)
 SERIAL_VAL(CARDINALITY_VAL, 22)
 SERIAL_VAL(X509_VAL, 23)
+SERIAL_VAL(COMM_STORE_HANDLE_VAL, 24)
+SERIAL_VAL(COMM_DATA_VAL, 25)
 
 #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR)
 SERIAL_EXPR(EXPR, 1)
@@ -181,6 +183,7 @@ SERIAL_STMT(INIT_STMT, 17)
 SERIAL_STMT(NULL_STMT, 18)
 SERIAL_STMT(WHEN_STMT, 19)
 SERIAL_STMT(FALLTHROUGH_STMT, 20)
+SERIAL_STMT(WHILE_STMT, 21)
 
 #define SERIAL_TYPE(name, val) SERIAL_CONST(name, val, BRO_TYPE)
 SERIAL_TYPE(BRO_TYPE, 1)

src/Sessions.cc

@@ -466,6 +466,7 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 	id.src_addr = ip_hdr->SrcAddr();
 	id.dst_addr = ip_hdr->DstAddr();
 	Dictionary* d = 0;
+	BifEnum::Tunnel::Type tunnel_type = BifEnum::Tunnel::IP;
 
 	switch ( proto ) {
 	case IPPROTO_TCP:
@@ -606,6 +607,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		// Treat GRE tunnel like IP tunnels, fallthrough to logic below now
 		// that GRE header is stripped and only payload packet remains.
+		// The only thing different is the tunnel type enum value to use.
+		tunnel_type = BifEnum::Tunnel::GRE;
 		}
 
 	case IPPROTO_IPV4:
@@ -653,7 +656,8 @@ void NetSessions::DoNextPacket(double t, const struct pcap_pkthdr* hdr,
 
 		if ( it == ip_tunnels.end() )
 			{
-			EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr());
+			EncapsulatingConn ec(ip_hdr->SrcAddr(), ip_hdr->DstAddr(),
+			                     tunnel_type);
 			ip_tunnels[tunnel_idx] = TunnelActivity(ec, network_time);
 			timer_mgr->Add(new IPTunnelTimer(network_time, tunnel_idx));
 			}

src/Stats.cc

@@ -10,6 +10,10 @@
 #include "Trigger.h"
 #include "threading/Manager.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#endif
+
 int killed_by_inactivity = 0;
 
 uint64 tot_ack_events = 0;
@@ -222,6 +226,26 @@ void ProfileLogger::Log()
 			    ));
 		}
 
+#ifdef ENABLE_BROKER
+	auto cs = broker_mgr->ConsumeStatistics();
+
+	file->Write(fmt("%0.6f Comm: peers=%zu stores=%zu "
+	                "store_queries=%zu store_responses=%zu "
+	                "outgoing_conn_status=%zu incoming_conn_status=%zu "
+	                "reports=%zu\n",
+	                network_time, cs.outgoing_peer_count, cs.data_store_count,
+	                cs.pending_query_count, cs.response_count,
+	                cs.outgoing_conn_status_count, cs.incoming_conn_status_count,
+	                cs.report_count));
+
+	for ( const auto& s : cs.print_count )
+		file->Write(fmt("    %-25s prints dequeued=%zu\n", s.first.data(), s.second));
+	for ( const auto& s : cs.event_count )
+		file->Write(fmt("    %-25s events dequeued=%zu\n", s.first.data(), s.second));
+	for ( const auto& s : cs.log_count )
+		file->Write(fmt("    %-25s logs dequeued=%zu\n", s.first.data(), s.second));
+#endif
+
 	// Script-level state.
 	unsigned int size, mem = 0;
 	PDict(ID)* globals = global_scope()->Vars();

src/Stmt.cc

@@ -23,7 +23,7 @@ const char* stmt_name(BroStmtTag t)
 		"print", "event", "expr", "if", "when", "switch",
 		"for", "next", "break", "return", "add", "delete",
 		"list", "bodylist",
-		"<init>", "fallthrough",
+		"<init>", "fallthrough", "while",
 		"null",
 	};
 
@@ -1127,6 +1127,126 @@ bool EventStmt::DoUnserialize(UnserialInfo* info)
 	return event_expr != 0;
 	}
 
+WhileStmt::WhileStmt(Expr* arg_loop_condition, Stmt* arg_body)
+	: loop_condition(arg_loop_condition), body(arg_body)
+	{
+	if ( ! loop_condition->IsError() &&
+	     ! IsBool(loop_condition->Type()->Tag()) )
+		loop_condition->Error("while conditional must be boolean");
+	}
+
+WhileStmt::~WhileStmt()
+	{
+	Unref(loop_condition);
+	Unref(body);
+	}
+
+int WhileStmt::IsPure() const
+	{
+	return loop_condition->IsPure() && body->IsPure();
+	}
+
+void WhileStmt::Describe(ODesc* d) const
+	{
+	Stmt::Describe(d);
+
+	if ( d->IsReadable() )
+		d->Add("(");
+
+	loop_condition->Describe(d);
+
+	if ( d->IsReadable() )
+		d->Add(")");
+
+	d->SP();
+	d->PushIndent();
+	body->AccessStats(d);
+	body->Describe(d);
+	d->PopIndent();
+	}
+
+TraversalCode WhileStmt::Traverse(TraversalCallback* cb) const
+	{
+	TraversalCode tc = cb->PreStmt(this);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = loop_condition->Traverse(cb);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = body->Traverse(cb);
+	HANDLE_TC_STMT_PRE(tc);
+
+	tc = cb->PostStmt(this);
+	HANDLE_TC_STMT_POST(tc);
+	}
+
+Val* WhileStmt::Exec(Frame* f, stmt_flow_type& flow) const
+	{
+	RegisterAccess();
+	flow = FLOW_NEXT;
+	Val* rval = 0;
+
+	for ( ; ; )
+		{
+		Val* cond = loop_condition->Eval(f);
+
+		if ( ! cond )
+			break;
+
+		bool cont = cond->AsBool();
+		Unref(cond);
+
+		if ( ! cont )
+			break;
+
+		flow = FLOW_NEXT;
+		rval = body->Exec(f, flow);
+
+		if ( flow == FLOW_BREAK || flow == FLOW_RETURN )
+			break;
+		}
+
+	if ( flow == FLOW_LOOP || flow == FLOW_BREAK )
+		flow = FLOW_NEXT;
+
+	return rval;
+	}
+
+Stmt* WhileStmt::Simplify()
+	{
+	loop_condition = simplify_expr(loop_condition, SIMPLIFY_GENERAL);
+
+	if ( loop_condition->IsConst() && loop_condition->IsZero() )
+		return new NullStmt();
+
+	body = simplify_stmt(body);
+	return this;
+	}
+
+IMPLEMENT_SERIAL(WhileStmt, SER_WHILE_STMT);
+
+bool WhileStmt::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_WHILE_STMT, Stmt);
+
+	if ( ! loop_condition->Serialize(info) )
+		return false;
+
+	return body->Serialize(info);
+	}
+
+bool WhileStmt::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(Stmt);
+	loop_condition = Expr::Unserialize(info);
+
+	if ( ! loop_condition )
+		return false;
+
+	body = Stmt::Unserialize(info);
+	return body != 0;
+	}
+
 ForStmt::ForStmt(id_list* arg_loop_vars, Expr* loop_expr)
 : ExprStmt(STMT_FOR, loop_expr)
 	{

src/Stmt.h

@@ -310,6 +310,33 @@ protected:
 	EventExpr* event_expr;
 };
 
+class WhileStmt : public Stmt {
+public:
+
+	WhileStmt(Expr* loop_condition, Stmt* body);
+	~WhileStmt();
+
+	int IsPure() const;
+
+	void Describe(ODesc* d) const;
+
+	TraversalCode Traverse(TraversalCallback* cb) const;
+
+protected:
+	friend class Stmt;
+
+	WhileStmt()
+		{ loop_condition = 0; body = 0; }
+
+	Val* Exec(Frame* f, stmt_flow_type& flow) const;
+	Stmt* Simplify();
+
+	DECLARE_SERIAL(WhileStmt);
+
+	Expr* loop_condition;
+	Stmt* body;
+};
+
 class ForStmt : public ExprStmt {
 public:
 	ForStmt(id_list* loop_vars, Expr* loop_expr);

src/StmtEnums.h

@@ -17,6 +17,7 @@ typedef enum {
 	STMT_LIST, STMT_EVENT_BODY_LIST,
 	STMT_INIT,
 	STMT_FALLTHROUGH,
+	STMT_WHILE,
 	STMT_NULL
 #define NUM_STMTS (int(STMT_NULL) + 1)
 } BroStmtTag;

src/Trigger.cc

@@ -112,6 +112,7 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 	attached = 0;
 	is_return = arg_is_return;
 	location = arg_location;
+	timeout_value = -1;
 
 	++total_triggers;
 
@@ -133,17 +134,22 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 
 	Val* timeout_val = arg_timeout ? arg_timeout->Eval(arg_frame) : 0;
 
+	if ( timeout_val )
+		{
+		Unref(timeout_val);
+		timeout_value = timeout_val->AsInterval();
+		}
+
 	// Make sure we don't get deleted if somebody calls a method like
 	// Timeout() while evaluating the trigger. 
 	Ref(this);
 
-	if ( ! Eval() && timeout_val )
+	if ( ! Eval() && timeout_value >= 0 )
 		{
-		timer = new TriggerTimer(timeout_val->AsInterval(), this);
+		timer = new TriggerTimer(timeout_value, this);
 		timer_mgr->Add(timer);
 		}
 
-	Unref(timeout_val);
 	Unref(this);
 	}
 

src/Trigger.h

@@ -32,6 +32,10 @@ public:
 	// Executes timeout code and deletes the object.
 	void Timeout();
 
+	// Return the timeout interval (negative if none was specified).
+	double TimeoutValue() const
+		{ return timeout_value; }
+
 	// Called if another entity needs to complete its operations first
 	// in any case before this trigger can proceed.
 	void Hold()	{ delayed = true; }
@@ -51,6 +55,8 @@ public:
 	// may not immediately delete it as other references may still exist.
 	void Disable();
 
+	bool Disabled() const { return disabled; }
+
 	virtual void Describe(ODesc* d) const { d->Add("<trigger>"); }
 
 	// Overidden from Notifier.  We queue the trigger and evaluate it
@@ -87,6 +93,7 @@ private:
 	Stmt* body;
 	Stmt* timeout_stmts;
 	Expr* timeout;
+	double timeout_value;
 	Frame* frame;
 	bool is_return;
 	const Location* location;

src/TunnelEncapsulation.h

@@ -37,10 +37,12 @@ public:
 	 *
 	 * @param s The tunnel source address, likely taken from an IP header.
 	 * @param d The tunnel destination address, likely taken from an IP header.
+	 * @param t The type of IP tunnel.
 	 */
-	EncapsulatingConn(const IPAddr& s, const IPAddr& d)
+	EncapsulatingConn(const IPAddr& s, const IPAddr& d,
+	                  BifEnum::Tunnel::Type t = BifEnum::Tunnel::IP)
 		: src_addr(s), dst_addr(d), src_port(0), dst_port(0),
-		  proto(TRANSPORT_UNKNOWN), type(BifEnum::Tunnel::IP),
+		  proto(TRANSPORT_UNKNOWN), type(t),
 		  uid(Bro::UID(bits_per_uid))
 		{
 		}
@@ -85,7 +87,8 @@ public:
 		if ( ec1.type != ec2.type )
 			return false;
 
-		if ( ec1.type == BifEnum::Tunnel::IP )
+		if ( ec1.type == BifEnum::Tunnel::IP ||
+		     ec1.type == BifEnum::Tunnel::GRE )
 			// Reversing endpoints is still same tunnel.
 			return ec1.uid == ec2.uid && ec1.proto == ec2.proto &&
 			  ((ec1.src_addr == ec2.src_addr && ec1.dst_addr == ec2.dst_addr) ||

src/analyzer/protocol/socks/SOCKS.cc

@@ -57,8 +57,7 @@ void SOCKS_Analyzer::DeliverStream(int len, const u_char* data, bool orig)
 		// with the rest of the conneciton.
 		//
 		// Note that we assume that no payload data arrives before both endpoints
-		// are done with there part of the SOCKS protocol.
+		// are done with their part of the SOCKS protocol.
-
 		if ( ! pia )
 			{
 			pia = new pia::PIA_TCP(Conn());

src/analyzer/protocol/socks/events.bif

@@ -27,3 +27,19 @@ event socks_request%(c: connection, version: count, request_type: count, sa: SOC
 ## p: The destination port for the proxied traffic.
 event socks_reply%(c: connection, version: count, reply: count, sa: SOCKS::Address, p: port%);
 
+## Generated when a SOCKS client performs username and password based login.
+##
+## c: The parent connection of the proxy.
+##
+## user: The given username.
+##
+## password: The given password.
+event socks_login_userpass_request%(c: connection, user: string, password: string%);
+
+## Generated when a SOCKS server replies to a username/password login attempt.
+##
+## c: The parent connection of the proxy.
+## 
+## code: The response code for the attempted login.
+event socks_login_userpass_reply%(c: connection, code: count%);
+

src/analyzer/protocol/socks/socks-analyzer.pac

@@ -148,6 +148,37 @@ refine connection SOCKS_Conn += {
 		return true;
 		%}
 
+	function socks5_auth_request_userpass(request: SOCKS5_Auth_Request_UserPass_v1): bool
+		%{
+		StringVal* user = new StringVal(${request.username}.length(), (const char*) ${request.username}.begin());
+		StringVal* pass = new StringVal(${request.password}.length(), (const char*) ${request.password}.begin());
+		
+		BifEvent::generate_socks_login_userpass_request(bro_analyzer(),
+		                                                bro_analyzer()->Conn(),
+		                                                user, pass);
+		return true;
+		%}
+
+	function socks5_unsupported_authentication_method(auth_method: uint8): bool
+		%{
+		reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_method_%d", auth_method));
+		return true;
+		%}
+
+	function socks5_unsupported_authentication_version(auth_method: uint8, version: uint8): bool
+		%{
+		reporter->Weird(bro_analyzer()->Conn(), fmt("socks5_unsupported_authentication_%d_%d", auth_method, version));
+		return true;
+		%}
+	
+	function socks5_auth_reply_userpass(reply: SOCKS5_Auth_Reply_UserPass_v1): bool
+		%{
+		BifEvent::generate_socks_login_userpass_reply(bro_analyzer(),
+		                                              bro_analyzer()->Conn(),
+		                                              ${reply.code});
+		return true;
+		%}
+
 	function version_error(version: uint8): bool
 		%{
 		bro_analyzer()->ProtocolViolation(fmt("unsupported/unknown SOCKS version %d", version));
@@ -176,3 +207,22 @@ refine typeattr SOCKS5_Request += &let {
 refine typeattr SOCKS5_Reply += &let {
 	proc: bool = $context.connection.socks5_reply(this);
 };
+
+refine typeattr SOCKS5_Auth_Negotiation_Reply += &let {
+};
+
+refine typeattr SOCKS5_Auth_Request_UserPass_v1 += &let {
+	proc: bool = $context.connection.socks5_auth_request_userpass(this);
+};
+
+refine typeattr SOCKS5_Auth_Reply_UserPass_v1 += &let {
+	proc: bool = $context.connection.socks5_auth_reply_userpass(this);
+};
+
+refine typeattr SOCKS5_Unsupported_Authentication_Method += &let {
+	proc: bool = $context.connection.socks5_unsupported_authentication_method($context.connection.v5_auth_method());
+};
+
+refine typeattr SOCKS5_Unsupported_Authentication_Version += &let {
+	proc: bool = $context.connection.socks5_unsupported_authentication_version($context.connection.v5_auth_method(), version);
+};

src/analyzer/protocol/socks/socks-protocol.pac

@@ -1,10 +1,15 @@
 
+type SOCKS_Message(is_orig: bool) = case $context.connection.v5_in_auth_sub_negotiation() of {
+	true  -> auth: SOCKS5_Auth_Message(is_orig);
+	false -> msg:  SOCKS_Version(is_orig);
+};
+
 type SOCKS_Version(is_orig: bool) = record {
 	version: uint8;
 	msg:     case version of {
-		4       -> socks4_msg:     SOCKS4_Message(is_orig);
+		4       -> socks4_msg:      SOCKS4_Message(is_orig);
-		5       -> socks5_msg:     SOCKS5_Message(is_orig);
+		5       -> socks5_msg:      SOCKS5_Message(is_orig);
-		default -> socks_msg_fail: SOCKS_Version_Error(version);
+		default -> socks_msg_fail:  SOCKS_Version_Error(version);
 	};
 };
 
@@ -14,10 +19,11 @@ type SOCKS_Version_Error(version: uint8) = record {
 
 # SOCKS5 Implementation
 type SOCKS5_Message(is_orig: bool) = case $context.connection.v5_past_authentication() of {
-	true  -> msg:  SOCKS5_Real_Message(is_orig);
 	false -> auth: SOCKS5_Auth_Negotiation(is_orig);
+	true  -> msg:  SOCKS5_Real_Message(is_orig);
 };
 
+
 type SOCKS5_Auth_Negotiation(is_orig: bool) = case is_orig of {
 	true  -> req:  SOCKS5_Auth_Negotiation_Request;
 	false -> rep:  SOCKS5_Auth_Negotiation_Reply;
@@ -31,7 +37,61 @@ type SOCKS5_Auth_Negotiation_Request = record {
 type SOCKS5_Auth_Negotiation_Reply = record {
 	selected_auth_method: uint8;
 } &let {
+	in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(selected_auth_method == 0 || selected_auth_method == 0xff ? false : true);
 	past_auth = $context.connection.set_v5_past_authentication();
+	set_auth = $context.connection.set_v5_auth_method(selected_auth_method);
+};
+
+type SOCKS5_Auth_Message(is_orig: bool) = case is_orig of {
+	true  -> req: SOCKS5_Auth_Request;
+	false -> rep: SOCKS5_Auth_Reply;
+};
+
+type SOCKS5_Auth_Request = case $context.connection.v5_auth_method() of {
+	0x02    -> userpass    : SOCKS5_Auth_Request_UserPass;
+	default -> unsupported : SOCKS5_Unsupported_Authentication_Method;
+};
+
+type SOCKS5_Unsupported_Authentication_Method = record {
+	crap: bytestring &restofdata;
+};
+
+type SOCKS5_Unsupported_Authentication_Version(version: uint8) = record {
+	crap: bytestring &restofdata;
+};
+
+type SOCKS5_Auth_Request_UserPass = record {
+	version: uint8;
+	msg:     case version of {
+		1       -> v1:           SOCKS5_Auth_Request_UserPass_v1;
+		default -> unsupported:  SOCKS5_Unsupported_Authentication_Version(version);
+	};
+};
+
+type SOCKS5_Auth_Request_UserPass_v1 = record {
+	ulen     : uint8;
+	username : bytestring &length=ulen;
+	plen     : uint8;
+	password : bytestring &length=plen;
+};
+
+type SOCKS5_Auth_Reply = case $context.connection.v5_auth_method() of {
+	0x02    -> userpass    : SOCKS5_Auth_Reply_UserPass;
+	default -> unsupported : SOCKS5_Unsupported_Authentication_Method;
+} &let {
+	in_auth_sub_neg = $context.connection.set_v5_in_auth_sub_negotiation(false);
+};
+
+type SOCKS5_Auth_Reply_UserPass = record {
+	version: uint8;
+	msg:     case version of {
+		1       -> v1:           SOCKS5_Auth_Reply_UserPass_v1;
+		default -> unsupported:  SOCKS5_Unsupported_Authentication_Version(version);
+	};
+};
+
+type SOCKS5_Auth_Reply_UserPass_v1 = record {
+	code : uint8;
 };
 
 type SOCKS5_Real_Message(is_orig: bool) = case is_orig of {
@@ -55,10 +115,10 @@ type SOCKS5_Address = record {
 } &byteorder = bigendian;
 
 type SOCKS5_Request = record {
-	command: uint8;
+	command     : uint8;
-	reserved: uint8;
+	reserved    : uint8;
-	remote_name: SOCKS5_Address;
+	remote_name : SOCKS5_Address;
-	port: uint16;
+	port        : uint16;
 } &byteorder = bigendian;
 
 type SOCKS5_Reply = record {
@@ -98,13 +158,28 @@ type SOCKS4_Reply = record {
 
 refine connection SOCKS_Conn += {
 	%member{
+		bool v5_in_auth_sub_negotiation_;
 		bool v5_authenticated_;
+		uint8 selected_auth_method_;
 	%}
 
 	%init{
+		v5_in_auth_sub_negotiation_ = false;
 		v5_authenticated_ = false;
+		selected_auth_method_ = 255;
 	%}
 
+	function v5_in_auth_sub_negotiation(): bool
+		%{
+		return v5_in_auth_sub_negotiation_;
+		%}
+
+	function set_v5_in_auth_sub_negotiation(b: bool): bool
+		%{
+		v5_in_auth_sub_negotiation_ = b;
+		return true;
+		%}
+
 	function v5_past_authentication(): bool
 		%{
 		return v5_authenticated_;
@@ -115,5 +190,16 @@ refine connection SOCKS_Conn += {
 		v5_authenticated_ = true;
 		return true;
 		%}
+
+	function set_v5_auth_method(method: uint8): bool
+		%{
+		selected_auth_method_ = method;
+		return true;
+		%}
+
+	function v5_auth_method(): uint8
+		%{
+		return selected_auth_method_;
+		%}
 };
 

src/analyzer/protocol/socks/socks.pac

@@ -20,7 +20,7 @@ connection SOCKS_Conn(bro_analyzer: BroAnalyzer) {
 %include socks-protocol.pac
 
 flow SOCKS_Flow(is_orig: bool) {
-	datagram = SOCKS_Version(is_orig) withcontext(connection, this);
+	datagram = SOCKS_Message(is_orig) withcontext(connection, this);
 };
 
 %include socks-analyzer.pac

src/analyzer/protocol/ssl/ssl-analyzer.pac

@@ -207,7 +207,7 @@ refine connection SSL_Conn += {
 			{
 			// This should be impossible due to the binpac parser
 			// and protocol description
-			bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %lu", length));
+			bro_analyzer()->ProtocolViolation(fmt("Impossible extension length: %zu", length));
 			bro_analyzer()->SetSkip(true);
 			return true;
 			}

src/bro.bif

@@ -1675,6 +1675,7 @@ function net_stats%(%): NetStats
 	unsigned int recv = 0;
 	unsigned int drop = 0;
 	unsigned int link = 0;
+	unsigned int bytes_recv = 0;
 
 	const iosource::Manager::PktSrcList& pkt_srcs(iosource_mgr->GetPktSrcs());
 
@@ -1688,12 +1689,14 @@ function net_stats%(%): NetStats
 		recv += stat.received;
 		drop += stat.dropped;
 		link += stat.link;
+		bytes_recv += stat.bytes_received;
 		}
 
 	RecordVal* ns = new RecordVal(net_stats);
 	ns->Assign(0, new Val(recv, TYPE_COUNT));
 	ns->Assign(1, new Val(drop, TYPE_COUNT));
 	ns->Assign(2, new Val(link, TYPE_COUNT));
+	ns->Assign(3, new Val(bytes_recv, TYPE_COUNT));
 
 	return ns;
 	%}

src/broker-dummy/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Placeholder for Broker-based communication functionality, not enabled
+# by default.  This helps satisfy coverage unit tests pass regardless of
+# whether Broker is enabled or not.
+
+include(BroSubdir)
+
+bif_target(comm.bif)
+bif_target(data.bif)
+bif_target(messaging.bif)
+bif_target(store.bif)
+
+bro_add_subdir_library(broker_dummy ${BIF_OUTPUT_CC})
+add_dependencies(bro_broker_dummy generate_outputs)

src/broker-dummy/comm.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default.

src/broker-dummy/data.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default

src/broker-dummy/messaging.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default

src/broker-dummy/store.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default

src/broker/CMakeLists.txt

@@ -0,0 +1,28 @@
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+if ( ROCKSDB_INCLUDE_DIR )
+    add_definitions(-DHAVE_ROCKSDB)
+    include_directories(BEFORE ${ROCKSDB_INCLUDE_DIR})
+endif ()
+
+include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_CORE})
+include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_IO})
+
+set(comm_SRCS
+    Data.cc
+    Manager.cc
+    Store.cc
+)
+
+bif_target(comm.bif)
+bif_target(data.bif)
+bif_target(messaging.bif)
+bif_target(store.bif)
+
+bro_add_subdir_library(brokercomm ${comm_SRCS} ${BIF_OUTPUT_CC})
+add_dependencies(bro_brokercomm generate_outputs)

src/broker/Data.cc

@@ -0,0 +1,708 @@
+#include "Data.h"
+#include "broker/data.bif.h"
+#include <caf/binary_serializer.hpp>
+#include <caf/binary_deserializer.hpp>
+
+using namespace std;
+
+OpaqueType* bro_broker::opaque_of_data_type;
+OpaqueType* bro_broker::opaque_of_set_iterator;
+OpaqueType* bro_broker::opaque_of_table_iterator;
+OpaqueType* bro_broker::opaque_of_vector_iterator;
+OpaqueType* bro_broker::opaque_of_record_iterator;
+
+static broker::port::protocol to_broker_port_proto(TransportProto tp)
+	{
+	switch ( tp ) {
+	case TRANSPORT_TCP:
+		return broker::port::protocol::tcp;
+	case TRANSPORT_UDP:
+		return broker::port::protocol::udp;
+	case TRANSPORT_ICMP:
+		return broker::port::protocol::icmp;
+	case TRANSPORT_UNKNOWN:
+	default:
+		return broker::port::protocol::unknown;
+	}
+	}
+
+TransportProto bro_broker::to_bro_port_proto(broker::port::protocol tp)
+	{
+	switch ( tp ) {
+	case broker::port::protocol::tcp:
+		return TRANSPORT_TCP;
+	case broker::port::protocol::udp:
+		return TRANSPORT_UDP;
+	case broker::port::protocol::icmp:
+		return TRANSPORT_ICMP;
+	case broker::port::protocol::unknown:
+	default:
+		return TRANSPORT_UNKNOWN;
+	}
+	}
+
+struct val_converter {
+	using result_type = Val*;
+
+	BroType* type;
+	bool require_log_attr;
+
+	result_type operator()(bool a)
+		{
+		if ( type->Tag() == TYPE_BOOL )
+			return new Val(a, TYPE_BOOL);
+		return nullptr;
+		}
+
+	result_type operator()(uint64_t a)
+		{
+		if ( type->Tag() == TYPE_COUNT )
+			return new Val(a, TYPE_COUNT);
+		if ( type->Tag() == TYPE_COUNTER )
+			return new Val(a, TYPE_COUNTER);
+		return nullptr;
+		}
+
+	result_type operator()(int64_t a)
+		{
+		if ( type->Tag() == TYPE_INT )
+			return new Val(a, TYPE_INT);
+		return nullptr;
+		}
+
+	result_type operator()(double a)
+		{
+		if ( type->Tag() == TYPE_DOUBLE )
+			return new Val(a, TYPE_DOUBLE);
+		return nullptr;
+		}
+
+	result_type operator()(std::string& a)
+		{
+		switch ( type->Tag() ) {
+		case TYPE_STRING:
+			return new StringVal(a.size(), a.data());
+		case TYPE_FILE:
+			{
+			auto file = BroFile::GetFile(a.data());
+
+			if ( file )
+				{
+				Ref(file);
+				return new Val(file);
+				}
+
+			return nullptr;
+			}
+		case TYPE_FUNC:
+			{
+			auto id = lookup_ID(a.data(), GLOBAL_MODULE_NAME);
+			auto rval = id ? id->ID_Val() : nullptr;
+			Unref(id);
+
+			if ( rval && rval->Type()->Tag() == TYPE_FUNC )
+				return rval;
+
+			return nullptr;
+			}
+		default:
+			return nullptr;
+		}
+		}
+
+	result_type operator()(broker::address& a)
+		{
+		if ( type->Tag() == TYPE_ADDR )
+			{
+			auto bits = reinterpret_cast<const in6_addr*>(&a.bytes());
+			return new AddrVal(IPAddr(*bits));
+			}
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::subnet& a)
+		{
+		if ( type->Tag() == TYPE_SUBNET )
+			{
+			auto bits = reinterpret_cast<const in6_addr*>(&a.network().bytes());
+			return new SubNetVal(IPPrefix(IPAddr(*bits), a.length()));
+			}
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::port& a)
+		{
+		if ( type->Tag() == TYPE_PORT )
+			return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type()));
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::time_point& a)
+		{
+		if ( type->Tag() == TYPE_TIME )
+			return new Val(a.value, TYPE_TIME);
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::time_duration& a)
+		{
+		if ( type->Tag() == TYPE_INTERVAL )
+			return new Val(a.value, TYPE_INTERVAL);
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::enum_value& a)
+		{
+		if ( type->Tag() == TYPE_ENUM )
+			{
+			auto etype = type->AsEnumType();
+			auto i = etype->Lookup(GLOBAL_MODULE_NAME, a.name.data());
+
+			if ( i == -1 )
+				return nullptr;
+
+			return new EnumVal(i, etype);
+			}
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::set& a)
+		{
+		if ( ! type->IsSet() )
+			return nullptr;
+
+		auto tt = type->AsTableType();
+		auto rval = new TableVal(tt);
+
+		for ( auto& item : a )
+			{
+			broker::vector composite_key;
+			auto indices = broker::get<broker::vector>(item);
+
+			if ( ! indices )
+				{
+				composite_key.emplace_back(move(item));
+				indices = &composite_key;
+				}
+
+			auto expected_index_types = tt->Indices()->Types();
+
+			if ( static_cast<size_t>(expected_index_types->length()) !=
+			     indices->size() )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			auto list_val = new ListVal(TYPE_ANY);
+
+			for ( auto i = 0u; i < indices->size(); ++i )
+				{
+				auto index_val = bro_broker::data_to_val(move((*indices)[i]),
+				                                         (*expected_index_types)[i]);
+
+				if ( ! index_val )
+					{
+					Unref(rval);
+					Unref(list_val);
+					return nullptr;
+					}
+
+				list_val->Append(index_val);
+				}
+
+
+			rval->Assign(list_val, nullptr);
+			Unref(list_val);
+			}
+
+		return rval;
+		}
+
+	result_type operator()(broker::table& a)
+		{
+		if ( ! type->IsTable() )
+			return nullptr;
+
+		auto tt = type->AsTableType();
+		auto rval = new TableVal(tt);
+
+		for ( auto& item : a )
+			{
+			broker::vector composite_key;
+			auto indices = broker::get<broker::vector>(item.first);
+
+			if ( ! indices )
+				{
+				composite_key.emplace_back(move(item.first));
+				indices = &composite_key;
+				}
+
+			auto expected_index_types = tt->Indices()->Types();
+
+			if ( static_cast<size_t>(expected_index_types->length()) !=
+			     indices->size() )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			auto list_val = new ListVal(TYPE_ANY);
+
+			for ( auto i = 0u; i < indices->size(); ++i )
+				{
+				auto index_val = bro_broker::data_to_val(move((*indices)[i]),
+				                                         (*expected_index_types)[i]);
+
+				if ( ! index_val )
+					{
+					Unref(rval);
+					Unref(list_val);
+					return nullptr;
+					}
+
+				list_val->Append(index_val);
+				}
+
+			auto value_val = bro_broker::data_to_val(move(item.second),
+			                                         tt->YieldType());
+
+			if ( ! value_val )
+				{
+				Unref(rval);
+				Unref(list_val);
+				return nullptr;
+				}
+
+			rval->Assign(list_val, value_val);
+			Unref(list_val);
+			}
+
+		return rval;
+		}
+
+	result_type operator()(broker::vector& a)
+		{
+		if ( type->Tag() != TYPE_VECTOR )
+			return nullptr;
+
+		auto vt = type->AsVectorType();
+		auto rval = new VectorVal(vt);
+
+		for ( auto& item : a )
+			{
+			auto item_val = bro_broker::data_to_val(move(item), vt->YieldType());
+
+			if ( ! item_val )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			rval->Assign(rval->Size(), item_val);
+			}
+
+		return rval;
+		}
+
+	result_type operator()(broker::record& a)
+		{
+		if ( type->Tag() != TYPE_RECORD )
+			return nullptr;
+
+		auto rt = type->AsRecordType();
+		auto rval = new RecordVal(rt);
+
+		for ( auto i = 0u; i < static_cast<size_t>(rt->NumFields()); ++i )
+			{
+			if ( require_log_attr && ! rt->FieldDecl(i)->FindAttr(ATTR_LOG) )
+				continue;
+
+			if ( i >= a.fields.size() )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			if ( ! a.fields[i] )
+				{
+				rval->Assign(i, nullptr);
+				continue;
+				}
+
+			auto item_val = bro_broker::data_to_val(move(*a.fields[i]),
+			                                        rt->FieldType(i));
+
+			if ( ! item_val )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			rval->Assign(i, item_val);
+			}
+
+		return rval;
+		}
+};
+
+Val* bro_broker::data_to_val(broker::data d, BroType* type, bool require_log_attr)
+	{
+	return broker::visit(val_converter{type, require_log_attr}, d);
+	}
+
+broker::util::optional<broker::data> bro_broker::val_to_data(Val* v)
+	{
+	switch ( v->Type()->Tag() ) {
+	case TYPE_BOOL:
+		return {v->AsBool()};
+	case TYPE_INT:
+		return {v->AsInt()};
+	case TYPE_COUNT:
+		return {v->AsCount()};
+	case TYPE_COUNTER:
+		return {v->AsCounter()};
+	case TYPE_PORT:
+		{
+		auto p = v->AsPortVal();
+		return {broker::port(p->Port(), to_broker_port_proto(p->PortType()))};
+		}
+	case TYPE_ADDR:
+		{
+		auto a = v->AsAddr();
+		in6_addr tmp;
+		a.CopyIPv6(&tmp);
+		return {broker::address(reinterpret_cast<const uint32_t*>(&tmp),
+			                    broker::address::family::ipv6,
+			                    broker::address::byte_order::network)};
+		}
+		break;
+	case TYPE_SUBNET:
+		{
+		auto s = v->AsSubNet();
+		in6_addr tmp;
+		s.Prefix().CopyIPv6(&tmp);
+		auto a = broker::address(reinterpret_cast<const uint32_t*>(&tmp),
+		                         broker::address::family::ipv6,
+		                         broker::address::byte_order::network);
+		return {broker::subnet(a, s.Length())};
+		}
+		break;
+	case TYPE_DOUBLE:
+		return {v->AsDouble()};
+	case TYPE_TIME:
+		return {broker::time_point(v->AsTime())};
+	case TYPE_INTERVAL:
+		return {broker::time_duration(v->AsInterval())};
+	case TYPE_ENUM:
+		{
+		auto enum_type = v->Type()->AsEnumType();
+		auto enum_name = enum_type->Lookup(v->AsEnum());
+		return {broker::enum_value(enum_name ? enum_name : "<unknown enum>")};
+		}
+	case TYPE_STRING:
+		{
+		auto s = v->AsString();
+		return {string(reinterpret_cast<const char*>(s->Bytes()), s->Len())};
+		}
+	case TYPE_FILE:
+		return {string(v->AsFile()->Name())};
+	case TYPE_FUNC:
+		return {string(v->AsFunc()->Name())};
+	case TYPE_TABLE:
+		{
+		auto is_set = v->Type()->IsSet();
+		auto table = v->AsTable();
+		auto table_val = v->AsTableVal();
+		broker::data rval;
+
+		if ( is_set )
+			rval = broker::set();
+		else
+			rval = broker::table();
+
+		struct iter_guard {
+			iter_guard(HashKey* arg_k, ListVal* arg_lv)
+			    : k(arg_k), lv(arg_lv)
+				{}
+
+			~iter_guard()
+				{
+				delete k;
+				Unref(lv);
+				}
+
+			HashKey* k;
+			ListVal* lv;
+		};
+
+		HashKey* k;
+		TableEntryVal* entry;
+		auto c = table->InitForIteration();
+
+		while ( (entry = table->NextEntry(k, c)) )
+			{
+			auto vl = table_val->RecoverIndex(k);
+			iter_guard ig(k, vl);
+
+			broker::vector composite_key;
+			composite_key.reserve(vl->Length());
+
+			for ( auto k = 0; k < vl->Length(); ++k )
+				{
+				auto key_part = val_to_data((*vl->Vals())[k]);
+
+				if ( ! key_part )
+					return {};
+
+				composite_key.emplace_back(move(*key_part));
+				}
+
+			broker::data key;
+
+			if ( composite_key.size() == 1 )
+				key = move(composite_key[0]);
+			else
+				key = move(composite_key);
+
+			if ( is_set )
+				broker::get<broker::set>(rval)->emplace(move(key));
+			else
+				{
+				auto val = val_to_data(entry->Value());
+
+				if ( ! val )
+					return {};
+
+				broker::get<broker::table>(rval)->emplace(move(key),
+				                                          move(*val));
+				}
+			}
+
+		return {rval};
+		}
+	case TYPE_VECTOR:
+		{
+		auto vec = v->AsVectorVal();
+		broker::vector rval;
+		rval.reserve(vec->Size());
+
+		for ( auto i = 0u; i < vec->Size(); ++i )
+			{
+			auto item_val = vec->Lookup(i);
+
+			if ( ! item_val )
+				continue;
+
+			auto item = val_to_data(item_val);
+
+			if ( ! item )
+				return {};
+
+			rval.emplace_back(move(*item));
+			}
+
+		return {rval};
+		}
+	case TYPE_RECORD:
+		{
+		auto rec = v->AsRecordVal();
+		broker::record rval;
+		size_t num_fields = v->Type()->AsRecordType()->NumFields();
+		rval.fields.reserve(num_fields);
+
+		for ( auto i = 0u; i < num_fields; ++i )
+			{
+			auto item_val = rec->LookupWithDefault(i);
+
+			if ( ! item_val )
+				{
+				rval.fields.emplace_back(broker::record::field{});
+				continue;
+				}
+
+			auto item = val_to_data(item_val);
+			Unref(item_val);
+
+			if ( ! item )
+				return {};
+
+			rval.fields.emplace_back(broker::record::field{move(*item)});
+			}
+
+		return {rval};
+		}
+	default:
+		reporter->Error("unsupported BrokerComm::Data type: %s",
+		                type_name(v->Type()->Tag()));
+		break;
+	}
+
+	return {};
+	}
+
+RecordVal* bro_broker::make_data_val(Val* v)
+	{
+	auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
+	auto data = val_to_data(v);
+
+	if ( data )
+		rval->Assign(0, new DataVal(move(*data)));
+
+	return rval;
+	}
+
+RecordVal* bro_broker::make_data_val(broker::data d)
+	{
+	auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
+	rval->Assign(0, new DataVal(move(d)));
+	return rval;
+	}
+
+struct data_type_getter {
+	using result_type = EnumVal*;
+
+	result_type operator()(bool a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::BOOL,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(uint64_t a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::COUNT,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(int64_t a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::INT,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(double a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::DOUBLE,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const std::string& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::STRING,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::address& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::ADDR,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::subnet& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::SUBNET,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::port& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::PORT,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::time_point& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::TIME,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::time_duration& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::INTERVAL,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::enum_value& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::ENUM,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::set& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::SET,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::table& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::TABLE,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::vector& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::VECTOR,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+
+	result_type operator()(const broker::record& a)
+		{
+		return new EnumVal(BifEnum::BrokerComm::RECORD,
+		                   BifType::Enum::BrokerComm::DataType);
+		}
+};
+
+EnumVal* bro_broker::get_data_type(RecordVal* v, Frame* frame)
+	{
+	return broker::visit(data_type_getter{}, opaque_field_to_data(v, frame));
+	}
+
+broker::data& bro_broker::opaque_field_to_data(RecordVal* v, Frame* f)
+	{
+	Val* d = v->Lookup(0);
+
+	if ( ! d )
+		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		                       "BrokerComm::Data's opaque field is not set");
+
+	return static_cast<DataVal*>(d)->data;
+	}
+
+IMPLEMENT_SERIAL(bro_broker::DataVal, SER_COMM_DATA_VAL);
+
+bool bro_broker::DataVal::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_COMM_DATA_VAL, OpaqueVal);
+
+	std::string serial;
+	caf::binary_serializer bs(std::back_inserter(serial));
+	bs << data;
+
+	if ( ! SERIALIZE_STR(serial.data(), serial.size()) )
+		return false;
+
+	return true;
+	}
+
+bool bro_broker::DataVal::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(OpaqueVal);
+
+	const char* serial;
+	int len;
+
+	if ( ! UNSERIALIZE_STR(&serial, &len) )
+		return false;
+
+	caf::binary_deserializer bd(serial, len);
+	caf::uniform_typeid<broker::data>()->deserialize(&data, &bd);
+	delete [] serial;
+	return true;
+	}

src/broker/Data.h

@@ -0,0 +1,256 @@
+#ifndef BRO_COMM_DATA_H
+#define BRO_COMM_DATA_H
+
+#include <broker/data.hh>
+#include "Val.h"
+#include "Reporter.h"
+#include "Frame.h"
+#include "Expr.h"
+
+namespace bro_broker {
+
+extern OpaqueType* opaque_of_data_type;
+extern OpaqueType* opaque_of_set_iterator;
+extern OpaqueType* opaque_of_table_iterator;
+extern OpaqueType* opaque_of_vector_iterator;
+extern OpaqueType* opaque_of_record_iterator;
+
+/**
+ * Convert a broker port protocol to a bro port protocol.
+ */
+TransportProto to_bro_port_proto(broker::port::protocol tp);
+
+/**
+ * Create a BrokerComm::Data value from a Bro value.
+ * @param v the Bro value to convert to a Broker data value.
+ * @return a BrokerComm::Data value, where the optional field is set if the conversion
+ * was possible, else it is unset.
+ */
+RecordVal* make_data_val(Val* v);
+
+/**
+ * Create a BrokerComm::Data value from a Broker data value.
+ * @param d the Broker value to wrap in an opaque type.
+ * @return a BrokerComm::Data value that wraps the Broker value.
+ */
+RecordVal* make_data_val(broker::data d);
+
+/**
+ * Get the type of Broker data that BrokerComm::Data wraps.
+ * @param v a BrokerComm::Data value.
+ * @param frame used to get location info upon error.
+ * @return a BrokerComm::DataType value.
+ */
+EnumVal* get_data_type(RecordVal* v, Frame* frame);
+
+/**
+ * Convert a Bro value to a Broker data value.
+ * @param v a Bro value.
+ * @return a Broker data value if the Bro value could be converted to one.
+ */
+broker::util::optional<broker::data> val_to_data(Val* v);
+
+/**
+ * Convert a Broker data value to a Bro value.
+ * @param d a Broker data value.
+ * @param type the expected type of the value to return.
+ * @param require_log_attr if true, skip over record fields that don't have the
+ * &log attribute.
+ * @return a pointer to a new Bro value or a nullptr if the conversion was not
+ * possible.
+ */
+Val* data_to_val(broker::data d, BroType* type, bool require_log_attr = false);
+
+/**
+ * A Bro value which wraps a Broker data value.
+ */
+class DataVal : public OpaqueVal {
+public:
+
+	DataVal(broker::data arg_data)
+		: OpaqueVal(bro_broker::opaque_of_data_type), data(std::move(arg_data))
+		{}
+
+	void ValDescribe(ODesc* d) const override
+		{
+		d->Add("broker::data{");
+		d->Add(broker::to_string(data));
+		d->Add("}");
+		}
+
+	DECLARE_SERIAL(DataVal);
+
+	broker::data data;
+
+protected:
+
+	DataVal()
+		{}
+};
+
+/**
+ * Visitor for retrieving type names a Broker data value.
+ */
+struct type_name_getter {
+	using result_type = const char*;
+
+	result_type operator()(bool a)
+		{ return "bool"; }
+
+	result_type operator()(uint64_t a)
+		{ return "uint64_t"; }
+
+	result_type operator()(int64_t a)
+		{ return "int64_t"; }
+
+	result_type operator()(double a)
+		{ return "double"; }
+
+	result_type operator()(const std::string& a)
+		{ return "string"; }
+
+	result_type operator()(const broker::address& a)
+		{ return "address"; }
+
+	result_type operator()(const broker::subnet& a)
+		{ return "subnet"; }
+
+	result_type operator()(const broker::port& a)
+		{ return "port"; }
+
+	result_type operator()(const broker::time_point& a)
+		{ return "time"; }
+
+	result_type operator()(const broker::time_duration& a)
+		{ return "interval"; }
+
+	result_type operator()(const broker::enum_value& a)
+		{ return "enum"; }
+
+	result_type operator()(const broker::set& a)
+		{ return "set"; }
+
+	result_type operator()(const broker::table& a)
+		{ return "table"; }
+
+	result_type operator()(const broker::vector& a)
+		{ return "vector"; }
+
+	result_type operator()(const broker::record& a)
+		{ return "record"; }
+};
+
+/**
+ * Retrieve Broker data value associated with a BrokerComm::Data Bro value.
+ * @param v a BrokerComm::Data value.
+ * @param f used to get location information on error.
+ * @return a reference to the wrapped Broker data value.  A runtime interpreter
+ * exception is thrown if the the optional opaque value of \a v is not set.
+ */
+broker::data& opaque_field_to_data(RecordVal* v, Frame* f);
+
+/**
+ * Retrieve variant data from a Broker data value.
+ * @tparam T a type that the variant may contain.
+ * @param d a Broker data value to get variant data out of.
+ * @param tag a Bro tag which corresponds to T (just used for error reporting).
+ * @param f used to get location information on error.
+ * @return a refrence to the requested type in the variant Broker data.
+ * A runtime interpret exception is thrown if trying to access a type which
+ * is not currently stored in the Broker data.
+ */
+template <typename T>
+T& require_data_type(broker::data& d, TypeTag tag, Frame* f)
+	{
+	auto ptr = broker::get<T>(d);
+
+	if ( ! ptr )
+		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		                       "data is of type '%s' not of type '%s'",
+		                       broker::visit(type_name_getter{}, d),
+		                       type_name(tag));
+
+	return *ptr;
+	}
+
+/**
+ * @see require_data_type() and opaque_field_to_data().
+ */
+template <typename T>
+inline T& require_data_type(RecordVal* v, TypeTag tag, Frame* f)
+	{
+	return require_data_type<T>(opaque_field_to_data(v, f), tag, f);
+	}
+
+/**
+ * Convert a BrokerComm::Data Bro value to a Bro value of a given type.
+ * @tparam a type that a Broker data variant may contain.
+ * @param v a BrokerComm::Data value.
+ * @param tag a Bro type to convert to.
+ * @param f used to get location information on error.
+ * A runtime interpret exception is thrown if trying to access a type which
+ * is not currently stored in the Broker data.
+ */
+template <typename T>
+inline Val* refine(RecordVal* v, TypeTag tag, Frame* f)
+	{
+	return new Val(require_data_type<T>(v, tag, f), tag);
+	}
+
+// Copying data in to iterator vals is not the fastest approach, but safer...
+
+class SetIterator : public OpaqueVal {
+public:
+
+	SetIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(bro_broker::opaque_of_set_iterator),
+	      dat(require_data_type<broker::set>(v, TYPE_TABLE, f)),
+	      it(dat.begin())
+		{}
+
+	broker::set dat;
+	broker::set::iterator it;
+};
+
+class TableIterator : public OpaqueVal {
+public:
+
+	TableIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(bro_broker::opaque_of_table_iterator),
+	      dat(require_data_type<broker::table>(v, TYPE_TABLE, f)),
+	      it(dat.begin())
+		{}
+
+	broker::table dat;
+	broker::table::iterator it;
+};
+
+class VectorIterator : public OpaqueVal {
+public:
+
+	VectorIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(bro_broker::opaque_of_vector_iterator),
+	      dat(require_data_type<broker::vector>(v, TYPE_VECTOR, f)),
+	      it(dat.begin())
+		{}
+
+	broker::vector dat;
+	broker::vector::iterator it;
+};
+
+class RecordIterator : public OpaqueVal {
+public:
+
+	RecordIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(bro_broker::opaque_of_record_iterator),
+	      dat(require_data_type<broker::record>(v, TYPE_VECTOR, f)),
+	      it(dat.fields.begin())
+		{}
+
+	broker::record dat;
+	decltype(broker::record::fields)::iterator it;
+};
+
+} // namespace bro_broker
+
+#endif // BRO_COMM_DATA_H

src/broker/Manager.h

@@ -0,0 +1,366 @@
+#ifndef BRO_COMM_MANAGER_H
+#define BRO_COMM_MANAGER_H
+
+#include <broker/endpoint.hh>
+#include <broker/message_queue.hh>
+#include <memory>
+#include <string>
+#include <map>
+#include <unordered_set>
+#include "broker/Store.h"
+#include "Reporter.h"
+#include "iosource/IOSource.h"
+#include "Val.h"
+
+namespace bro_broker {
+
+/**
+ * Communication statistics.  Some are tracked in relation to last
+ * sample (bro_broker::Manager::ConsumeStatistics()).
+ */
+struct Stats {
+	// Number of outgoing peer connections (at time of sample).
+	size_t outgoing_peer_count = 0;
+	// Number of data stores (at time of sample).
+	size_t data_store_count = 0;
+	// Number of pending data store queries (at time of sample).
+	size_t pending_query_count = 0;
+	// Number of data store responses received (since last sample).
+	size_t response_count = 0;
+	// Number of outgoing connection updates received (since last sample).
+	size_t outgoing_conn_status_count = 0;
+	// Number of incoming connection updates received (since last sample).
+	size_t incoming_conn_status_count = 0;
+	// Number of broker report messages (e.g. debug, warning, errors) received
+	// (since last sample).
+	size_t report_count = 0;
+	// Number of print messages received per topic-prefix (since last sample).
+	std::map<std::string, size_t> print_count;
+	// Number of event messages received per topic-prefix (since last sample).
+	std::map<std::string, size_t> event_count;
+	// Number of log messages received per topic-prefix (since last sample).
+	std::map<std::string, size_t> log_count;
+};
+
+/**
+ * Manages various forms of communication between peer Bro processes
+ * or other external applications via use of the Broker messaging library.
+ */
+class Manager : public iosource::IOSource {
+friend class StoreHandleVal;
+public:
+
+	/**
+	 * Destructor.  Any still-pending data store queries are aborted.
+	 */
+	~Manager();
+
+	/**
+	 * Enable use of communication.
+	 * @param flags used to tune the local Broker endpoint's behavior.
+	 * See the BrokerComm::EndpointFlags record type.
+	 * @return true if communication is successfully initialized.
+	 */
+	bool Enable(Val* flags);
+
+	/**
+	 * Changes endpoint flags originally supplied to bro_broker::Manager::Enable().
+	 * @param flags the new behavior flags to use.
+	 * @return true if flags were changed.
+	 */
+	bool SetEndpointFlags(Val* flags);
+
+	/**
+	 * @return true if bro_broker::Manager::Enable() has previously been called and
+	 * it succeeded.
+	 */
+	bool Enabled()
+		{ return endpoint != nullptr; }
+
+	/**
+	 * Listen for remote connections.
+	 * @param port the TCP port to listen on.
+	 * @param addr an address string on which to accept connections, e.g.
+	 * "127.0.0.1".  A nullptr refers to @p INADDR_ANY.
+	 * @param reuse_addr equivalent to behavior of SO_REUSEADDR.
+	 * @return true if the local endpoint is now listening for connections.
+	 */
+	bool Listen(uint16_t port, const char* addr = nullptr,
+	            bool reuse_addr = true);
+
+	/**
+	 * Initiate a remote connection.
+	 * @param addr an address to connect to, e.g. "localhost" or "127.0.0.1".
+	 * @param port the TCP port on which the remote side is listening.
+	 * @param retry_interval an interval at which to retry establishing the
+	 * connection with the remote peer.
+	 * @return true if it's possible to try connecting with the peer and
+	 * it's a new peer.  The actual connection may not be established until a
+	 * later point in time.
+	 */
+	bool Connect(std::string addr, uint16_t port,
+	             std::chrono::duration<double> retry_interval);
+
+	/**
+	 * Remove a remote connection.
+	 * @param addr the address used in bro_broker::Manager::Connect().
+	 * @param port the port used in bro_broker::Manager::Connect().
+	 * @return true if the arguments match a previously successful call to
+	 * bro_broker::Manager::Connect().
+	 */
+	bool Disconnect(const std::string& addr, uint16_t port);
+
+	/**
+	 * Print a simple message to any interested peers.
+	 * @param topic a topic string associated with the print message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param msg the string to send to peers.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the BrokerComm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Print(std::string topic, std::string msg, Val* flags);
+
+	/**
+	 * Send an event to any interested peers.
+	 * @param topic a topic string associated with the print message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param msg the event to send to peers, which is the name of the event
+	 * as a string followed by all of its arguments.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the BrokerComm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Event(std::string topic, broker::message msg, int flags);
+
+	/**
+	 * Send an event to any interested peers.
+	 * @param topic a topic string associated with the print message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param args the event and its arguments to send to peers.  See the
+	 * BrokerComm::EventArgs record type.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the BrokerComm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Event(std::string topic, RecordVal* args, Val* flags);
+
+	/**
+	 * Send a log entry to any interested peers.  The topic name used is
+	 * implicitly "bro/log/<stream-name>".
+	 * @param stream_id the stream to which the log entry belongs.
+	 * @param columns the data which comprises the log entry.
+	 * @param info the record type corresponding to the log's columns.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the BrokerComm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Log(EnumVal* stream_id, RecordVal* columns, RecordType* info,
+	         int flags);
+
+	/**
+	 * Automatically send an event to any interested peers whenever it is
+	 * locally dispatched (e.g. using "event my_event(...);" in a script).
+	 * @param topic a topic string associated with the event message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param event a Bro event value.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the BrokerComm::SendFlags record type.
+	 * @return true if automatic event sending is now enabled.
+	 */
+	bool AutoEvent(std::string topic, Val* event, Val* flags);
+
+	/**
+	 * Stop automatically sending an event to peers upon local dispatch.
+	 * @param topic a topic originally given to bro_broker::Manager::AutoEvent().
+	 * @param event an event originally given to bro_broker::Manager::AutoEvent().
+	 * @return true if automatic events will no occur for the topic/event pair.
+	 */
+	bool AutoEventStop(const std::string& topic, Val* event);
+
+	/**
+	 * Create an EventArgs record value from an event and its arguments.
+	 * @param args the event and its arguments.  The event is always the first
+	 * elements in the list.
+	 * @return an EventArgs record value.  If an invalid event or arguments
+	 * were supplied the optional "name" field will not be set.
+	 */
+	RecordVal* MakeEventArgs(val_list* args);
+
+	/**
+	 * Register interest in peer print messages that use a certain topic prefix.
+	 * @param topic_prefix a prefix to match against remote message topics.
+	 * e.g. an empty prefix will match everything and "a" will match "alice"
+	 * and "amy" but not "bob".
+	 * @return true if it's a new print subscriptions and it is now registered.
+	 */
+	bool SubscribeToPrints(std::string topic_prefix);
+
+	/**
+	 * Unregister interest in peer print messages.
+	 * @param topic_prefix a prefix previously supplied to a successful call
+	 * to bro_broker::Manager::SubscribeToPrints().
+	 * @return true if interest in topic prefix is no longer advertised.
+	 */
+	bool UnsubscribeToPrints(const std::string& topic_prefix);
+
+	/**
+	 * Register interest in peer event messages that use a certain topic prefix.
+	 * @param topic_prefix a prefix to match against remote message topics.
+	 * e.g. an empty prefix will match everything and "a" will match "alice"
+	 * and "amy" but not "bob".
+	 * @return true if it's a new event subscription and it is now registered.
+	 */
+	bool SubscribeToEvents(std::string topic_prefix);
+
+	/**
+	 * Unregister interest in peer event messages.
+	 * @param topic_prefix a prefix previously supplied to a successful call
+	 * to bro_broker::Manager::SubscribeToEvents().
+	 * @return true if interest in topic prefix is no longer advertised.
+	 */
+	bool UnsubscribeToEvents(const std::string& topic_prefix);
+
+	/**
+	 * Register interest in peer log messages that use a certain topic prefix.
+	 * @param topic_prefix a prefix to match against remote message topics.
+	 * e.g. an empty prefix will match everything and "a" will match "alice"
+	 * and "amy" but not "bob".
+	 * @return true if it's a new log subscription and it is now registered.
+	 */
+	bool SubscribeToLogs(std::string topic_prefix);
+
+	/**
+	 * Unregister interest in peer log messages.
+	 * @param topic_prefix a prefix previously supplied to a successful call
+	 * to bro_broker::Manager::SubscribeToLogs().
+	 * @return true if interest in topic prefix is no longer advertised.
+	 */
+	bool UnsubscribeToLogs(const std::string& topic_prefix);
+
+	/**
+	 * Allow sending messages to peers if associated with the given topic.
+	 * This has no effect if auto publication behavior is enabled via the flags
+	 * supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
+	 * @param t a topic to allow messages to be published under.
+	 * @return true if successful.
+	 */
+	bool PublishTopic(broker::topic t);
+
+	/**
+	 * Disallow sending messages to peers if associated with the given topic.
+	 * This has no effect if auto publication behavior is enabled via the flags
+	 * supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
+	 * @param t a topic to disallow messages to be published under.
+	 * @return true if successful.
+	 */
+	bool UnpublishTopic(broker::topic t);
+
+	/**
+	 * Allow advertising interest in the given topic to peers.
+	 * This has no effect if auto advertise behavior is enabled via the flags
+	 * supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
+	 * @param t a topic to allow advertising interest/subscription to peers.
+	 * @return true if successful.
+	 */
+	bool AdvertiseTopic(broker::topic t);
+
+	/**
+	 * Disallow advertising interest in the given topic to peers.
+	 * This has no effect if auto advertise behavior is enabled via the flags
+	 * supplied to bro_broker::Manager::Enable() or bro_broker::Manager::SetEndpointFlags().
+	 * @param t a topic to disallow advertising interest/subscription to peers.
+	 * @return true if successful.
+	 */
+	bool UnadvertiseTopic(broker::topic t);
+
+	/**
+	 * Register the availability of a data store.
+	 * @param handle the data store.
+	 * @return true if the store was valid and not already away of it.
+	 */
+	bool AddStore(StoreHandleVal* handle);
+
+	/**
+	 * Lookup a data store by it's identifier name and type.
+	 * @param id the store's name.
+	 * @param type the type of data store.
+	 * @return a pointer to the store handle if it exists else nullptr.
+	 */
+	StoreHandleVal* LookupStore(const broker::store::identifier& id, StoreType type);
+
+	/**
+	 * Close and unregister a data store.  Any existing references to the
+	 * store handle will not be able to be used for any data store operations.
+	 * @param id the stores' name.
+	 * @param type the type of the data store.
+	 * @return true if such a store existed and is now closed.
+	 */
+	bool CloseStore(const broker::store::identifier& id, StoreType type);
+
+	/**
+	 * Register a data store query callback.
+	 * @param cb the callback info to use when the query completes or times out.
+	 * @return true if now tracking a data store query.
+	 */
+	bool TrackStoreQuery(StoreQueryCallback* cb);
+
+	/**
+	 * @return communication statistics.
+	 */
+	Stats ConsumeStatistics();
+
+	/**
+	 * Convert BrokerComm::SendFlags to int flags for use with broker::send().
+	 */
+	static int send_flags_to_int(Val* flags);
+
+private:
+
+	// IOSource interface overrides:
+	void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	            iosource::FD_Set* except) override;
+
+	double NextTimestamp(double* local_network_time) override;
+
+	void Process() override;
+
+	const char* Tag() override
+		{ return "BrokerComm::Manager"; }
+
+	broker::endpoint& Endpoint()
+		{ return *endpoint; }
+
+	struct QueueWithStats {
+		broker::message_queue q;
+		size_t received = 0;
+	};
+
+	std::unique_ptr<broker::endpoint> endpoint;
+	std::map<std::pair<std::string, uint16_t>, broker::peering> peers;
+	std::map<std::string, QueueWithStats> print_subscriptions;
+	std::map<std::string, QueueWithStats> event_subscriptions;
+	std::map<std::string, QueueWithStats> log_subscriptions;
+
+	std::map<std::pair<broker::store::identifier, StoreType>,
+	         StoreHandleVal*> data_stores;
+	std::unordered_set<StoreQueryCallback*> pending_queries;
+
+	Stats statistics;
+
+	static VectorType* vector_of_data_type;
+	static EnumType* log_id_type;
+	static int send_flags_self_idx;
+	static int send_flags_peers_idx;
+	static int send_flags_unsolicited_idx;
+};
+
+} // namespace bro_broker
+
+extern bro_broker::Manager* broker_mgr;
+
+#endif // BRO_COMM_MANAGER_H

src/broker/Store.cc

@@ -0,0 +1,204 @@
+#include "Store.h"
+#include "broker/Manager.h"
+
+#include <broker/store/master.hh>
+#include <broker/store/clone.hh>
+#include <broker/store/sqlite_backend.hh>
+
+#ifdef HAVE_ROCKSDB
+#include <broker/store/rocksdb_backend.hh>
+#include <rocksdb/db.h>
+#endif
+
+OpaqueType* bro_broker::opaque_of_store_handle;
+
+bro_broker::StoreHandleVal::StoreHandleVal(broker::store::identifier id,
+                                     bro_broker::StoreType arg_type,
+                                     broker::util::optional<BifEnum::BrokerStore::BackendType> arg_back,
+                                     RecordVal* backend_options, std::chrono::duration<double> resync)
+	: OpaqueVal(opaque_of_store_handle),
+	  store(), store_type(arg_type), backend_type(arg_back)
+	{
+	using BifEnum::BrokerStore::BackendType;
+	std::unique_ptr<broker::store::backend> backend;
+
+	if ( backend_type )
+		switch ( *backend_type ) {
+		case BackendType::MEMORY:
+			backend.reset(new broker::store::memory_backend);
+			break;
+		case BackendType::SQLITE:
+			{
+			auto sqlite = new broker::store::sqlite_backend;
+			std::string path = backend_options->Lookup(0)->AsRecordVal()
+			                   ->Lookup(0)->AsStringVal()->CheckString();
+
+			if ( sqlite->open(path) )
+				backend.reset(sqlite);
+			else
+				{
+				reporter->Error("failed to open sqlite backend at path %s: %s",
+				                path.data(), sqlite->last_error().data());
+				delete sqlite;
+				}
+			}
+			break;
+		case BackendType::ROCKSDB:
+			{
+#ifdef HAVE_ROCKSDB
+			std::string path = backend_options->Lookup(1)->AsRecordVal()
+			                   ->Lookup(0)->AsStringVal()->CheckString();
+			rocksdb::Options rock_op;
+			rock_op.create_if_missing = true;
+
+			auto rocksdb = new broker::store::rocksdb_backend;
+
+			if ( rocksdb->open(path, options).ok() )
+				backend.reset(rocksdb);
+			else
+				{
+				reporter->Error("failed to open rocksdb backend at path %s: %s",
+				                path.data(), rocksdb->last_error().data());
+				delete rocksdb;
+				}
+#else
+			reporter->Error("rocksdb backend support is not enabled");
+#endif
+			}
+			break;
+		default:
+			reporter->FatalError("unknown data store backend: %d",
+			                     static_cast<int>(*backend_type));
+			}
+
+	switch ( store_type ) {
+	case StoreType::FRONTEND:
+		store = new broker::store::frontend(broker_mgr->Endpoint(), move(id));
+		break;
+	case StoreType::MASTER:
+		store = new broker::store::master(broker_mgr->Endpoint(), move(id),
+		                                  move(backend));
+		break;
+	case StoreType::CLONE:
+		store = new broker::store::clone(broker_mgr->Endpoint(), move(id), resync,
+		                                 move(backend));
+		break;
+	default:
+		reporter->FatalError("unknown data store type: %d",
+		                     static_cast<int>(store_type));
+		}
+	}
+
+void bro_broker::StoreHandleVal::ValDescribe(ODesc* d) const
+	{
+	using BifEnum::BrokerStore::BackendType;
+	d->Add("broker::store::");
+
+	switch ( store_type ) {
+	case StoreType::FRONTEND:
+		d->Add("frontend");
+		break;
+	case StoreType::MASTER:
+		d->Add("master");
+		break;
+	case StoreType::CLONE:
+		d->Add("clone");
+		break;
+	default:
+		d->Add("unknown");
+		}
+
+	d->Add("{");
+	d->Add(store->id());
+
+	if ( backend_type )
+		{
+		d->Add(", ");
+
+		switch ( *backend_type ) {
+		case BackendType::MEMORY:
+			d->Add("memory");
+			break;
+		case BackendType::SQLITE:
+			d->Add("sqlite");
+			break;
+		case BackendType::ROCKSDB:
+			d->Add("rocksdb");
+			break;
+		default:
+			d->Add("unknown");
+			}
+		}
+
+	d->Add("}");
+	}
+
+IMPLEMENT_SERIAL(bro_broker::StoreHandleVal, SER_COMM_STORE_HANDLE_VAL);
+
+bool bro_broker::StoreHandleVal::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_COMM_STORE_HANDLE_VAL, OpaqueVal);
+
+	bool have_store = store != nullptr;
+
+	if ( ! SERIALIZE(have_store) )
+		return false;
+
+	if ( ! have_store )
+		return true;
+
+	if ( ! SERIALIZE(static_cast<int>(store_type)) )
+		return false;
+
+	if ( ! SERIALIZE_STR(store->id().data(), store->id().size()) )
+		return false;
+
+	return true;
+	}
+
+bool bro_broker::StoreHandleVal::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(OpaqueVal);
+
+	bool have_store;
+
+	if ( ! UNSERIALIZE(&have_store) )
+		return false;
+
+	if ( ! have_store )
+		{
+		store = nullptr;
+		return true;
+		}
+
+	int type;
+
+	if ( ! UNSERIALIZE(&type) )
+		return false;
+
+	const char* id_str;
+	int len;
+
+	if ( ! UNSERIALIZE_STR(&id_str, &len) )
+		return false;
+
+	broker::store::identifier id(id_str, len);
+	delete [] id_str;
+
+	auto handle = broker_mgr->LookupStore(id, static_cast<bro_broker::StoreType>(type));
+
+	if ( ! handle )
+		{
+		// Passing serialized version of store handles to other Bro processes
+		// doesn't make sense, only allow local clones of the handle val.
+		reporter->Error("failed to look up unserialized store handle %s, %d",
+		                id.data(), type);
+		store = nullptr;
+		return false;
+		}
+
+	store = handle->store;
+	store_type = handle->store_type;
+	backend_type = handle->backend_type;
+	return true;
+	}

src/broker/Store.h

@@ -0,0 +1,153 @@
+#ifndef BRO_COMM_STORE_H
+#define BRO_COMM_STORE_H
+
+#include "broker/store.bif.h"
+#include "broker/data.bif.h"
+#include "Reporter.h"
+#include "Type.h"
+#include "Val.h"
+#include "Trigger.h"
+
+#include <broker/store/frontend.hh>
+
+namespace bro_broker {
+
+extern OpaqueType* opaque_of_store_handle;
+
+/**
+ * Enumerates the possible types of data stores.
+ */
+enum StoreType {
+	// Just a view in to a remote store, contains no data itself.
+	FRONTEND,
+	MASTER,
+	CLONE,
+};
+
+/**
+ * Create a BrokerStore::QueryStatus value.
+ * @param success whether the query status should be set to success or failure.
+ * @return a BrokerStore::QueryStatus value.
+ */
+inline EnumVal* query_status(bool success)
+	{
+	static EnumType* store_query_status = nullptr;
+	static int success_val;
+	static int failure_val;
+
+	if ( ! store_query_status )
+		{
+		store_query_status = internal_type("BrokerStore::QueryStatus")->AsEnumType();
+		success_val = store_query_status->Lookup("BrokerStore", "SUCCESS");
+		failure_val = store_query_status->Lookup("BrokerStore", "FAILURE");
+		}
+
+	return new EnumVal(success ? success_val : failure_val, store_query_status);
+	}
+
+/**
+ * @return a BrokerStore::QueryResult value that has a BrokerStore::QueryStatus indicating
+ * a failure.
+ */
+inline RecordVal* query_result()
+	{
+	auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult);
+	rval->Assign(0, query_status(false));
+	rval->Assign(1, new RecordVal(BifType::Record::BrokerComm::Data));
+	return rval;
+	}
+
+/**
+ * @param data the result of the query.
+ * @return a BrokerStore::QueryResult value that has a BrokerStore::QueryStatus indicating
+ * a success.
+ */
+inline RecordVal* query_result(RecordVal* data)
+	{
+	auto rval = new RecordVal(BifType::Record::BrokerStore::QueryResult);
+	rval->Assign(0, query_status(true));
+	rval->Assign(1, data);
+	return rval;
+	}
+
+/**
+ * Used for asynchronous data store queries which use "when" statements.
+ */
+class StoreQueryCallback {
+public:
+
+	StoreQueryCallback(Trigger* arg_trigger, const CallExpr* arg_call,
+					   broker::store::identifier arg_store_id,
+	                   StoreType arg_store_type)
+		: trigger(arg_trigger), call(arg_call), store_id(move(arg_store_id)),
+	      store_type(arg_store_type)
+		{
+		Ref(trigger);
+		}
+
+	~StoreQueryCallback()
+		{
+		Unref(trigger);
+		}
+
+	void Result(RecordVal* result)
+		{
+		trigger->Cache(call, result);
+		trigger->Release();
+		Unref(result);
+		}
+
+	void Abort()
+		{
+		auto result = query_result();
+		trigger->Cache(call, result);
+		trigger->Release();
+		Unref(result);
+		}
+
+	bool Disabled() const
+		{ return trigger->Disabled(); }
+
+	const broker::store::identifier& StoreID() const
+		{ return store_id; }
+
+	StoreType GetStoreType() const
+		{ return store_type; }
+
+private:
+
+	Trigger* trigger;
+	const CallExpr* call;
+	broker::store::identifier store_id;
+	StoreType store_type;
+};
+
+/**
+ * An opaque handle which wraps a Broker data store.
+ */
+class StoreHandleVal : public OpaqueVal {
+public:
+
+	StoreHandleVal(broker::store::identifier id,
+		       bro_broker::StoreType arg_type,
+		       broker::util::optional<BifEnum::BrokerStore::BackendType> arg_back,
+		       RecordVal* backend_options,
+		       std::chrono::duration<double> resync = std::chrono::seconds(1));
+
+	void ValDescribe(ODesc* d) const override;
+
+	DECLARE_SERIAL(StoreHandleVal);
+
+	broker::store::frontend* store;
+	bro_broker::StoreType store_type;
+	broker::util::optional<BifEnum::BrokerStore::BackendType> backend_type;
+
+protected:
+
+	StoreHandleVal()
+		{}
+};
+
+} // namespace bro_broker
+
+#endif // BRO_COMM_STORE_H

src/broker/comm.bif

@@ -0,0 +1,199 @@
+
+##! General functions regarding Bro's broker communication mechanisms.
+
+%%{
+#include "broker/Manager.h"
+%%}
+
+module BrokerComm;
+
+type BrokerComm::EndpointFlags: record;
+
+## Enable use of communication.
+##
+## flags: used to tune the local Broker endpoint behavior.
+##
+## Returns: true if communication is successfully initialized.
+function BrokerComm::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool
+	%{
+	return new Val(broker_mgr->Enable(flags), TYPE_BOOL);
+	%}
+
+## Changes endpoint flags originally supplied to :bro:see:`BrokerComm::enable`.
+##
+## flags: the new endpoint behavior flags to use.
+##
+## Returns: true of flags were changed.
+function BrokerComm::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool
+	%{
+	return new Val(broker_mgr->SetEndpointFlags(flags), TYPE_BOOL);
+	%}
+
+## Allow sending messages to peers if associated with the given topic.
+## This has no effect if auto publication behavior is enabled via the flags
+## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
+##
+## topic: a topic to allow messages to be published under.
+##
+## Returns: true if successful.
+function BrokerComm::publish_topic%(topic: string%): bool
+	%{
+	return new Val(broker_mgr->PublishTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Disallow sending messages to peers if associated with the given topic.
+## This has no effect if auto publication behavior is enabled via the flags
+## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
+##
+## topic: a topic to disallow messages to be published under.
+##
+## Returns: true if successful.
+function BrokerComm::unpublish_topic%(topic: string%): bool
+	%{
+	return new Val(broker_mgr->UnpublishTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Allow advertising interest in the given topic to peers.
+## This has no effect if auto advertise behavior is enabled via the flags
+## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
+##
+## topic: a topic to allow advertising interest/subscription to peers.
+##
+## Returns: true if successful.
+function BrokerComm::advertise_topic%(topic: string%): bool
+	%{
+	return new Val(broker_mgr->AdvertiseTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Disallow advertising interest in the given topic to peers.
+## This has no effect if auto advertise behavior is enabled via the flags
+## supplied to :bro:see:`BrokerComm::enable` or :bro:see:`BrokerComm::set_endpoint_flags`.
+##
+## topic: a topic to disallow advertising interest/subscription to peers.
+##
+## Returns: true if successful.
+function BrokerComm::unadvertise_topic%(topic: string%): bool
+	%{
+	return new Val(broker_mgr->UnadvertiseTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Generated when a connection has been established due to a previous call
+## to :bro:see:`BrokerComm::connect`.
+##
+## peer_address: the address used to connect to the peer.
+##
+## peer_port: the port used to connect to the peer.
+##
+## peer_name: the name by which the peer identified itself.
+event BrokerComm::outgoing_connection_established%(peer_address: string,
+                                             peer_port: port,
+                                             peer_name: string%);
+
+## Generated when a previously established connection becomes broken.
+## Reconnection will automatically be attempted at a frequency given
+## by the original call to :bro:see:`BrokerComm::connect`.
+##
+## peer_address: the address used to connect to the peer.
+##
+## peer_port: the port used to connect to the peer.
+##
+## .. bro:see:: BrokerComm::outgoing_connection_established
+event BrokerComm::outgoing_connection_broken%(peer_address: string,
+                                        peer_port: port%);
+
+## Generated when a connection via :bro:see:`BrokerComm::connect` has failed
+## because the remote side is incompatible.
+##
+## peer_address: the address used to connect to the peer.
+##
+## peer_port: the port used to connect to the peer.
+event BrokerComm::outgoing_connection_incompatible%(peer_address: string,
+                                              peer_port: port%);
+
+## Generated when a peer has established a connection with this process
+## as a result of previously performing a :bro:see:`BrokerComm::listen`.
+##
+## peer_name: the name by which the peer identified itself.
+event BrokerComm::incoming_connection_established%(peer_name: string%);
+
+## Generated when a peer that previously established a connection with this
+## process becomes disconnected.
+##
+## peer_name: the name by which the peer identified itself.
+##
+## .. bro:see:: BrokerComm::incoming_connection_established
+event BrokerComm::incoming_connection_broken%(peer_name: string%);
+
+## Listen for remote connections.
+##
+## p: the TCP port to listen on.
+##
+## a: an address string on which to accept connections, e.g.
+##    "127.0.0.1".  An empty string refers to @p INADDR_ANY.
+##
+## reuse: equivalent to behavior of SO_REUSEADDR.
+##
+## Returns: true if the local endpoint is now listening for connections.
+##
+## .. bro:see:: BrokerComm::incoming_connection_established
+function BrokerComm::listen%(p: port, a: string &default = "",
+					   reuse: bool &default = T%): bool
+	%{
+	if ( ! p->IsTCP() )
+		{
+		reporter->Error("listen port must use tcp");
+		return new Val(false, TYPE_BOOL);
+		}
+
+	auto rval = broker_mgr->Listen(p->Port(), a->Len() ? a->CheckString() : 0,
+								 reuse);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Initiate a remote connection.
+##
+## a: an address to connect to, e.g. "localhost" or "127.0.0.1".
+##
+## p: the TCP port on which the remote side is listening.
+##
+## retry: an interval at which to retry establishing the
+##        connection with the remote peer if it cannot be made initially, or
+##        if it ever becomes disconnected.
+##
+## Returns: true if it's possible to try connecting with the peer and
+##          it's a new peer.  The actual connection may not be established
+##          a later point in time.
+##
+## .. bro:see:: BrokerComm::outgoing_connection_established
+function BrokerComm::connect%(a: string, p: port, retry: interval%): bool
+	%{
+	if ( ! p->IsTCP() )
+		{
+		reporter->Error("remote connection port must use tcp");
+		return new Val(false, TYPE_BOOL);
+		}
+
+	auto rval = broker_mgr->Connect(a->CheckString(), p->Port(),
+	                              std::chrono::duration<double>(retry));
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Remove a remote connection.
+##
+## a: the address used in previous successful call to :bro:see:`BrokerComm::connect`.
+##
+## p: the port used in previous successful call to :bro:see:`BrokerComm::connect`.
+##
+## Returns: true if the arguments match a previously successful call to
+##          :bro:see:`BrokerComm::connect`.
+function BrokerComm::disconnect%(a: string, p: port%): bool
+	%{
+	if ( ! p->IsTCP() )
+		{
+		reporter->Error("remote connection port must use tcp");
+		return new Val(false, TYPE_BOOL);
+		}
+
+	auto rval = broker_mgr->Disconnect(a->CheckString(), p->Port());
+	return new Val(rval, TYPE_BOOL);
+	%}

src/broker/data.bif

@@ -0,0 +1,834 @@
+
+##! Functions for inspecting and manipulating broker data.
+
+%%{
+#include "broker/Data.h"
+%%}
+
+module BrokerComm;
+
+## Enumerates the possible types that :bro:see:`BrokerComm::Data` may be in terms of
+## Bro data types.
+enum DataType %{
+	BOOL,
+	INT,
+	COUNT,
+	DOUBLE,
+	STRING,
+	ADDR,
+	SUBNET,
+	PORT,
+	TIME,
+	INTERVAL,
+	ENUM,
+	SET,
+	TABLE,
+	VECTOR,
+	RECORD,
+%}
+
+type BrokerComm::Data: record;
+
+type BrokerComm::TableItem: record;
+
+## Convert any Bro value in to communication data.
+##
+## d: any Bro value to attempt to convert (not all types are supported).
+##
+## Returns: the converted communication data which may not set its only
+##          opaque field of the the conversion was not possible (the Bro data
+##          type does not support being converted to communicaiton data).
+function BrokerComm::data%(d: any%): BrokerComm::Data
+	%{
+	return bro_broker::make_data_val(d);
+	%}
+
+## Retrieve the type of data associated with communication data.
+##
+## d: the communication data.
+##
+## Returns: the data type associated with the communication data.
+function BrokerComm::data_type%(d: BrokerComm::Data%): BrokerComm::DataType
+	%{
+	return bro_broker::get_data_type(d->AsRecordVal(), frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::BOOL` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_bool%(d: BrokerComm::Data%): bool
+	%{
+	return bro_broker::refine<bool>(d->AsRecordVal(), TYPE_BOOL, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::INT` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_int%(d: BrokerComm::Data%): int
+	%{
+	return bro_broker::refine<int64_t>(d->AsRecordVal(), TYPE_INT, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::COUNT` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_count%(d: BrokerComm::Data%): count
+	%{
+	return bro_broker::refine<uint64_t>(d->AsRecordVal(), TYPE_COUNT, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::DOUBLE` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_double%(d: BrokerComm::Data%): double
+	%{
+	return bro_broker::refine<double>(d->AsRecordVal(), TYPE_DOUBLE, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::STRING` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_string%(d: BrokerComm::Data%): string
+	%{
+	return new StringVal(bro_broker::require_data_type<std::string>(d->AsRecordVal(),
+												              TYPE_STRING,
+															  frame));
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::ADDR` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_addr%(d: BrokerComm::Data%): addr
+	%{
+	auto& a = bro_broker::require_data_type<broker::address>(d->AsRecordVal(),
+													   TYPE_ADDR, frame);
+	auto bits = reinterpret_cast<const in6_addr*>(&a.bytes());
+	return new AddrVal(IPAddr(*bits));
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::SUBNET` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_subnet%(d: BrokerComm::Data%): subnet
+	%{
+	auto& a = bro_broker::require_data_type<broker::subnet>(d->AsRecordVal(),
+													  TYPE_SUBNET, frame);
+	auto bits = reinterpret_cast<const in6_addr*>(&a.network().bytes());
+	return new SubNetVal(IPPrefix(IPAddr(*bits), a.length()));
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::PORT` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_port%(d: BrokerComm::Data%): port
+	%{
+	auto& a = bro_broker::require_data_type<broker::port>(d->AsRecordVal(),
+													TYPE_SUBNET, frame);
+	return new PortVal(a.number(), bro_broker::to_bro_port_proto(a.type()));
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::TIME` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_time%(d: BrokerComm::Data%): time
+	%{
+	auto v = bro_broker::require_data_type<broker::time_point>(d->AsRecordVal(),
+														TYPE_TIME, frame).value;
+	return new Val(v, TYPE_TIME);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::INTERVAL` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function BrokerComm::refine_to_interval%(d: BrokerComm::Data%): interval
+	%{
+	auto v = bro_broker::require_data_type<broker::time_duration>(d->AsRecordVal(),
+														TYPE_TIME, frame).value;
+	return new Val(v, TYPE_INTERVAL);
+	%}
+
+## Convert communication data with a type of :bro:see:`BrokerComm::ENUM` to
+## the name of the enum value.  :bro:see:`lookup_ID` may be used to convert
+## the name to the actual enum value.
+##
+## d: the communication data to convert.
+##
+## Returns: the enum name retrieved from the communication data.
+function BrokerComm::refine_to_enum_name%(d: BrokerComm::Data%): string
+	%{
+	auto& v = bro_broker::require_data_type<broker::enum_value>(d->AsRecordVal(),
+														TYPE_ENUM, frame).name;
+	return new StringVal(v);
+	%}
+
+## Create communication data of type "set".
+function BrokerComm::set_create%(%): BrokerComm::Data
+	%{
+	return bro_broker::make_data_val(broker::set());
+	%}
+
+## Remove all elements within a set.
+##
+## s: the set to clear.
+##
+## Returns: always true.
+function BrokerComm::set_clear%(s: BrokerComm::Data%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	v.clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Get the number of elements within a set.
+##
+## s: the set to query.
+##
+## Returns: the number of elements in the set.
+function BrokerComm::set_size%(s: BrokerComm::Data%): count
+	%{
+	auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	return new Val(static_cast<uint64_t>(v.size()), TYPE_COUNT);
+	%}
+
+## Check if a set contains a particular element.
+##
+## s: the set to query.
+##
+## key: the element to check for existence.
+##
+## Returns: true if the key exists in the set.
+function BrokerComm::set_contains%(s: BrokerComm::Data, key: BrokerComm::Data%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.find(k) != v.end(), TYPE_BOOL);
+	%}
+
+### Insert an element into a set.
+##
+## s: the set to modify.
+##
+## key: the element to insert.
+##
+## Returns: true if the key was inserted, or false if it already existed.
+function BrokerComm::set_insert%(s: BrokerComm::Data, key: BrokerComm::Data%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.insert(k).second, TYPE_BOOL);
+	%}
+
+## Remove an element from a set.
+##
+## s: the set to modify.
+##
+## key: the element to remove.
+##
+## Returns: true if the element existed in the set and is now removed.
+function BrokerComm::set_remove%(s: BrokerComm::Data, key: BrokerComm::Data%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.erase(k) > 0, TYPE_BOOL);
+	%}
+
+## Create an iterator for a set.  Note that this makes a copy of the set
+## internally to ensure the iterator is always valid.
+##
+## s: the set to iterate over.
+##
+## Returns: an iterator.
+function BrokerComm::set_iterator%(s: BrokerComm::Data%): opaque of BrokerComm::SetIterator
+	%{
+	return new bro_broker::SetIterator(s->AsRecordVal(), TYPE_TABLE, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function BrokerComm::set_iterator_last%(it: opaque of BrokerComm::SetIterator%): bool
+	%{
+	auto set_it = static_cast<bro_broker::SetIterator*>(it);
+	return new Val(set_it->it == set_it->dat.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function BrokerComm::set_iterator_next%(it: opaque of BrokerComm::SetIterator%): bool
+	%{
+	auto set_it = static_cast<bro_broker::SetIterator*>(it);
+
+	if ( set_it->it == set_it->dat.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++set_it->it;
+	return new Val(set_it->it != set_it->dat.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function BrokerComm::set_iterator_value%(it: opaque of BrokerComm::SetIterator%): BrokerComm::Data
+	%{
+	auto set_it = static_cast<bro_broker::SetIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
+
+	if ( set_it->it == set_it->dat.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid set iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	rval->Assign(0, new bro_broker::DataVal(*set_it->it));
+	return rval;
+	%}
+
+## Create communication data of type "table".
+function BrokerComm::table_create%(%): BrokerComm::Data
+	%{
+	return bro_broker::make_data_val(broker::table());
+	%}
+
+## Remove all elements within a table.
+##
+## t: the table to clear.
+##
+## Returns: always true.
+function BrokerComm::table_clear%(t: BrokerComm::Data%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
+													 TYPE_TABLE, frame);
+	v.clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Get the number of elements within a table.
+##
+## t: the table to query.
+##
+## Returns: the number of elements in the table.
+function BrokerComm::table_size%(t: BrokerComm::Data%): count
+	%{
+	auto& v = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
+													 TYPE_TABLE, frame);
+	return new Val(static_cast<uint64_t>(v.size()), TYPE_COUNT);
+	%}
+
+## Check if a table contains a particular key.
+##
+## t: the table to query.
+##
+## key: the key to check for existence.
+##
+## Returns: true if the key exists in the set.
+function BrokerComm::table_contains%(t: BrokerComm::Data, key: BrokerComm::Data%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
+													 TYPE_TABLE, frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.find(k) != v.end(), TYPE_BOOL);
+	%}
+
+## Insert a key-value pair into a table.
+##
+## t: the table to modify.
+##
+## key: the key at which to insert the value.
+##
+## val: the value to insert.
+##
+## Returns: true if the key-value pair was inserted, or false if the key
+##          already existed in the table.
+function BrokerComm::table_insert%(t: BrokerComm::Data, key: BrokerComm::Data, val: BrokerComm::Data%): BrokerComm::Data
+	%{
+	auto& table = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
+													     TYPE_TABLE, frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	auto& v = bro_broker::opaque_field_to_data(val->AsRecordVal(), frame);
+
+	try
+		{
+		auto& prev = table.at(k);
+		auto rval = bro_broker::make_data_val(move(prev));
+		prev = v;
+		return rval;
+		}
+	catch (const std::out_of_range&)
+		{
+		table[k] = v;
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+		}
+	%}
+
+## Remove a key-value pair from a table.
+##
+## t: the table to modify.
+##
+## key: the key to remove from the table.
+##
+## Returns: the value associated with the key.  If the key did not exist, then
+##          the optional field of the returned record is not set.
+function BrokerComm::table_remove%(t: BrokerComm::Data, key: BrokerComm::Data%): BrokerComm::Data
+	%{
+	auto& table = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
+													     TYPE_TABLE, frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	auto it = table.find(k);
+
+	if ( it == table.end() )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+	else
+		{
+		auto rval = bro_broker::make_data_val(move(it->second));
+		table.erase(it);
+		return rval;
+		}
+	%}
+
+## Retrieve a value from a table.
+##
+## t: the table to query.
+##
+## key: the key to lookup.
+##
+## Returns: the value associated with the key.  If the key did not exist, then
+##          the optional field of the returned record is not set.
+function BrokerComm::table_lookup%(t: BrokerComm::Data, key: BrokerComm::Data%): BrokerComm::Data
+	%{
+	auto& table = bro_broker::require_data_type<broker::table>(t->AsRecordVal(),
+													     TYPE_TABLE, frame);
+	auto& k = bro_broker::opaque_field_to_data(key->AsRecordVal(), frame);
+	auto it = table.find(k);
+
+	if ( it == table.end() )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+	else
+		return bro_broker::make_data_val(it->second);
+	%}
+
+## Create an iterator for a table.  Note that this makes a copy of the table
+## internally to ensure the iterator is always valid.
+##
+## t: the table to iterate over.
+##
+## Returns: an iterator.
+function BrokerComm::table_iterator%(t: BrokerComm::Data%): opaque of BrokerComm::TableIterator
+	%{
+	return new bro_broker::TableIterator(t->AsRecordVal(), TYPE_TABLE, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function BrokerComm::table_iterator_last%(it: opaque of BrokerComm::TableIterator%): bool
+	%{
+	auto ti = static_cast<bro_broker::TableIterator*>(it);
+	return new Val(ti->it == ti->dat.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function BrokerComm::table_iterator_next%(it: opaque of BrokerComm::TableIterator%): bool
+	%{
+	auto ti = static_cast<bro_broker::TableIterator*>(it);
+
+	if ( ti->it == ti->dat.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++ti->it;
+	return new Val(ti->it != ti->dat.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function BrokerComm::table_iterator_value%(it: opaque of BrokerComm::TableIterator%): BrokerComm::TableItem
+	%{
+	auto ti = static_cast<bro_broker::TableIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::BrokerComm::TableItem);
+	auto key_val = new RecordVal(BifType::Record::BrokerComm::Data);
+	auto val_val = new RecordVal(BifType::Record::BrokerComm::Data);
+	rval->Assign(0, key_val);
+	rval->Assign(1, val_val);
+
+	if ( ti->it == ti->dat.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid table iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	key_val->Assign(0, new bro_broker::DataVal(ti->it->first));
+	val_val->Assign(0, new bro_broker::DataVal(ti->it->second));
+	return rval;
+	%}
+
+## Create communication data of type "vector".
+function BrokerComm::vector_create%(%): BrokerComm::Data
+	%{
+	return bro_broker::make_data_val(broker::vector());
+	%}
+
+## Remove all elements within a vector.
+##
+## v: the vector to clear.
+##
+## Returns: always true.
+function BrokerComm::vector_clear%(v: BrokerComm::Data%): bool
+	%{
+	auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	vec.clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Get the number of elements within a vector.
+##
+## v: the vector to query.
+##
+## Returns: the number of elements in the vector.
+function BrokerComm::vector_size%(v: BrokerComm::Data%): count
+	%{
+	auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	return new Val(static_cast<uint64_t>(vec.size()), TYPE_COUNT);
+	%}
+
+## Insert an element into a vector at a particular position, possibly displacing
+## existing elements (insertion always grows the size of the vector by one).
+##
+## v: the vector to modify.
+##
+## d: the element to insert.
+##
+## idx: the index at which to insert the data.  If it is greater than the
+##      current size of the vector, the element is inserted at the end.
+##
+## Returns: always true.
+function BrokerComm::vector_insert%(v: BrokerComm::Data, d: BrokerComm::Data, idx: count%): bool
+	%{
+	auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame);
+	idx = min(idx, static_cast<uint64_t>(vec.size()));
+	vec.insert(vec.begin() + idx, item);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Replace an element in a vector at a particular position.
+##
+## v: the vector to modify.
+##
+## d: the element to insert.
+##
+## idx: the index to replace.
+##
+## Returns: the value that was just evicted.  If the index was larger than any
+##          valid index, the optional field of the returned record is not set.
+function BrokerComm::vector_replace%(v: BrokerComm::Data, d: BrokerComm::Data, idx: count%): BrokerComm::Data
+	%{
+	auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame);
+
+	if ( idx >= vec.size() )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+
+	auto rval = bro_broker::make_data_val(move(vec[idx]));
+	vec[idx] = item;
+	return rval;
+	%}
+
+## Remove an element from a vector at a particular position.
+##
+## v: the vector to modify.
+##
+## idx: the index to remove.
+##
+## Returns: the value that was just evicted.  If the index was larger than any
+##          valid index, the optional field of the returned record is not set.
+function BrokerComm::vector_remove%(v: BrokerComm::Data, idx: count%): BrokerComm::Data
+	%{
+	auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+
+	if ( idx >= vec.size() )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+
+	auto rval = bro_broker::make_data_val(move(vec[idx]));
+	vec.erase(vec.begin() + idx);
+	return rval;
+	%}
+
+## Lookup an element in a vector at a particular position.
+##
+## v: the vector to query.
+##
+## idx: the index to lookup.
+##
+## Returns: the value at the index.  If the index was larger than any
+##          valid index, the optional field of the returned record is not set.
+function BrokerComm::vector_lookup%(v: BrokerComm::Data, idx: count%): BrokerComm::Data
+	%{
+	auto& vec = bro_broker::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+
+	if ( idx >= vec.size() )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+
+	return bro_broker::make_data_val(vec[idx]);
+	%}
+
+## Create an iterator for a vector.  Note that this makes a copy of the vector
+## internally to ensure the iterator is always valid.
+##
+## v: the vector to iterate over.
+##
+## Returns: an iterator.
+function BrokerComm::vector_iterator%(v: BrokerComm::Data%): opaque of BrokerComm::VectorIterator
+	%{
+	return new bro_broker::VectorIterator(v->AsRecordVal(), TYPE_VECTOR, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function BrokerComm::vector_iterator_last%(it: opaque of BrokerComm::VectorIterator%): bool
+	%{
+	auto vi = static_cast<bro_broker::VectorIterator*>(it);
+	return new Val(vi->it == vi->dat.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function BrokerComm::vector_iterator_next%(it: opaque of BrokerComm::VectorIterator%): bool
+	%{
+	auto vi = static_cast<bro_broker::VectorIterator*>(it);
+
+	if ( vi->it == vi->dat.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++vi->it;
+	return new Val(vi->it != vi->dat.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function BrokerComm::vector_iterator_value%(it: opaque of BrokerComm::VectorIterator%): BrokerComm::Data
+	%{
+	auto vi = static_cast<bro_broker::VectorIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
+
+	if ( vi->it == vi->dat.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid vector iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	rval->Assign(0, new bro_broker::DataVal(*vi->it));
+	return rval;
+	%}
+
+## Create communication data of type "record".
+##
+## sz: the number of fields in the record.
+##
+## Returns: record data, with all fields uninitialized.
+function BrokerComm::record_create%(sz: count%): BrokerComm::Data
+	%{
+	return bro_broker::make_data_val(broker::record(std::vector<broker::record::field>(sz)));
+	%}
+
+## Get the number of fields within a record.
+##
+## r: the record to query.
+##
+## Returns: the number of fields in the record.
+function BrokerComm::record_size%(r: BrokerComm::Data%): count
+	%{
+	auto& v = bro_broker::require_data_type<broker::record>(r->AsRecordVal(),
+													  TYPE_RECORD, frame);
+	return new Val(static_cast<uint64_t>(v.fields.size()), TYPE_COUNT);
+	%}
+
+## Replace a field in a record at a particular position.
+##
+## t: the table to modify.
+##
+## d: the new field value to assign.
+##
+## idx: the index to replace.
+##
+## Returns: false if the index was larger than any valid index, else true.
+function BrokerComm::record_assign%(r: BrokerComm::Data, d: BrokerComm::Data, idx: count%): bool
+	%{
+	auto& v = bro_broker::require_data_type<broker::record>(r->AsRecordVal(),
+													  TYPE_RECORD, frame);
+	auto& item = bro_broker::opaque_field_to_data(d->AsRecordVal(), frame);
+
+	if ( idx >= v.fields.size() )
+		return new Val(false, TYPE_BOOL);
+
+	v.fields[idx] = item;
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Lookup a field in a record at a particular position.
+##
+## r: the record to query.
+##
+## idx: the index to lookup.
+##
+## Returns: the value at the index.  The optional field of the returned record
+##          may not be set if the field of the record has no value or if the
+##          the index was not valid.
+function BrokerComm::record_lookup%(r: BrokerComm::Data, idx: count%): BrokerComm::Data
+	%{
+	auto& v = bro_broker::require_data_type<broker::record>(r->AsRecordVal(),
+													  TYPE_RECORD, frame);
+
+	if ( idx >= v.size() )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+
+	if ( ! v.fields[idx] )
+		return new RecordVal(BifType::Record::BrokerComm::Data);
+
+	return bro_broker::make_data_val(*v.fields[idx]);
+	%}
+
+## Create an iterator for a record.  Note that this makes a copy of the record
+## internally to ensure the iterator is always valid.
+##
+## r: the record to iterate over.
+##
+## Returns: an iterator.
+function BrokerComm::record_iterator%(r: BrokerComm::Data%): opaque of BrokerComm::RecordIterator
+	%{
+	return new bro_broker::RecordIterator(r->AsRecordVal(), TYPE_RECORD, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function BrokerComm::record_iterator_last%(it: opaque of BrokerComm::RecordIterator%): bool
+	%{
+	auto ri = static_cast<bro_broker::RecordIterator*>(it);
+	return new Val(ri->it == ri->dat.fields.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function BrokerComm::record_iterator_next%(it: opaque of BrokerComm::RecordIterator%): bool
+	%{
+	auto ri = static_cast<bro_broker::RecordIterator*>(it);
+
+	if ( ri->it == ri->dat.fields.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++ri->it;
+	return new Val(ri->it != ri->dat.fields.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function BrokerComm::record_iterator_value%(it: opaque of BrokerComm::RecordIterator%): BrokerComm::Data
+	%{
+	auto ri = static_cast<bro_broker::RecordIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::BrokerComm::Data);
+
+	if ( ri->it == ri->dat.fields.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid record iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	if ( ! *ri->it )
+		return rval; // field isn't set
+
+	rval->Assign(0, new bro_broker::DataVal(**ri->it));
+	return rval;
+	%}

src/broker/messaging.bif

@@ -0,0 +1,211 @@
+
+##! Functions for peering and various messaging patterns (e.g. print/log/event).
+
+%%{
+#include "broker/Manager.h"
+#include "logging/Manager.h"
+%%}
+
+module BrokerComm;
+
+type BrokerComm::SendFlags: record;
+
+type BrokerComm::EventArgs: record;
+
+## Used to handle remote print messages from peers that call
+## :bro:see:`BrokerComm::print`.
+event BrokerComm::print_handler%(msg: string%);
+
+## Print a simple message to any interested peers.  The receiver can use
+## :bro:see:`BrokerComm::print_handler` to handle messages.
+##
+## topic: a topic associated with the printed message.
+##
+## msg: the print message to send to peers.
+##
+## flags: tune the behavior of how the message is sent.
+##
+## Returns: true if the message is sent.
+function BrokerComm::print%(topic: string, msg: string,
+                      flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = broker_mgr->Print(topic->CheckString(), msg->CheckString(),
+	                            flags);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Register interest in all peer print messages that use a certain topic prefix.
+## use :bro:see:`BrokerComm::print_handler` to handle received messages.
+##
+## topic_prefix: a prefix to match against remote message topics.
+##               e.g. an empty prefix matches everything and "a" matches
+##               "alice" and "amy" but not "bob".
+##
+## Returns: true if it's a new print subscription and it is now registered.
+function BrokerComm::subscribe_to_prints%(topic_prefix: string%): bool
+	%{
+	auto rval = broker_mgr->SubscribeToPrints(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Unregister interest in all peer print messages that use a topic prefix.
+##
+## topic_prefix: a prefix previously supplied to a successful call to
+##               :bro:see:`BrokerComm::subscribe_to_prints`.
+##
+## Returns: true if interest in the topic prefix is no longer advertised.
+function BrokerComm::unsubscribe_to_prints%(topic_prefix: string%): bool
+	%{
+	auto rval = broker_mgr->UnsubscribeToPrints(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Create a data structure that may be used to send a remote event via
+## :bro:see:`BrokerComm::event`.
+##
+## args: an event, followed by a list of argument values that may be used
+##       to call it.
+##
+## Returns: opaque communication data that may be used to send a remote event.
+function BrokerComm::event_args%(...%): BrokerComm::EventArgs
+	%{
+	auto rval = broker_mgr->MakeEventArgs(@ARGS@);
+	return rval;
+	%}
+
+## Send an event to any interested peers.
+##
+## topic: a topic associated with the event message.
+##
+## args: event arguments as made by :bro:see:`BrokerComm::event_args`.
+##
+## flags: tune the behavior of how the message is sent.
+##
+## Returns: true if the message is sent.
+function BrokerComm::event%(topic: string, args: BrokerComm::EventArgs,
+                      flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = broker_mgr->Event(topic->CheckString(), args->AsRecordVal(),
+	                            flags);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Automatically send an event to any interested peers whenever it is
+## locally dispatched (e.g. using "event my_event(...);" in a script).
+##
+## topic: a topic string associated with the event message.
+##        Peers advertise interest by registering a subscription to some prefix
+##        of this topic name.
+##
+## ev: a Bro event value.
+##
+## flags: tune the behavior of how the message is send.
+##
+## Returns: true if automatic event sending is now enabled.
+function BrokerComm::auto_event%(topic: string, ev: any,
+                           flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = broker_mgr->AutoEvent(topic->CheckString(), ev, flags);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Stop automatically sending an event to peers upon local dispatch.
+##
+## topic: a topic originally given to :bro:see:`BrokerComm::auto_event`.
+##
+## ev: an event originally given to :bro:see:`BrokerComm::auto_event`.
+##
+## Returns: true if automatic events will no occur for the topic/event pair.
+function BrokerComm::auto_event_stop%(topic: string, ev: any%): bool
+	%{
+	auto rval = broker_mgr->AutoEventStop(topic->CheckString(), ev);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Register interest in all peer event messages that use a certain topic prefix.
+##
+## topic_prefix: a prefix to match against remote message topics.
+##               e.g. an empty prefix matches everything and "a" matches
+##               "alice" and "amy" but not "bob".
+##
+## Returns: true if it's a new event subscription and it is now registered.
+function BrokerComm::subscribe_to_events%(topic_prefix: string%): bool
+	%{
+	auto rval = broker_mgr->SubscribeToEvents(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Unregister interest in all peer event messages that use a topic prefix.
+##
+## topic_prefix: a prefix previously supplied to a successful call to
+##               :bro:see:`BrokerComm::subscribe_to_events`.
+##
+## Returns: true if interest in the topic prefix is no longer advertised.
+function BrokerComm::unsubscribe_to_events%(topic_prefix: string%): bool
+	%{
+	auto rval = broker_mgr->UnsubscribeToEvents(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Enable remote logs for a given log stream.
+##
+## id: the log stream to enable remote logs for.
+##
+## flags: tune the behavior of how log entry messages are sent.
+##
+## Returns: true if remote logs are enabled for the stream.
+function
+BrokerComm::enable_remote_logs%(id: Log::ID,
+                          flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = log_mgr->EnableRemoteLogs(id->AsEnumVal(),
+	                                   bro_broker::Manager::send_flags_to_int(flags));
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Disable remote logs for a given log stream.
+##
+## id: the log stream to disable remote logs for.
+##
+## Returns: true if remote logs are disabled for the stream.
+function BrokerComm::disable_remote_logs%(id: Log::ID%): bool
+	%{
+	auto rval = log_mgr->DisableRemoteLogs(id->AsEnumVal());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Returns: true if remote logs are enabled for the given stream.
+function BrokerComm::remote_logs_enabled%(id: Log::ID%): bool
+	%{
+	auto rval = log_mgr->RemoteLogsAreEnabled(id->AsEnumVal());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Register interest in all peer log messages that use a certain topic prefix.
+## Logs are implicitly sent with topic "bro/log/<stream-name>" and the
+## receiving side processes them through the logging framework as usual.
+##
+## topic_prefix: a prefix to match against remote message topics.
+##               e.g. an empty prefix matches everything and "a" matches
+##               "alice" and "amy" but not "bob".
+##
+## Returns: true if it's a new log subscription and it is now registered.
+function BrokerComm::subscribe_to_logs%(topic_prefix: string%): bool
+	%{
+	auto rval = broker_mgr->SubscribeToLogs(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Unregister interest in all peer log messages that use a topic prefix.
+## Logs are implicitly sent with topic "bro/log/<stream-name>" and the
+## receiving side processes them through the logging framework as usual.
+##
+## topic_prefix: a prefix previously supplied to a successful call to
+##               :bro:see:`BrokerComm::subscribe_to_logs`.
+##
+## Returns: true if interest in the topic prefix is no longer advertised.
+function BrokerComm::unsubscribe_to_logs%(topic_prefix: string%): bool
+	%{
+	auto rval = broker_mgr->UnsubscribeToLogs(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}

src/broker/store.bif

@@ -0,0 +1,597 @@
+
+##! Functions to interface with broker's distributed data store.
+
+%%{
+#include "broker/Manager.h"
+#include "broker/Store.h"
+#include "broker/Data.h"
+#include "Trigger.h"
+%%}
+
+module BrokerStore;
+
+type BrokerStore::ExpiryTime: record;
+
+type BrokerStore::QueryResult: record;
+
+type BrokerStore::BackendOptions: record;
+
+## Enumerates the possible storage backends.
+enum BackendType %{
+	MEMORY,
+	SQLITE,
+	ROCKSDB,
+%}
+
+## Create a master data store which contains key-value pairs.
+##
+## id: a unique name for the data store.
+##
+## b: the storage backend to use.
+##
+## options: tunes how some storage backends operate.
+##
+## Returns: a handle to the data store.
+function BrokerStore::create_master%(id: string, b: BackendType &default = MEMORY,
+                               options: BackendOptions &default = BackendOptions()%): opaque of BrokerStore::Handle
+	%{
+	auto id_str = id->CheckString();
+	auto type = bro_broker::StoreType::MASTER;
+	auto rval = broker_mgr->LookupStore(id_str, type);
+
+	if ( rval )
+		{
+		Ref(rval);
+		return rval;
+		}
+
+	rval = new bro_broker::StoreHandleVal(id_str, type,
+	                                static_cast<BifEnum::BrokerStore::BackendType>(b->AsEnum()),
+	                                options->AsRecordVal());
+	auto added = broker_mgr->AddStore(rval);
+	assert(added);
+	return rval;
+	%}
+
+## Create a clone of a master data store which may live with a remote peer.
+## A clone automatically synchronizes to the master by automatically receiving
+## modifications and applying them locally.  Direct modifications are not
+## possible, they must be sent through the master store, which then
+## automatically broadcasts the changes out to clones.  But queries may be made
+## directly against the local cloned copy, which may be resolved quicker than
+## reaching out to a remote master store.
+##
+## id: the unique name which identifies the master data store.
+##
+## b: the storage backend to use.
+##
+## options: tunes how some storage backends operate.
+##
+## resync: the interval at which to re-attempt synchronizing with the master
+##         store should the connection be lost.  If the clone has not yet
+##         synchronized for the first time, updates and queries queue up until
+##         the synchronization completes.  After, if the connection to the
+##         master store is lost, queries continue to use the clone's version,
+##         but updates will be lost until the master is once again available.
+##
+## Returns: a handle to the data store.
+function BrokerStore::create_clone%(id: string, b: BackendType &default = MEMORY,
+                              options: BackendOptions &default = BackendOptions(),
+                              resync: interval &default = 1sec%): opaque of BrokerStore::Handle
+	%{
+	auto id_str = id->CheckString();
+	auto type = bro_broker::StoreType::CLONE;
+	auto rval = broker_mgr->LookupStore(id_str, type);
+
+	if ( rval )
+		{
+		Ref(rval);
+		return rval;
+		}
+
+	rval = new bro_broker::StoreHandleVal(id_str, type,
+	                                static_cast<BifEnum::BrokerStore::BackendType>(b->AsEnum()),
+	                                options->AsRecordVal(),
+	                                std::chrono::duration<double>(resync));
+	auto added = broker_mgr->AddStore(rval);
+	assert(added);
+	return rval;
+	%}
+
+## Create a frontend interface to an existing master data store that allows
+## querying and updating its contents.
+##
+## id: the unique name which identifies the master data store.
+##
+## Returns: a handle to the data store.
+function BrokerStore::create_frontend%(id: string%): opaque of BrokerStore::Handle
+	%{
+	auto id_str = id->CheckString();
+	auto type = bro_broker::StoreType::FRONTEND;
+	auto rval = broker_mgr->LookupStore(id_str, type);
+
+	if ( rval )
+		{
+		Ref(rval);
+		return rval;
+		}
+
+	rval = new bro_broker::StoreHandleVal(id_str, type, {}, nullptr);
+	auto added = broker_mgr->AddStore(rval);
+	assert(added);
+	return rval;
+	%}
+
+## Close a data store.
+##
+## h: a data store handle.
+##
+## Returns: true if store was valid and is now closed.  The handle can no
+##          longer be used for data store operations.
+function BrokerStore::close_by_handle%(h: opaque of BrokerStore::Handle%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	return new Val(broker_mgr->CloseStore(handle->store->id(),
+										handle->store_type), TYPE_BOOL);
+	%}
+
+###########################
+# non-blocking update API #
+###########################
+
+## Insert a key-value pair in to the store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key to insert.
+##
+## v: the value to insert.
+##
+## e: the expiration time of the key-value pair.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::insert%(h: opaque of BrokerStore::Handle,
+                        k: BrokerComm::Data, v: BrokerComm::Data,
+                        e: BrokerStore::ExpiryTime &default = BrokerStore::ExpiryTime()%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	auto& val = bro_broker::opaque_field_to_data(v->AsRecordVal(), frame);
+
+	using broker::store::expiration_time;
+
+	auto abs_expiry_val = e->AsRecordVal()->Lookup(0);
+
+	if ( abs_expiry_val )
+		{
+		auto expiry = expiration_time(abs_expiry_val->AsTime());
+		handle->store->insert(key, val, expiry);
+		return new Val(true, TYPE_BOOL);
+		}
+
+	auto rel_expiry_val = e->AsRecordVal()->Lookup(1);
+
+	if ( rel_expiry_val )
+		{
+		auto ct = broker::time_point::now().value;
+		auto expiry = expiration_time(rel_expiry_val->AsInterval(), ct);
+		handle->store->insert(key, val, expiry);
+		return new Val(true, TYPE_BOOL);
+		}
+
+	handle->store->insert(key, val);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Remove a key-value pair from the store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key to remove.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::erase%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	handle->store->erase(key);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Remove all key-value pairs from the store.
+##
+## h: the handle of the store to modify.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::clear%(h: opaque of BrokerStore::Handle%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	handle->store->clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Increment an integer value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## by: the amount to increment the value by.  A non-existent key will first
+##     create it with an implicit value of zero before incrementing.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::increment%(h: opaque of BrokerStore::Handle,
+                           k: BrokerComm::Data, by: int &default = +1%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	handle->store->increment(key, by);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Decrement an integer value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## by: the amount to decrement the value by.  A non-existent key will first
+##     create it with an implicit value of zero before decrementing.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::decrement%(h: opaque of BrokerStore::Handle,
+                           k: BrokerComm::Data, by: int &default = +1%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	handle->store->decrement(key, by);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Add an element to a set value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## element: the element to add to the set.  A non-existent key will first
+##          create it with an implicit empty set value before modifying.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::add_to_set%(h: opaque of BrokerStore::Handle,
+                            k: BrokerComm::Data, element: BrokerComm::Data%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	auto& ele = bro_broker::opaque_field_to_data(element->AsRecordVal(), frame);
+	handle->store->add_to_set(key, ele);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Remove an element from a set value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## element: the element to remove from the set.  A non-existent key will
+##          implicitly create an empty set value associated with the key.
+##
+## Returns: false if the store handle was not valid.
+function BrokerStore::remove_from_set%(h: opaque of BrokerStore::Handle,
+                                 k: BrokerComm::Data, element: BrokerComm::Data%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	auto& ele = bro_broker::opaque_field_to_data(element->AsRecordVal(), frame);
+	handle->store->remove_from_set(key, ele);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Add a new item to the head of a vector value in a data store.
+##
+## h: the handle of store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## item: the element to insert in to the vector.  A non-existent key will first
+##       create empty vector value before modifying.
+##
+## Returns: the handle of store to modify.
+function BrokerStore::push_left%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data,
+                           items: BrokerComm::DataVector%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	broker::vector items_vector;
+	auto items_vv = items->AsVector();
+
+	for ( auto i = 0u; i < items_vv->size(); ++i )
+		{
+		auto& item = bro_broker::opaque_field_to_data((*items_vv)[i]->AsRecordVal(),
+		                                        frame);
+		items_vector.emplace_back(item);
+		}
+
+	handle->store->push_left(key, move(items_vector));
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Add a new item to the tail of a vector value in a data store.
+##
+## h: the handle of store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## item: the element to insert in to the vector.  A non-existent key will first
+##       create empty vector value before modifying.
+##
+## Returns: the handle of store to modify.
+function BrokerStore::push_right%(h: opaque of BrokerStore::Handle, k: BrokerComm::Data,
+                            items: BrokerComm::DataVector%): bool
+	%{
+	auto handle = static_cast<bro_broker::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = bro_broker::opaque_field_to_data(k->AsRecordVal(), frame);
+	broker::vector items_vector;
+	auto items_vv = items->AsVector();
+
+	for ( auto i = 0u; i < items_vv->size(); ++i )
+		{
+		auto& item = bro_broker::opaque_field_to_data((*items_vv)[i]->AsRecordVal(),
+		                                        frame);
+		items_vector.emplace_back(item);
+		}
+
+	handle->store->push_right(key, move(items_vector));
+	return new Val(true, TYPE_BOOL);
+	%}
+
+##########################
+# non-blocking query API #
+##########################
+
+%%{
+static bool prepare_for_query(Val* opaque, Frame* frame,
+			      bro_broker::StoreHandleVal** handle,
+			      double* timeout,
+			      bro_broker::StoreQueryCallback** cb)
+	{
+	*handle = static_cast<bro_broker::StoreHandleVal*>(opaque);
+
+	if ( ! (*handle)->store )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Error("BrokerStore query has an invalid data store");
+		reporter->PopLocation();
+		return false;
+		}
+
+	Trigger* trigger = frame->GetTrigger();
+
+	if ( ! trigger )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Error("BrokerStore queries can only be called inside when-condition");
+		reporter->PopLocation();
+		return false;
+		}
+
+	*timeout = trigger->TimeoutValue();
+
+	if ( *timeout < 0 )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Error("BrokerStore queries must specify a timeout block");
+		reporter->PopLocation();
+		return false;
+		}
+
+	frame->SetDelayed();
+	trigger->Hold();
+	*cb = new bro_broker::StoreQueryCallback(trigger, frame->GetCall(),
+					   (*handle)->store->id(),
+					   (*handle)->store_type);
+	broker_mgr->TrackStoreQuery(*cb);
+	return true;
+	}
+
+%%}
+
+## Pop the head of a data store vector value.
+##
+## h: the handle of the store to query.
+##
+## k: the key associated with the vector to modify.
+##
+## Returns: the result of the query.
+function BrokerStore::pop_left%(h: opaque of BrokerStore::Handle,
+                          k: BrokerComm::Data%): BrokerStore::QueryResult
+	%{
+	if ( ! broker_mgr->Enabled() )
+		return bro_broker::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return bro_broker::query_result();
+
+	double timeout;
+	bro_broker::StoreQueryCallback* cb;
+	bro_broker::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return bro_broker::query_result();
+
+	handle->store->pop_left(static_cast<bro_broker::DataVal*>(key)->data,
+	                         std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Pop the tail of a data store vector value.
+##
+## h: the handle of the store to query.
+##
+## k: the key associated with the vector to modify.
+##
+## Returns: the result of the query.
+function BrokerStore::pop_right%(h: opaque of BrokerStore::Handle,
+                           k: BrokerComm::Data%): BrokerStore::QueryResult
+	%{
+	if ( ! broker_mgr->Enabled() )
+		return bro_broker::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return bro_broker::query_result();
+
+	double timeout;
+	bro_broker::StoreQueryCallback* cb;
+	bro_broker::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return bro_broker::query_result();
+
+	handle->store->pop_right(static_cast<bro_broker::DataVal*>(key)->data,
+	                         std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Lookup the value associated with a key in a data store.
+##
+## h: the handle of the store to query.
+##
+## k: the key to lookup.
+##
+## Returns: the result of the query.
+function BrokerStore::lookup%(h: opaque of BrokerStore::Handle,
+                       k: BrokerComm::Data%): BrokerStore::QueryResult
+	%{
+	if ( ! broker_mgr->Enabled() )
+		return bro_broker::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return bro_broker::query_result();
+
+	double timeout;
+	bro_broker::StoreQueryCallback* cb;
+	bro_broker::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return bro_broker::query_result();
+
+	handle->store->lookup(static_cast<bro_broker::DataVal*>(key)->data,
+	                      std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Check if a data store contains a given key.
+##
+## h: the handle of the store to query.
+##
+## k: the key to check for existence.
+##
+## Returns: the result of the query (uses :bro:see:`BrokerComm::BOOL`).
+function BrokerStore::exists%(h: opaque of BrokerStore::Handle,
+                        k: BrokerComm::Data%): BrokerStore::QueryResult
+	%{
+	if ( ! broker_mgr->Enabled() )
+		return bro_broker::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return bro_broker::query_result();
+
+	double timeout;
+	bro_broker::StoreQueryCallback* cb;
+	bro_broker::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return bro_broker::query_result();
+
+	handle->store->exists(static_cast<bro_broker::DataVal*>(key)->data,
+	                      std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Retrieve all keys in a data store.
+##
+## h: the handle of the store to query.
+##
+## Returns: the result of the query (uses :bro:see:`BrokerComm::VECTOR`).
+function BrokerStore::keys%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult
+	%{
+	double timeout;
+	bro_broker::StoreQueryCallback* cb;
+	bro_broker::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return bro_broker::query_result();
+
+	handle->store->keys(std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Get the number of key-value pairs in a data store.
+##
+## h: the handle of the store to query.
+##
+## Returns: the result of the query (uses :bro:see:`BrokerComm::COUNT`).
+function BrokerStore::size%(h: opaque of BrokerStore::Handle%): BrokerStore::QueryResult
+	%{
+	if ( ! broker_mgr->Enabled() )
+		return bro_broker::query_result();
+
+	double timeout;
+	bro_broker::StoreQueryCallback* cb;
+	bro_broker::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return bro_broker::query_result();
+
+	handle->store->size(std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}

src/file_analysis/analyzer/x509/X509.cc

@@ -104,13 +104,35 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	len = BIO_gets(bio, buf, sizeof(buf));
 	pX509Cert->Assign(2, new StringVal(len, buf));
 	BIO_reset(bio);
+
+	X509_NAME *subject_name = X509_get_subject_name(ssl_cert);
+	// extract the most specific (last) common name from the subject
+	int namepos = -1;
+	for ( ;; )
+		{
+		int j = X509_NAME_get_index_by_NID(subject_name, NID_commonName, namepos);
+		if ( j == -1 )
+			break;
+
+		namepos = j;
+		}
+
+	if ( namepos != -1 )
+		{
+		// we found a common name
+		ASN1_STRING_print(bio, X509_NAME_ENTRY_get_data(X509_NAME_get_entry(subject_name, namepos)));
+		len = BIO_gets(bio, buf, sizeof(buf));
+		pX509Cert->Assign(4, new StringVal(len, buf));
+		BIO_reset(bio);
+		}
+
 	X509_NAME_print_ex(bio, X509_get_issuer_name(ssl_cert), 0, XN_FLAG_RFC2253);
 	len = BIO_gets(bio, buf, sizeof(buf));
 	pX509Cert->Assign(3, new StringVal(len, buf));
 	BIO_free(bio);
 
-	pX509Cert->Assign(4, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notBefore(ssl_cert)), TYPE_TIME));
-	pX509Cert->Assign(5, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
+	pX509Cert->Assign(6, new Val(GetTimeFromAsn1(X509_get_notAfter(ssl_cert)), TYPE_TIME));
 
 	// we only read 255 bytes because byte 256 is always 0.
 	// if the string is longer than 255, that will be our null-termination,
@@ -118,28 +140,28 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->cert_info->key->algor->algorithm) )
 		buf[0] = 0;
 
-	pX509Cert->Assign(6, new StringVal(buf));
+	pX509Cert->Assign(7, new StringVal(buf));
 
 	if ( ! i2t_ASN1_OBJECT(buf, 255, ssl_cert->sig_alg->algorithm) )
 		buf[0] = 0;
 
-	pX509Cert->Assign(7, new StringVal(buf));
+	pX509Cert->Assign(8, new StringVal(buf));
 
 	// Things we can do when we have the key...
 	EVP_PKEY *pkey = X509_extract_key(ssl_cert);
 	if ( pkey != NULL )
 		{
 		if ( pkey->type == EVP_PKEY_DSA )
-			pX509Cert->Assign(8, new StringVal("dsa"));
+			pX509Cert->Assign(9, new StringVal("dsa"));
 
 		else if ( pkey->type == EVP_PKEY_RSA )
 			{
-			pX509Cert->Assign(8, new StringVal("rsa"));
+			pX509Cert->Assign(9, new StringVal("rsa"));
 
 			char *exponent = BN_bn2dec(pkey->pkey.rsa->e);
 			if ( exponent != NULL )
 				{
-				pX509Cert->Assign(10, new StringVal(exponent));
+				pX509Cert->Assign(11, new StringVal(exponent));
 				OPENSSL_free(exponent);
 				exponent = NULL;
 				}
@@ -147,14 +169,14 @@ RecordVal* file_analysis::X509::ParseCertificate(X509Val* cert_val)
 #ifndef OPENSSL_NO_EC
 		else if ( pkey->type == EVP_PKEY_EC )
 			{
-			pX509Cert->Assign(8, new StringVal("ecdsa"));
+			pX509Cert->Assign(9, new StringVal("ecdsa"));
-			pX509Cert->Assign(11, KeyCurve(pkey));
+			pX509Cert->Assign(12, KeyCurve(pkey));
 			}
 #endif
 
 		unsigned int length = KeyLength(pkey);
 		if ( length > 0 )
-			pX509Cert->Assign(9, new Val(length, TYPE_COUNT));
+			pX509Cert->Assign(10, new Val(length, TYPE_COUNT));
 
 		EVP_PKEY_free(pkey);
 		}

src/iosource/PktSrc.cc

@@ -13,6 +13,15 @@
 
 using namespace iosource;
 
+PktSrc::Properties::Properties()
+	{
+	selectable_fd = -1;
+	link_type = -1;
+	hdr_size = -1;
+	netmask = NETMASK_UNKNOWN;
+	is_live = false;
+	}
+
 PktSrc::PktSrc()
 	{
 	have_packet = false;
@@ -50,7 +59,7 @@ int PktSrc::LinkType() const
 
 uint32 PktSrc::Netmask() const
 	{
-	return IsOpen() ? props.netmask : PCAP_NETMASK_UNKNOWN;
+	return IsOpen() ? props.netmask : NETMASK_UNKNOWN;
 	}
 
 bool PktSrc::IsError() const

src/iosource/PktSrc.h

@@ -16,6 +16,8 @@ namespace iosource {
  */
 class PktSrc : public IOSource {
 public:
+	static const int NETMASK_UNKNOWN = 0xffffffff;
+
 	/**
 	 * Struct for returning statistics on a packet source.
 	 */
@@ -36,7 +38,12 @@ public:
 		 */
 		unsigned int link;
 
-		Stats()	{ received = dropped = link = 0; }
+		/**
+		  * Bytes received by source after filtering (w/o drops).
+		*/
+		uint64 bytes_received;
+
+		Stats()	{ received = dropped = link = bytes_received = 0; }
 	};
 
 	/**
@@ -67,7 +74,7 @@ public:
 
 	/**
 	 * Returns the netmask associated with the source, or \c
-	 * PCAP_NETMASK_UNKNOWN if unknown.
+	 * NETMASK_UNKNOWN if unknown.
 	 */
 	uint32 Netmask() const;
 
@@ -253,8 +260,8 @@ protected:
 		int hdr_size;
 
 		/**
-		 * The netmask associated with the source, or \c
+		 * Returns the netmask associated with the source, or \c
-		 * PCAP_NETMASK_UNKNOWN if unknown.
+		 * NETMASK_UNKNOWN if unknown.
 		 */
 		uint32 netmask;
 
@@ -264,14 +271,7 @@ protected:
 		 */
 		bool is_live;
 
-		Properties()
+		Properties();
-			{
-			selectable_fd = -1;
-			link_type = -1;
-			hdr_size = -1;
-			netmask = PCAP_NETMASK_UNKNOWN;
-			is_live = false;
-			}
 	};
 
 	/**

src/iosource/pcap/Source.cc

@@ -77,6 +77,12 @@ void PcapSource::OpenLive()
 		props.netmask = 0xffffff00;
 		}
 
+#ifdef PCAP_NETMASK_UNKNOWN
+	// Defined in libpcap >= 1.1.1
+	if ( props.netmask == PCAP_NETMASK_UNKNOWN )
+		props.netmask = PktSrc::NETMASK_UNKNOWN;
+#endif
+
 	// We use the smallest time-out possible to return almost immediately if
 	// no packets are available. (We can't use set_nonblocking() as it's
 	// broken on FreeBSD: even when select() indicates that we can read
@@ -174,6 +180,8 @@ bool PcapSource::ExtractNextPacket(Packet* pkt)
 	last_hdr = current_hdr;
 	last_data = data;
 	++stats.received;
+	stats.bytes_received += current_hdr.len;
+
 	return true;
 	}
 
@@ -213,7 +221,7 @@ bool PcapSource::SetFilter(int index)
 
 #ifndef HAVE_LINUX
 	// Linux doesn't clear counters when resetting filter.
-	stats.received = stats.dropped = stats.link = 0;
+	stats.received = stats.dropped = stats.link = stats.bytes_received = 0;
 #endif
 
 	return true;
@@ -224,7 +232,7 @@ void PcapSource::Statistics(Stats* s)
 	char errbuf[PCAP_ERRBUF_SIZE];
 
 	if ( ! (props.is_live && pd) )
-		s->received = s->dropped = s->link = 0;
+		s->received = s->dropped = s->link = s->bytes_received = 0;
 
 	else
 		{
@@ -232,7 +240,7 @@ void PcapSource::Statistics(Stats* s)
 		if ( pcap_stats(pd, &pstat) < 0 )
 			{
 			PcapError();
-			s->received = s->dropped = s->link = 0;
+			s->received = s->dropped = s->link = s->bytes_received = 0;
 			}
 
 		else
@@ -243,6 +251,7 @@ void PcapSource::Statistics(Stats* s)
 		}
 
 	s->received = stats.received;
+	s->bytes_received = stats.bytes_received;
 
 	if ( ! props.is_live )
 		s->dropped = 0;

src/logging/Manager.cc

@@ -16,6 +16,10 @@
 #include "WriterBackend.h"
 #include "logging.bif.h"
 
+#ifdef ENABLE_BROKER
+#include "broker/Manager.h"
+#endif
+
 using namespace logging;
 
 struct Manager::Filter {
@@ -69,6 +73,11 @@ struct Manager::Stream {
 
 	WriterMap writers;	// Writers indexed by id/path pair.
 
+#ifdef ENABLE_BROKER
+	bool enable_remote;
+	int remote_flags;
+#endif
+
 	~Stream();
 	};
 
@@ -287,6 +296,11 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval)
 	streams[idx]->event = event ? event_registry->Lookup(event->Name()) : 0;
 	streams[idx]->columns = columns->Ref()->AsRecordType();
 
+#ifdef ENABLE_BROKER
+	streams[idx]->enable_remote = internal_val("Log::enable_remote_logging")->AsBool();
+	streams[idx]->remote_flags = broker::PEERS;
+#endif
+
 	DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s",
 		streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : "<none>");
 
@@ -828,6 +842,12 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 #endif
 		}
 
+#ifdef ENABLE_BROKER
+	if ( stream->enable_remote &&
+	     ! broker_mgr->Log(id, columns, stream->columns, stream->remote_flags) )
+			stream->enable_remote = false;
+#endif
+
 	Unref(columns);
 
 	if ( error )
@@ -1206,6 +1226,53 @@ void Manager::Terminate()
 		}
 	}
 
+#ifdef ENABLE_BROKER
+
+bool Manager::EnableRemoteLogs(EnumVal* stream_id, int flags)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return false;
+
+	stream->enable_remote = true;
+	stream->remote_flags = flags;
+	return true;
+	}
+
+bool Manager::DisableRemoteLogs(EnumVal* stream_id)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return false;
+
+	stream->enable_remote = false;
+	return true;
+	}
+
+bool Manager::RemoteLogsAreEnabled(EnumVal* stream_id)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return false;
+
+	return stream->enable_remote;
+	}
+
+RecordType* Manager::StreamColumns(EnumVal* stream_id)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return nullptr;
+
+	return stream->columns;
+	}
+
+#endif
+
 // Timer which on dispatching rotates the filter.
 class RotationTimer : public Timer {
 public: