Merge remote-tracking branch 'origin/topic/jsiwek/broker'

* origin/topic/jsiwek/broker: (34 commits)
  Update broker submodule.
  Update broker submodule.
  broker integration: add missing baselines for doc tests
  broker integration: add prof.log statistics
  broker integration: add high-level usage documentation
  broker integration: add API documentation (broxygen/doxygen)
  broker integration: fix memory leak, add leak tests
  Update broker submodule.
  Improve comm tests.
  Fix gcc compile warnings.
  broker integration: fix unit tests to work when broker is not enabled.
  Add --enable-c++11 configure flag.
  broker integration: add (un)publish/(un)advertise functions
  broker integration: add knobs to set auto publish/advertise behavior
  broker integration: move listen port for unit tests to a btest variable
  broker integration: add events for incoming connection status updates
  broker integration: adapt to change in expiration_time
  Update coverage unit test baselines.
  broker integration: add Comm::enable function
  broker integration: process debug/diagnostic reports from broker
  ...

Conflicts:
	cmake
	testing/btest/Baseline/plugins.hooks/output

.gitmodules

@@ -22,3 +22,6 @@
 [submodule "aux/plugins"]
 	path = aux/plugins
 	url = git://git.bro.org/bro-plugins
+[submodule "aux/broker"]
+	path = aux/broker
+	url = git://git.bro.org/broker

CHANGES

@@ -1,4 +1,21 @@
 
+2.3-489 | 2015-03-02 15:29:30 -0800
+
+  * Integrate Broker, Bro's new communication library. (Jon Siwek)
+
+    See aux/broker/README for more information on Broker, and
+    doc/frameworks/comm.rst for the corresponding Bro script API.
+
+    Broker support is by default off for now; it can be enabled at
+    configure time with --enable-broker. It requires CAF
+    (https://github.com/actor-framework/actor-framework); for now
+    needs the "develop" branch. Broker also requires a C++11 compiler.
+
+    Broker will become a mandatory dependency in future Bro versions.
+
+  * Add --enable-c++11 configure flag to compile Bro's source code in
+    C++11 mode with a corresponding compiler. (Jon Siwek)
+
 2.3-451 | 2015-02-24 16:37:08 -0800
 
   * Updating submodule(s).

CMakeLists.txt

@@ -177,6 +177,17 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR})
 ########################################################################
 ## Recurse on sub-directories
 
+if ( ENABLE_CXX11 )
+    include(RequireCXX11)
+endif ()
+
+if ( ENABLE_BROKER )
+    add_subdirectory(aux/broker)
+    set(brodeps ${brodeps} broker)
+    add_definitions(-DENABLE_BROKER)
+    include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker)
+endif ()
+
 add_subdirectory(src)
 add_subdirectory(scripts)
 add_subdirectory(doc)
@@ -224,6 +235,7 @@ message(
     "\nCXXFLAGS:          ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}"
     "\nCPP:               ${CMAKE_CXX_COMPILER}"
     "\n"
+    "\nBroker:            ${ENABLE_BROKER}"
     "\nBroccoli:          ${INSTALL_BROCCOLI}"
     "\nBroctl:            ${INSTALL_BROCTL}"
     "\nAux. Tools:        ${INSTALL_AUX_TOOLS}"

NEWS

@@ -39,6 +39,25 @@ New Functionality
   ``next`` and ``break`` can be used inside the loop's body just like
   with ``for`` loops.
 
+- Bro now integrates Broker, it's new communication library. See
+  aux/broker/README for more information on Broker, and
+  doc/frameworks/comm.rst for the corresponding Bro script API.
+
+  TODO: Extend with some more information on  Broker.
+
+  Broker support is by default off for now; it can be enabled at
+  configure time with --enable-broker. It requires CAF
+  (https://github.com/actor-framework/actor-framework) as well as a
+  C++11 compiler.
+
+  TODO: Add minumim version for CAF.
+
+  Broker will become a mandatory dependency in future Bro versions.
+
+- Add --enable-c++11 configure flag to compile Bro's source code in
+  C++11 mode with a corresponding compiler. Note that 2.4 will be the
+  last version of Bro that compiles without C++11 support.
+
 Changed Functionality
 ---------------------
 

VERSION

@@ -1 +1 @@
-2.3-451
+2.3-489

aux/broker

@@ -0,0 +1 @@
+Subproject commit 8f7b32e773976fec513ce15237a7e0fdb05b0ebd

cmake

@@ -1 +1 @@
-Subproject commit ff08be5aa1b8eaadbe2775cbc11b499c5f93349e
+Subproject commit 2fd35ab6a6245a005828c32f0aa87eb21698c054

configure

@@ -41,6 +41,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --enable-perftools-debug use Google's perftools for debugging
     --enable-jemalloc      link against jemalloc
     --enable-ruby          build ruby bindings for broccoli (deprecated)
+    --enable-c++11         build using the C++11 standard
+    --enable-broker        enable use of the Broker communication library
+                           (requires C++ Actor Framework and C++11)
     --disable-broccoli     don't build or install the Broccoli library
     --disable-broctl       don't install Broctl
     --disable-auxtools     don't build or install auxiliary tools
@@ -55,6 +58,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-flex=PATH       path to flex executable
     --with-bison=PATH      path to bison executable
     --with-perl=PATH       path to perl executable
+    --with-libcaf=PATH     path to C++ Actor Framework installation
+                           (a required Broker dependency)
 
   Optional Packages in Non-Standard Locations:
     --with-geoip=PATH      path to the libGeoIP install root
@@ -67,6 +72,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-ruby-lib=PATH   path to ruby library
     --with-ruby-inc=PATH   path to ruby headers
     --with-swig=PATH       path to SWIG executable
+    --with-rocksdb=PATH    path to RocksDB installation
+                           (an optional Broker dependency)
 
   Packaging Options (for developers):
     --binary-package       toggle special logic for binary packaging
@@ -176,6 +183,13 @@ while [ $# -ne 0 ]; do
         --enable-jemalloc)
             append_cache_entry ENABLE_JEMALLOC     BOOL   true
             ;;
+        --enable-c++11)
+            append_cache_entry ENABLE_CXX11         BOOL   true
+            ;;
+        --enable-broker)
+            append_cache_entry ENABLE_CXX11         BOOL   true
+            append_cache_entry ENABLE_BROKER        BOOL   true
+            ;;
         --disable-broccoli)
             append_cache_entry INSTALL_BROCCOLI     BOOL   false
             ;;
@@ -248,6 +262,12 @@ while [ $# -ne 0 ]; do
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE      PATH    $optarg
             ;;
+        --with-libcaf=*)
+            append_cache_entry LIBCAF_ROOT_DIR      PATH   $optarg
+            ;;
+        --with-rocksdb=*)
+            append_cache_entry ROCKSDB_ROOT_DIR     PATH   $optarg
+            ;;
         --binary-package)
             append_cache_entry BINARY_PACKAGING_MODE BOOL true
             ;;

doc/components/broker/README.rst

@@ -0,0 +1 @@
+../../../aux/broker/README

doc/components/broker/broker-manual.rst

@@ -0,0 +1 @@
+../../../aux/broker/broker-manual.rst

doc/components/index.rst

@@ -17,6 +17,8 @@ current, independent component releases.
    Broccoli - User Manual <broccoli/broccoli-manual>
    Broccoli Python Bindings <broccoli-python/README>
    Broccoli Ruby Bindings <broccoli-ruby/README>
+   Broker - Bro's (New) Messaging Library (README) <broker/README>
+   Broker - User Manual <broker/broker-manual.rst>
    BroControl - Interactive Bro management shell <broctl/README>
    Bro-Aux - Small auxiliary tools for Bro <bro-aux/README>
    BTest - A unit testing framework <btest/README>

doc/frameworks/comm.rst

@@ -0,0 +1,202 @@
+
+.. _comm-framework:
+
+======================================
+Broker-Enabled Communication Framework
+======================================
+
+.. rst-class:: opening
+
+    Bro can now use the `Broker Library
+    <../components/broker/README.html>`_ to exchange information with
+    other Bro processes.  To enable it run Bro's ``configure`` script
+    with the ``--enable-broker`` option.  Note that a C++11 compatible
+    compiler is required as well as the `C++ Actor Framework
+    <http://actor-framework.org/>`_.
+
+.. contents::
+
+Connecting to Peers
+===================
+
+Communication via Broker must first be turned on via
+:bro:see:`Comm::enable`.
+
+Bro can accept incoming connections by calling :bro:see:`Comm::listen`
+and then monitor connection status updates via
+:bro:see:`Comm::incoming_connection_established` and
+:bro:see:`Comm::incoming_connection_broken`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/connecting-listener.bro
+
+Bro can initiate outgoing connections by calling :bro:see:`Comm::connect`
+and then monitor connection status updates via
+:bro:see:`Comm::outgoing_connection_established`,
+:bro:see:`Comm::outgoing_connection_broken`, and
+:bro:see:`Comm::outgoing_connection_incompatible`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/connecting-connector.bro
+
+Remote Printing
+===============
+
+To receive remote print messages, first use
+:bro:see:`Comm::subscribe_to_prints` to advertise to peers a topic
+prefix of interest and then create an event handler for
+:bro:see:`Comm::print_handler` to handle any print messages that are
+received.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/printing-listener.bro
+
+To send remote print messages, just call :bro:see:`Comm::print`.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/printing-connector.bro
+
+Notice that the subscriber only used the prefix "bro/print/", but is
+able to receive messages with full topics of "bro/print/hi",
+"bro/print/stuff", and "bro/print/bye".  The model here is that the
+publisher of a message checks for all subscribers who advertised
+interest in a prefix of that message's topic and sends it to them.
+
+Message Format
+--------------
+
+For other applications that want to exchange print messages with Bro,
+the Broker message format is simply:
+
+.. code:: c++
+
+    broker::message{std::string{}};
+
+Remote Events
+=============
+
+Receiving remote events is similar to remote prints.  Just use
+:bro:see:`Comm::subscribe_to_events` and possibly define any new events
+along with handlers that peers may want to send.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/events-listener.bro
+
+To send events, there are two choices.  The first is to use call
+:bro:see:`Comm::event` directly.  The second option is to use
+:bro:see:`Comm::auto_event` to make it so a particular event is
+automatically sent to peers whenever it is called locally via the normal
+event invocation syntax.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/events-connector.bro
+
+Again, the subscription model is prefix-based.
+
+Message Format
+--------------
+
+For other applications that want to exchange event messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{std::string{}, ...};
+
+The first parameter is the name of the event and the remaining ``...``
+are its arguments, which are any of the support Broker data types as
+they correspond to the Bro types for the event named in the first
+parameter of the message.
+
+Remote Logging
+==============
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/testlog.bro
+
+Use :bro:see:`Comm::subscribe_to_logs` to advertise interest in logs
+written by peers.  The topic names that Bro uses are implicitly of the
+form "bro/log/<stream-name>".
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/logs-listener.bro
+
+To send remote logs either use :bro:see:`Log::enable_remote_logging` or
+:bro:see:`Comm::enable_remote_logs`.  The former allows any log stream
+to be sent to peers while the later toggles remote logging for
+particular streams.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/logs-connector.bro
+
+Message Format
+--------------
+
+For other applications that want to exchange logs messages with Bro,
+the Broker message format is:
+
+.. code:: c++
+
+    broker::message{broker::enum_value{}, broker::record{}};
+
+The enum value corresponds to the stream's :bro:see:`Log::ID` value, and
+the record corresponds to a single entry of that log's columns record,
+in this case a ``Test::INFO`` value.
+
+Tuning Access Control
+=====================
+
+By default, endpoints do not restrict the message topics that it sends
+to peers and do not restrict what message topics and data store
+identifiers get advertised to peers.  These are the default
+:bro:see:`Comm::EndpointFlags` supplied to :bro:see:`Comm::enable`.
+
+If not using the ``auto_publish`` flag, one can use the
+:bro:see:`Comm::publish_topic` and :bro:see:`Comm::unpublish_topic`
+functions to manipulate the set of message topics (must match exactly)
+that are allowed to be sent to peer endpoints.  These settings take
+precedence over the per-message ``peers`` flag supplied to functions
+that take a :bro:see:`Comm::SendFlags` such as :bro:see:`Comm::print`,
+:bro:see:`Comm::event`, :bro:see:`Comm::auto_event` or
+:bro:see:`Comm::enable_remote_logs`.
+
+If not using the ``auto_advertise`` flag, one can use the
+:bro:see:`Comm::advertise_topic` and :bro:see:`Comm::unadvertise_topic`
+to manupulate the set of topic prefixes that are allowed to be
+advertised to peers.  If an endpoint does not advertise a topic prefix,
+the only way a peers can send messages to it is via the ``unsolicited``
+flag of :bro:see:`Comm::SendFlags`  and choosing a topic with a matching
+prefix (i.e. full topic may be longer than receivers prefix, just the
+prefix needs to match).
+
+Distributed Data Stores
+=======================
+
+There are three flavors of key-value data store interfaces: master,
+clone, and frontend.
+
+A frontend is the common interface to query and modify data stores.
+That is, a clone is a specific type of frontend and a master is also a
+specific type of frontend, but a standalone frontend can also exist to
+e.g. query and modify the contents of a remote master store without
+actually "owning" any of the contents itself.
+
+A master data store can be be cloned from remote peers which may then
+perform lightweight, local queries against the clone, which
+automatically stays synchronized with the master store.  Clones cannot
+modify their content directly, instead they send modifications to the
+centralized master store which applies them and then broadcasts them to
+all clones.
+
+Master and clone stores get to choose what type of storage backend to
+use.  E.g. In-memory versus SQLite for persistence.  Note that if clones
+are used, data store sizes should still be able to fit within memory
+regardless of the storage backend as a single snapshot of the master
+store is sent in a single chunk to initialize the clone.
+
+Data stores also support expiration on a per-key basis either using an
+absolute point in time or a relative amount of time since the entry's
+last modification time.
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/stores-listener.bro
+
+.. btest-include:: ${DOC_ROOT}/frameworks/comm/stores-connector.bro
+
+In the above example, if a local copy of the store contents isn't
+needed, just replace the :bro:see:`Store::create_clone` call with
+:bro:see:`Store::create_frontend`.  Queries will then be made against
+the remote master store instead of the local clone.
+
+Note that all queries are made within Bro's asynchrounous ``when``
+statements and must specify a timeout block.

doc/frameworks/comm/connecting-connector.bro

@@ -0,0 +1,19 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+		  peer_address, peer_port, peer_name;
+	terminate();
+	}

doc/frameworks/comm/connecting-listener.bro

@@ -0,0 +1,21 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event Comm::incoming_connection_broken(peer_name: string)
+	{
+	print "Comm::incoming_connection_broken", peer_name;
+	terminate();
+	}

doc/frameworks/comm/events-connector.bro

@@ -0,0 +1,31 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	Comm::auto_event("bro/event/my_auto_event", my_auto_event);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	Comm::event("bro/event/my_event", Comm::event_args(my_event, "hi", 0));
+	event my_auto_event("stuff", 88);
+	Comm::event("bro/event/my_event", Comm::event_args(my_event, "...", 1));
+	event my_auto_event("more stuff", 51);
+	Comm::event("bro/event/my_event", Comm::event_args(my_event, "bye", 2));
+	}
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/comm/events-listener.bro

@@ -0,0 +1,37 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+global msg_count = 0;
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_events("bro/event/");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event my_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}
+
+event my_auto_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_auto_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}

doc/frameworks/comm/logs-connector.bro

@@ -0,0 +1,40 @@
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+redef Log::enable_local_logging = F;
+redef Log::enable_remote_logging = F;
+global n = 0;
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::enable_remote_logs(Test::LOG);
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event do_write()
+	{
+	if ( n == 6 )
+		return;
+
+	Log::write(Test::LOG, [$msg = "ping", $num = n]);
+	++n;
+	event do_write();
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	event do_write();
+	}
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/comm/logs-listener.bro

@@ -0,0 +1,25 @@
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_logs("bro/log/Test::LOG");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event Test::log_test(rec: Test::Info)
+	{
+	print "wrote log", rec;
+
+	if ( rec$num == 5 )
+		terminate();
+	}

doc/frameworks/comm/printing-connector.bro

@@ -0,0 +1,26 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	Comm::print("bro/print/hi", "hello");
+	Comm::print("bro/print/stuff", "...");
+	Comm::print("bro/print/bye", "goodbye");
+	}
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

doc/frameworks/comm/printing-listener.bro

@@ -0,0 +1,26 @@
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+global msg_count = 0;
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_prints("bro/print/");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event Comm::print_handler(msg: string)
+	{
+	++msg_count;
+	print "got print message", msg;
+
+	if ( msg_count == 3 )
+		terminate();
+	}

doc/frameworks/comm/stores-connector.bro

@@ -0,0 +1,53 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of Store::Handle;
+
+function dv(d: Comm::Data): Comm::DataVector
+	{
+	local rval: Comm::DataVector;
+	rval[0] = d;
+	return rval;
+	}
+
+global ready: event();
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	local myset: set[string] = {"a", "b", "c"};
+	local myvec: vector of string = {"alpha", "beta", "gamma"};
+	h = Store::create_master("mystore");
+	Store::insert(h, Comm::data("one"), Comm::data(110));
+	Store::insert(h, Comm::data("two"), Comm::data(223));
+	Store::insert(h, Comm::data("myset"), Comm::data(myset));
+	Store::insert(h, Comm::data("myvec"), Comm::data(myvec));
+	Store::increment(h, Comm::data("one"));
+	Store::decrement(h, Comm::data("two"));
+	Store::add_to_set(h, Comm::data("myset"), Comm::data("d"));
+	Store::remove_from_set(h, Comm::data("myset"), Comm::data("b"));
+	Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta")));
+	Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega")));
+
+	when ( local res = Store::size(h) )
+		{
+		print "master size", res;
+		event ready();
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1secs);
+	Comm::auto_event("bro/event/ready", ready);
+	}

doc/frameworks/comm/stores-listener.bro

@@ -0,0 +1,43 @@
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of Store::Handle;
+global expected_key_count = 4;
+global key_count = 0;
+
+function do_lookup(key: string)
+	{
+	when ( local res = Store::lookup(h, Comm::data(key)) )
+		{
+		++key_count;
+		print "lookup", key, res;
+
+		if ( key_count == expected_key_count )
+			terminate();
+		}
+	timeout 10sec
+		{ print "timeout", key; }
+	}
+
+event ready()
+	{
+	h = Store::create_clone("mystore");
+
+	when ( local res = Store::keys(h) )
+		{
+		print "clone keys", res;
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3)));
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_events("bro/event/ready");
+	Comm::listen(broker_port, "127.0.0.1");
+	}

doc/frameworks/comm/testlog.bro

@@ -0,0 +1,19 @@
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		msg: string &log;
+		num: count &log;
+	};
+
+	global log_test: event(rec: Test::Info);
+}
+
+event bro_init() &priority=5
+	{
+	Comm::enable();
+	Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]);
+	}

doc/frameworks/index.rst

@@ -14,4 +14,4 @@ Frameworks
    notice
    signatures
    sumstats
-
+   comm

scripts/base/frameworks/comm/__load__.bro

@@ -0,0 +1 @@
+@load ./main

scripts/base/frameworks/comm/main.bro

@@ -0,0 +1,103 @@
+##! Various data structure definitions for use with Bro's communication system.
+
+module Comm;
+
+export {
+
+	## A name used to identify this endpoint to peers.
+	## .. bro:see:: Comm::connect Comm::listen
+	const endpoint_name = "" &redef;
+
+	## Change communication behavior.
+	type EndpointFlags: record {
+		## Whether to restrict message topics that can be published to peers.
+		auto_publish: bool &default = T;
+		## Whether to restrict what message topics or data store identifiers
+		## the local endpoint advertises to peers (e.g. subscribing to
+		## events or making a master data store available).
+		auto_advertise: bool &default = T;
+	};
+
+	## Fine-grained tuning of communication behavior for a particular message.
+	type SendFlags: record {
+		## Send the message to the local endpoint.
+		self: bool &default = F;
+		## Send the message to peer endpoints that advertise interest in
+		## the topic associated with the message.
+		peers: bool &default = T;
+		## Send the message to peer endpoints even if they don't advertise
+		## interest in the topic associated with the message.
+		unsolicited: bool &default = F;
+	};
+
+	## Opaque communication data.
+	type Data: record {
+		d: opaque of Comm::Data &optional;
+	};
+
+	## Opaque communication data.
+	type DataVector: vector of Comm::Data;
+
+	## Opaque event communication data.
+	type EventArgs: record {
+		## The name of the event.  Not set if invalid event or arguments.
+		name: string &optional;
+		## The arguments to the event.
+		args: DataVector;
+	};
+
+	## Opaque communication data used as a convenient way to wrap key-value
+	## pairs that comprise table entries.
+	type Comm::TableItem : record {
+		key: Comm::Data;
+		val: Comm::Data;
+	};
+}
+
+module Store;
+
+export {
+
+	## Whether a data store query could be completed or not.
+	type QueryStatus: enum {
+		SUCCESS,
+		FAILURE,
+	};
+
+	## An expiry time for a key-value pair inserted in to a data store.
+	type ExpiryTime: record {
+		## Absolute point in time at which to expire the entry.
+		absolute: time &optional;
+		## A point in time relative to the last modification time at which
+		## to expire the entry.  New modifications will delay the expiration.
+		since_last_modification: interval &optional;
+	};
+
+	## The result of a data store query.
+	type QueryResult: record {
+		## Whether the query completed or not.
+		status: Store::QueryStatus;
+		## The result of the query.  Certain queries may use a particular
+		## data type (e.g. querying store size always returns a count, but
+		## a lookup may return various data types).
+		result: Comm::Data;
+	};
+
+	## Options to tune the SQLite storage backend.
+	type SQLiteOptions: record {
+		## File system path of the database.
+		path: string &default = "store.sqlite";
+	};
+
+	## Options to tune the RocksDB storage backend.
+	type RocksDBOptions: record {
+		## File system path of the database.
+		path: string &default = "store.rocksdb";
+	};
+
+	## Options to tune the particular storage backends.
+	type BackendOptions: record {
+		sqlite: SQLiteOptions &default = SQLiteOptions();
+		rocksdb: RocksDBOptions &default = RocksDBOptions();
+	};
+}

scripts/base/init-bare.bro

@@ -3358,6 +3358,7 @@ const bits_per_uid: count = 96 &redef;
 
 # Load these frameworks here because they use fairly deep integration with
 # BiFs and script-land defined types.
+@load base/frameworks/comm
 @load base/frameworks/logging
 @load base/frameworks/input
 @load base/frameworks/analyzer

src/CMakeLists.txt

@@ -161,6 +161,14 @@ add_subdirectory(iosource)
 add_subdirectory(logging)
 add_subdirectory(probabilistic)
 
+if ( ENABLE_BROKER )
+    add_subdirectory(comm)
+else ()
+    # Just to satisfy coverage unit tests until new Broker-based
+    # communication is enabled by default.
+    add_subdirectory(comm-dummy)
+endif ()
+
 set(bro_SUBDIRS
     # Order is important here.
     ${bro_PLUGIN_LIBS}

src/DebugLogger.cc

@@ -19,7 +19,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = {
 	{ "logging", 0, false }, {"input", 0, false },
 	{ "threading", 0, false }, { "file_analysis", 0, false },
 	{ "plugins", 0, false }, { "broxygen", 0, false },
-	{ "pktio", 0, false}
+	{ "pktio", 0, false }, { "broker", 0, false }
 };
 
 DebugLogger::DebugLogger(const char* filename)

src/DebugLogger.h

@@ -32,6 +32,7 @@ enum DebugStream {
 	DBG_PLUGINS,	// Plugin system
 	DBG_BROXYGEN,	// Broxygen
 	DBG_PKTIO,	// Packet sources and dumpers.
+	DBG_BROKER,	// Broker communication
 
 	NUM_DBGS // Has to be last
 };

src/EventHandler.cc

@@ -5,6 +5,11 @@
 #include "RemoteSerializer.h"
 #include "NetVar.h"
 
+#ifdef ENABLE_BROKER
+#include "comm/Manager.h"
+#include "comm/Data.h"
+#endif
+
 EventHandler::EventHandler(const char* arg_name)
 	{
 	name = copy_string(arg_name);
@@ -26,7 +31,12 @@ EventHandler::operator bool() const
 	{
 	return enabled && ((local && local->HasBodies())
 			   || receivers.length()
-			   || generate_always);
+			   || generate_always
+#ifdef ENABLE_BROKER
+			   || ! auto_remote_send.empty()
+			   // TODO: and require a subscriber interested in a topic or unsolicited flags?
+#endif
+	                   );
 	}
 
 FuncType* EventHandler::FType()
@@ -73,6 +83,46 @@ void EventHandler::Call(val_list* vl, bool no_remote)
 			SerialInfo info(remote_serializer);
 			remote_serializer->SendCall(&info, receivers[i], name, vl);
 			}
+
+#ifdef ENABLE_BROKER
+
+		if ( ! auto_remote_send.empty() )
+			{
+			// TODO: also short-circuit based on interested subscribers/flags?
+			broker::message msg;
+			msg.reserve(vl->length() + 1);
+			msg.emplace_back(Name());
+			bool valid_args = true;
+
+			for ( auto i = 0; i < vl->length(); ++i )
+				{
+				auto opt_data = comm::val_to_data((*vl)[i]);
+
+				if ( opt_data )
+					msg.emplace_back(move(*opt_data));
+				else
+					{
+					valid_args = false;
+					auto_remote_send.clear();
+					reporter->Error("failed auto-remote event '%s', disabled",
+					                Name());
+					break;
+					}
+				}
+
+			if ( valid_args )
+				{
+				for ( auto it = auto_remote_send.begin();
+				      it != auto_remote_send.end(); ++it )
+					{
+					if ( std::next(it) == auto_remote_send.end() )
+						comm_mgr->Event(it->first, move(msg), it->second);
+					else
+						comm_mgr->Event(it->first, msg, it->second);
+					}
+				}
+			}
+#endif
 		}
 
 	if ( local )

src/EventHandler.h

@@ -4,7 +4,8 @@
 #define EVENTHANDLER
 
 #include <assert.h>
-
+#include <map>
+#include <string>
 #include "List.h"
 #include "BroList.h"
 
@@ -28,6 +29,18 @@ public:
 	void AddRemoteHandler(SourceID peer);
 	void RemoveRemoteHandler(SourceID peer);
 
+#ifdef ENABLE_BROKER
+	void AutoRemote(std::string topic, int flags)
+		{
+		auto_remote_send[std::move(topic)] = flags;
+		}
+
+	void AutoRemoteStop(const std::string& topic)
+		{
+		auto_remote_send.erase(topic);
+		}
+#endif
+
 	void Call(val_list* vl, bool no_remote = false);
 
 	// Returns true if there is at least one local or remote handler.
@@ -67,6 +80,10 @@ private:
 	declare(List, SourceID);
 	typedef List(SourceID) receiver_list;
 	receiver_list receivers;
+
+#ifdef ENABLE_BROKER
+	std::map<std::string, int> auto_remote_send;	// topic -> flags
+#endif
 };
 
 // Encapsulates a ptr to an event handler to overload the boolean operator.

src/Net.cc

@@ -34,6 +34,10 @@
 #include "iosource/PktDumper.h"
 #include "plugin/Manager.h"
 
+#ifdef ENABLE_BROKER
+#include "comm/Manager.h"
+#endif
+
 extern "C" {
 #include "setsignal.h"
 };
@@ -315,6 +319,11 @@ void net_run()
 			}
 #endif
 		current_iosrc = src;
+		bool communication_enabled = using_communication;
+
+#ifdef ENABLE_BROKER
+		communication_enabled |= comm_mgr->Enabled();
+#endif
 
 		if ( src )
 			src->Process();	// which will call net_packet_dispatch()
@@ -332,7 +341,7 @@ void net_run()
 				}
 			}
 
-		else if ( (have_pending_timers || using_communication) &&
+		else if ( (have_pending_timers || communication_enabled) &&
 			  ! pseudo_realtime )
 			{
 			// Take advantage of the lull to get up to
@@ -347,7 +356,7 @@ void net_run()
 			// us a lot of idle time, but doesn't delay near-term
 			// timers too much.  (Delaying them somewhat is okay,
 			// since Bro timers are not high-precision anyway.)
-			if ( ! using_communication )
+			if ( ! communication_enabled )
 				usleep(100000);
 			else
 				usleep(1000);

src/Reporter.cc

@@ -123,6 +123,19 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...)
 	throw InterpreterException();
 	}
 
+void Reporter::RuntimeError(const Location* location, const char* fmt, ...)
+	{
+	++errors;
+	PushLocation(location);
+	va_list ap;
+	va_start(ap, fmt);
+	FILE* out = errors_to_stderr ? stderr : 0;
+	DoLog("runtime error", reporter_error, out, 0, 0, true, true, "", fmt, ap);
+	va_end(ap);
+	PopLocation();
+	throw InterpreterException();
+	}
+
 void Reporter::InternalError(const char* fmt, ...)
 	{
 	va_list ap;

src/Reporter.h

@@ -73,6 +73,10 @@ public:
 	// function will not return but raise an InterpreterException.
 	void ExprRuntimeError(const Expr* expr, const char* fmt, ...);
 
+	// Report a runtime error in evaluating a Bro script expression. This
+	// function will not return but raise an InterpreterException.
+	void RuntimeError(const Location* location, const char* fmt, ...);
+
 	// Report a traffic weirdness, i.e., an unexpected protocol situation
 	// that may lead to incorrectly processing a connnection.
 	void Weird(const char* name);	// Raises net_weird().

src/SerialTypes.h

@@ -113,6 +113,8 @@ SERIAL_VAL(TOPK_VAL, 20)
 SERIAL_VAL(BLOOMFILTER_VAL, 21)
 SERIAL_VAL(CARDINALITY_VAL, 22)
 SERIAL_VAL(X509_VAL, 23)
+SERIAL_VAL(COMM_STORE_HANDLE_VAL, 24)
+SERIAL_VAL(COMM_DATA_VAL, 25)
 
 #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR)
 SERIAL_EXPR(EXPR, 1)

src/Stats.cc

@@ -10,6 +10,10 @@
 #include "Trigger.h"
 #include "threading/Manager.h"
 
+#ifdef ENABLE_BROKER
+#include "comm/Manager.h"
+#endif
+
 int killed_by_inactivity = 0;
 
 uint64 tot_ack_events = 0;
@@ -222,6 +226,26 @@ void ProfileLogger::Log()
 			    ));
 		}
 
+#ifdef ENABLE_BROKER
+	auto cs = comm_mgr->ConsumeStatistics();
+
+	file->Write(fmt("%0.6f Comm: peers=%zu stores=%zu "
+	                "store_queries=%zu store_responses=%zu "
+	                "outgoing_conn_status=%zu incoming_conn_status=%zu "
+	                "reports=%zu\n",
+	                network_time, cs.outgoing_peer_count, cs.data_store_count,
+	                cs.pending_query_count, cs.response_count,
+	                cs.outgoing_conn_status_count, cs.incoming_conn_status_count,
+	                cs.report_count));
+
+	for ( const auto& s : cs.print_count )
+		file->Write(fmt("    %-25s prints dequeued=%zu\n", s.first.data(), s.second));
+	for ( const auto& s : cs.event_count )
+		file->Write(fmt("    %-25s events dequeued=%zu\n", s.first.data(), s.second));
+	for ( const auto& s : cs.log_count )
+		file->Write(fmt("    %-25s logs dequeued=%zu\n", s.first.data(), s.second));
+#endif
+
 	// Script-level state.
 	unsigned int size, mem = 0;
 	PDict(ID)* globals = global_scope()->Vars();

src/Trigger.cc

@@ -112,6 +112,7 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 	attached = 0;
 	is_return = arg_is_return;
 	location = arg_location;
+	timeout_value = -1;
 
 	++total_triggers;
 
@@ -133,17 +134,22 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts,
 
 	Val* timeout_val = arg_timeout ? arg_timeout->Eval(arg_frame) : 0;
 
+	if ( timeout_val )
+		{
+		Unref(timeout_val);
+		timeout_value = timeout_val->AsInterval();
+		}
+
 	// Make sure we don't get deleted if somebody calls a method like
 	// Timeout() while evaluating the trigger. 
 	Ref(this);
 
-	if ( ! Eval() && timeout_val )
+	if ( ! Eval() && timeout_value >= 0 )
 		{
-		timer = new TriggerTimer(timeout_val->AsInterval(), this);
+		timer = new TriggerTimer(timeout_value, this);
 		timer_mgr->Add(timer);
 		}
 
-	Unref(timeout_val);
 	Unref(this);
 	}
 

src/Trigger.h

@@ -32,6 +32,10 @@ public:
 	// Executes timeout code and deletes the object.
 	void Timeout();
 
+	// Return the timeout interval (negative if none was specified).
+	double TimeoutValue() const
+		{ return timeout_value; }
+
 	// Called if another entity needs to complete its operations first
 	// in any case before this trigger can proceed.
 	void Hold()	{ delayed = true; }
@@ -51,6 +55,8 @@ public:
 	// may not immediately delete it as other references may still exist.
 	void Disable();
 
+	bool Disabled() const { return disabled; }
+
 	virtual void Describe(ODesc* d) const { d->Add("<trigger>"); }
 
 	// Overidden from Notifier.  We queue the trigger and evaluate it
@@ -87,6 +93,7 @@ private:
 	Stmt* body;
 	Stmt* timeout_stmts;
 	Expr* timeout;
+	double timeout_value;
 	Frame* frame;
 	bool is_return;
 	const Location* location;

src/comm-dummy/CMakeLists.txt

@@ -0,0 +1,13 @@
+# Placeholder for Broker-based communication functionality, not enabled
+# by default.  This helps satisfy coverage unit tests pass regardless of
+# whether Broker is enabled or not.
+
+include(BroSubdir)
+
+bif_target(comm.bif)
+bif_target(data.bif)
+bif_target(messaging.bif)
+bif_target(store.bif)
+
+bro_add_subdir_library(comm_dummy ${BIF_OUTPUT_CC})
+add_dependencies(bro_comm_dummy generate_outputs)

src/comm-dummy/comm.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default.

src/comm-dummy/data.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default

src/comm-dummy/messaging.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default

src/comm-dummy/store.bif

@@ -0,0 +1,3 @@
+
+##! Placeholder for Broker-based communication functionality, not enabled
+##! by default

src/comm/CMakeLists.txt

@@ -0,0 +1,28 @@
+include(BroSubdir)
+
+include_directories(BEFORE
+                    ${CMAKE_CURRENT_SOURCE_DIR}
+                    ${CMAKE_CURRENT_BINARY_DIR}
+)
+
+if ( ROCKSDB_INCLUDE_DIR )
+    add_definitions(-DHAVE_ROCKSDB)
+    include_directories(BEFORE ${ROCKSDB_INCLUDE_DIR})
+endif ()
+
+include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_CORE})
+include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_IO})
+
+set(comm_SRCS
+    Data.cc
+    Manager.cc
+    Store.cc
+)
+
+bif_target(comm.bif)
+bif_target(data.bif)
+bif_target(messaging.bif)
+bif_target(store.bif)
+
+bro_add_subdir_library(comm ${comm_SRCS} ${BIF_OUTPUT_CC})
+add_dependencies(bro_comm generate_outputs)

src/comm/Data.cc

@@ -0,0 +1,702 @@
+#include "Data.h"
+#include "comm/data.bif.h"
+#include <caf/binary_serializer.hpp>
+#include <caf/binary_deserializer.hpp>
+
+using namespace std;
+
+OpaqueType* comm::opaque_of_data_type;
+OpaqueType* comm::opaque_of_set_iterator;
+OpaqueType* comm::opaque_of_table_iterator;
+OpaqueType* comm::opaque_of_vector_iterator;
+OpaqueType* comm::opaque_of_record_iterator;
+
+static broker::port::protocol to_broker_port_proto(TransportProto tp)
+	{
+	switch ( tp ) {
+	case TRANSPORT_TCP:
+		return broker::port::protocol::tcp;
+	case TRANSPORT_UDP:
+		return broker::port::protocol::udp;
+	case TRANSPORT_ICMP:
+		return broker::port::protocol::icmp;
+	case TRANSPORT_UNKNOWN:
+	default:
+		return broker::port::protocol::unknown;
+	}
+	}
+
+TransportProto comm::to_bro_port_proto(broker::port::protocol tp)
+	{
+	switch ( tp ) {
+	case broker::port::protocol::tcp:
+		return TRANSPORT_TCP;
+	case broker::port::protocol::udp:
+		return TRANSPORT_UDP;
+	case broker::port::protocol::icmp:
+		return TRANSPORT_ICMP;
+	case broker::port::protocol::unknown:
+	default:
+		return TRANSPORT_UNKNOWN;
+	}
+	}
+
+struct val_converter {
+	using result_type = Val*;
+
+	BroType* type;
+
+	result_type operator()(bool a)
+		{
+		if ( type->Tag() == TYPE_BOOL )
+			return new Val(a, TYPE_BOOL);
+		return nullptr;
+		}
+
+	result_type operator()(uint64_t a)
+		{
+		if ( type->Tag() == TYPE_COUNT )
+			return new Val(a, TYPE_COUNT);
+		if ( type->Tag() == TYPE_COUNTER )
+			return new Val(a, TYPE_COUNTER);
+		return nullptr;
+		}
+
+	result_type operator()(int64_t a)
+		{
+		if ( type->Tag() == TYPE_INT )
+			return new Val(a, TYPE_INT);
+		return nullptr;
+		}
+
+	result_type operator()(double a)
+		{
+		if ( type->Tag() == TYPE_DOUBLE )
+			return new Val(a, TYPE_DOUBLE);
+		return nullptr;
+		}
+
+	result_type operator()(std::string& a)
+		{
+		switch ( type->Tag() ) {
+		case TYPE_STRING:
+			return new StringVal(a.size(), a.data());
+		case TYPE_FILE:
+			{
+			auto file = BroFile::GetFile(a.data());
+
+			if ( file )
+				{
+				Ref(file);
+				return new Val(file);
+				}
+
+			return nullptr;
+			}
+		case TYPE_FUNC:
+			{
+			auto id = lookup_ID(a.data(), GLOBAL_MODULE_NAME);
+			auto rval = id ? id->ID_Val() : nullptr;
+			Unref(id);
+
+			if ( rval && rval->Type()->Tag() == TYPE_FUNC )
+				return rval;
+
+			return nullptr;
+			}
+		default:
+			return nullptr;
+		}
+		}
+
+	result_type operator()(broker::address& a)
+		{
+		if ( type->Tag() == TYPE_ADDR )
+			{
+			auto bits = reinterpret_cast<const in6_addr*>(&a.bytes());
+			return new AddrVal(IPAddr(*bits));
+			}
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::subnet& a)
+		{
+		if ( type->Tag() == TYPE_SUBNET )
+			{
+			auto bits = reinterpret_cast<const in6_addr*>(&a.network().bytes());
+			return new SubNetVal(IPPrefix(IPAddr(*bits), a.length()));
+			}
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::port& a)
+		{
+		if ( type->Tag() == TYPE_PORT )
+			return new PortVal(a.number(), comm::to_bro_port_proto(a.type()));
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::time_point& a)
+		{
+		if ( type->Tag() == TYPE_TIME )
+			return new Val(a.value, TYPE_TIME);
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::time_duration& a)
+		{
+		if ( type->Tag() == TYPE_INTERVAL )
+			return new Val(a.value, TYPE_INTERVAL);
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::enum_value& a)
+		{
+		if ( type->Tag() == TYPE_ENUM )
+			{
+			auto etype = type->AsEnumType();
+			auto i = etype->Lookup(GLOBAL_MODULE_NAME, a.name.data());
+
+			if ( i == -1 )
+				return nullptr;
+
+			return new EnumVal(i, etype);
+			}
+
+		return nullptr;
+		}
+
+	result_type operator()(broker::set& a)
+		{
+		if ( ! type->IsSet() )
+			return nullptr;
+
+		auto tt = type->AsTableType();
+		auto rval = new TableVal(tt);
+
+		for ( auto& item : a )
+			{
+			broker::vector composite_key;
+			auto indices = broker::get<broker::vector>(item);
+
+			if ( ! indices )
+				{
+				composite_key.emplace_back(move(item));
+				indices = &composite_key;
+				}
+
+			auto expected_index_types = tt->Indices()->Types();
+
+			if ( static_cast<size_t>(expected_index_types->length()) !=
+			     indices->size() )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			auto list_val = new ListVal(TYPE_ANY);
+
+			for ( auto i = 0u; i < indices->size(); ++i )
+				{
+				auto index_val = comm::data_to_val(move((*indices)[i]),
+				                                   (*expected_index_types)[i]);
+
+				if ( ! index_val )
+					{
+					Unref(rval);
+					Unref(list_val);
+					return nullptr;
+					}
+
+				list_val->Append(index_val);
+				}
+
+
+			rval->Assign(list_val, nullptr);
+			Unref(list_val);
+			}
+
+		return rval;
+		}
+
+	result_type operator()(broker::table& a)
+		{
+		if ( ! type->IsTable() )
+			return nullptr;
+
+		auto tt = type->AsTableType();
+		auto rval = new TableVal(tt);
+
+		for ( auto& item : a )
+			{
+			broker::vector composite_key;
+			auto indices = broker::get<broker::vector>(item.first);
+
+			if ( ! indices )
+				{
+				composite_key.emplace_back(move(item.first));
+				indices = &composite_key;
+				}
+
+			auto expected_index_types = tt->Indices()->Types();
+
+			if ( static_cast<size_t>(expected_index_types->length()) !=
+			     indices->size() )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			auto list_val = new ListVal(TYPE_ANY);
+
+			for ( auto i = 0u; i < indices->size(); ++i )
+				{
+				auto index_val = comm::data_to_val(move((*indices)[i]),
+				                                   (*expected_index_types)[i]);
+
+				if ( ! index_val )
+					{
+					Unref(rval);
+					Unref(list_val);
+					return nullptr;
+					}
+
+				list_val->Append(index_val);
+				}
+
+			auto value_val = comm::data_to_val(move(item.second),
+			                                   tt->YieldType());
+
+			if ( ! value_val )
+				{
+				Unref(rval);
+				Unref(list_val);
+				return nullptr;
+				}
+
+			rval->Assign(list_val, value_val);
+			Unref(list_val);
+			}
+
+		return rval;
+		}
+
+	result_type operator()(broker::vector& a)
+		{
+		if ( type->Tag() != TYPE_VECTOR )
+			return nullptr;
+
+		auto vt = type->AsVectorType();
+		auto rval = new VectorVal(vt);
+
+		for ( auto& item : a )
+			{
+			auto item_val = comm::data_to_val(move(item), vt->YieldType());
+
+			if ( ! item_val )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			rval->Assign(rval->Size(), item_val);
+			}
+
+		return rval;
+		}
+
+	result_type operator()(broker::record& a)
+		{
+		if ( type->Tag() != TYPE_RECORD )
+			return nullptr;
+
+		auto rt = type->AsRecordType();
+
+		if ( a.fields.size() != static_cast<size_t>(rt->NumFields()) )
+			return nullptr;
+
+		auto rval = new RecordVal(rt);
+
+		for ( auto i = 0u; i < a.fields.size(); ++i )
+			{
+			if ( ! a.fields[i] )
+				{
+				rval->Assign(i, nullptr);
+				continue;
+				}
+
+			auto item_val = comm::data_to_val(move(*a.fields[i]),
+			                                  rt->FieldType(i));
+
+			if ( ! item_val )
+				{
+				Unref(rval);
+				return nullptr;
+				}
+
+			rval->Assign(i, item_val);
+			}
+
+		return rval;
+		}
+};
+
+Val* comm::data_to_val(broker::data d, BroType* type)
+	{
+	return broker::visit(val_converter{type}, d);
+	}
+
+broker::util::optional<broker::data> comm::val_to_data(Val* v)
+	{
+	switch ( v->Type()->Tag() ) {
+	case TYPE_BOOL:
+		return {v->AsBool()};
+	case TYPE_INT:
+		return {v->AsInt()};
+	case TYPE_COUNT:
+		return {v->AsCount()};
+	case TYPE_COUNTER:
+		return {v->AsCounter()};
+	case TYPE_PORT:
+		{
+		auto p = v->AsPortVal();
+		return {broker::port(p->Port(), to_broker_port_proto(p->PortType()))};
+		}
+	case TYPE_ADDR:
+		{
+		auto a = v->AsAddr();
+		in6_addr tmp;
+		a.CopyIPv6(&tmp);
+		return {broker::address(reinterpret_cast<const uint32_t*>(&tmp),
+			                    broker::address::family::ipv6,
+			                    broker::address::byte_order::network)};
+		}
+		break;
+	case TYPE_SUBNET:
+		{
+		auto s = v->AsSubNet();
+		in6_addr tmp;
+		s.Prefix().CopyIPv6(&tmp);
+		auto a = broker::address(reinterpret_cast<const uint32_t*>(&tmp),
+		                         broker::address::family::ipv6,
+		                         broker::address::byte_order::network);
+		return {broker::subnet(a, s.Length())};
+		}
+		break;
+	case TYPE_DOUBLE:
+		return {v->AsDouble()};
+	case TYPE_TIME:
+		return {broker::time_point(v->AsTime())};
+	case TYPE_INTERVAL:
+		return {broker::time_duration(v->AsInterval())};
+	case TYPE_ENUM:
+		{
+		auto enum_type = v->Type()->AsEnumType();
+		auto enum_name = enum_type->Lookup(v->AsEnum());
+		return {broker::enum_value(enum_name ? enum_name : "<unknown enum>")};
+		}
+	case TYPE_STRING:
+		{
+		auto s = v->AsString();
+		return {string(reinterpret_cast<const char*>(s->Bytes()), s->Len())};
+		}
+	case TYPE_FILE:
+		return {string(v->AsFile()->Name())};
+	case TYPE_FUNC:
+		return {string(v->AsFunc()->Name())};
+	case TYPE_TABLE:
+		{
+		auto is_set = v->Type()->IsSet();
+		auto table = v->AsTable();
+		auto table_val = v->AsTableVal();
+		broker::data rval;
+
+		if ( is_set )
+			rval = broker::set();
+		else
+			rval = broker::table();
+
+		struct iter_guard {
+			iter_guard(HashKey* arg_k, ListVal* arg_lv)
+			    : k(arg_k), lv(arg_lv)
+				{}
+
+			~iter_guard()
+				{
+				delete k;
+				Unref(lv);
+				}
+
+			HashKey* k;
+			ListVal* lv;
+		};
+
+		HashKey* k;
+		TableEntryVal* entry;
+		auto c = table->InitForIteration();
+
+		while ( (entry = table->NextEntry(k, c)) )
+			{
+			auto vl = table_val->RecoverIndex(k);
+			iter_guard ig(k, vl);
+
+			broker::vector composite_key;
+			composite_key.reserve(vl->Length());
+
+			for ( auto k = 0; k < vl->Length(); ++k )
+				{
+				auto key_part = val_to_data((*vl->Vals())[k]);
+
+				if ( ! key_part )
+					return {};
+
+				composite_key.emplace_back(move(*key_part));
+				}
+
+			broker::data key;
+
+			if ( composite_key.size() == 1 )
+				key = move(composite_key[0]);
+			else
+				key = move(composite_key);
+
+			if ( is_set )
+				broker::get<broker::set>(rval)->emplace(move(key));
+			else
+				{
+				auto val = val_to_data(entry->Value());
+
+				if ( ! val )
+					return {};
+
+				broker::get<broker::table>(rval)->emplace(move(key),
+				                                          move(*val));
+				}
+			}
+
+		return {rval};
+		}
+	case TYPE_VECTOR:
+		{
+		auto vec = v->AsVectorVal();
+		broker::vector rval;
+		rval.reserve(vec->Size());
+
+		for ( auto i = 0u; i < vec->Size(); ++i )
+			{
+			auto item_val = vec->Lookup(i);
+
+			if ( ! item_val )
+				continue;
+
+			auto item = val_to_data(item_val);
+
+			if ( ! item )
+				return {};
+
+			rval.emplace_back(move(*item));
+			}
+
+		return {rval};
+		}
+	case TYPE_RECORD:
+		{
+		auto rec = v->AsRecordVal();
+		broker::record rval;
+		size_t num_fields = v->Type()->AsRecordType()->NumFields();
+		rval.fields.reserve(num_fields);
+
+		for ( auto i = 0u; i < num_fields; ++i )
+			{
+			auto item_val = rec->LookupWithDefault(i);
+
+			if ( ! item_val )
+				{
+				rval.fields.emplace_back(broker::record::field{});
+				continue;
+				}
+
+			auto item = val_to_data(item_val);
+			Unref(item_val);
+
+			if ( ! item )
+				return {};
+
+			rval.fields.emplace_back(broker::record::field{move(*item)});
+			}
+
+		return {rval};
+		}
+	default:
+		reporter->Error("unsupported Comm::Data type: %s",
+		                type_name(v->Type()->Tag()));
+		break;
+	}
+
+	return {};
+	}
+
+RecordVal* comm::make_data_val(Val* v)
+	{
+	auto rval = new RecordVal(BifType::Record::Comm::Data);
+	auto data = val_to_data(v);
+
+	if ( data )
+		rval->Assign(0, new DataVal(move(*data)));
+
+	return rval;
+	}
+
+RecordVal* comm::make_data_val(broker::data d)
+	{
+	auto rval = new RecordVal(BifType::Record::Comm::Data);
+	rval->Assign(0, new DataVal(move(d)));
+	return rval;
+	}
+
+struct data_type_getter {
+	using result_type = EnumVal*;
+
+	result_type operator()(bool a)
+		{
+		return new EnumVal(BifEnum::Comm::BOOL,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(uint64_t a)
+		{
+		return new EnumVal(BifEnum::Comm::COUNT,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(int64_t a)
+		{
+		return new EnumVal(BifEnum::Comm::INT,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(double a)
+		{
+		return new EnumVal(BifEnum::Comm::DOUBLE,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const std::string& a)
+		{
+		return new EnumVal(BifEnum::Comm::STRING,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::address& a)
+		{
+		return new EnumVal(BifEnum::Comm::ADDR,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::subnet& a)
+		{
+		return new EnumVal(BifEnum::Comm::SUBNET,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::port& a)
+		{
+		return new EnumVal(BifEnum::Comm::PORT,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::time_point& a)
+		{
+		return new EnumVal(BifEnum::Comm::TIME,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::time_duration& a)
+		{
+		return new EnumVal(BifEnum::Comm::INTERVAL,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::enum_value& a)
+		{
+		return new EnumVal(BifEnum::Comm::ENUM,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::set& a)
+		{
+		return new EnumVal(BifEnum::Comm::SET,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::table& a)
+		{
+		return new EnumVal(BifEnum::Comm::TABLE,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::vector& a)
+		{
+		return new EnumVal(BifEnum::Comm::VECTOR,
+		                   BifType::Enum::Comm::DataType);
+		}
+
+	result_type operator()(const broker::record& a)
+		{
+		return new EnumVal(BifEnum::Comm::RECORD,
+		                   BifType::Enum::Comm::DataType);
+		}
+};
+
+EnumVal* comm::get_data_type(RecordVal* v, Frame* frame)
+	{
+	return broker::visit(data_type_getter{}, opaque_field_to_data(v, frame));
+	}
+
+broker::data& comm::opaque_field_to_data(RecordVal* v, Frame* f)
+	{
+	Val* d = v->Lookup(0);
+
+	if ( ! d )
+		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		                       "Comm::Data's opaque field is not set");
+
+	return static_cast<DataVal*>(d)->data;
+	}
+
+IMPLEMENT_SERIAL(comm::DataVal, SER_COMM_DATA_VAL);
+
+bool comm::DataVal::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_COMM_DATA_VAL, OpaqueVal);
+
+	std::string serial;
+	caf::binary_serializer bs(std::back_inserter(serial));
+	bs << data;
+
+	if ( ! SERIALIZE_STR(serial.data(), serial.size()) )
+		return false;
+
+	return true;
+	}
+
+bool comm::DataVal::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(OpaqueVal);
+
+	const char* serial;
+	int len;
+
+	if ( ! UNSERIALIZE_STR(&serial, &len) )
+		return false;
+
+	caf::binary_deserializer bd(serial, len);
+	caf::uniform_typeid<broker::data>()->deserialize(&data, &bd);
+	delete [] serial;
+	return true;
+	}

src/comm/Data.h

@@ -0,0 +1,254 @@
+#ifndef BRO_COMM_DATA_H
+#define BRO_COMM_DATA_H
+
+#include <broker/data.hh>
+#include "Val.h"
+#include "Reporter.h"
+#include "Frame.h"
+#include "Expr.h"
+
+namespace comm {
+
+extern OpaqueType* opaque_of_data_type;
+extern OpaqueType* opaque_of_set_iterator;
+extern OpaqueType* opaque_of_table_iterator;
+extern OpaqueType* opaque_of_vector_iterator;
+extern OpaqueType* opaque_of_record_iterator;
+
+/**
+ * Convert a broker port protocol to a bro port protocol.
+ */
+TransportProto to_bro_port_proto(broker::port::protocol tp);
+
+/**
+ * Create a Comm::Data value from a Bro value.
+ * @param v the Bro value to convert to a Broker data value.
+ * @return a Comm::Data value, where the optional field is set if the conversion
+ * was possible, else it is unset.
+ */
+RecordVal* make_data_val(Val* v);
+
+/**
+ * Create a Comm::Data value from a Broker data value.
+ * @param d the Broker value to wrap in an opaque type.
+ * @return a Comm::Data value that wraps the Broker value.
+ */
+RecordVal* make_data_val(broker::data d);
+
+/**
+ * Get the type of Broker data that Comm::Data wraps.
+ * @param v a Comm::Data value.
+ * @param frame used to get location info upon error.
+ * @return a Comm::DataType value.
+ */
+EnumVal* get_data_type(RecordVal* v, Frame* frame);
+
+/**
+ * Convert a Bro value to a Broker data value.
+ * @param v a Bro value.
+ * @return a Broker data value if the Bro value could be converted to one.
+ */
+broker::util::optional<broker::data> val_to_data(Val* v);
+
+/**
+ * Convert a Broker data value to a Bro value.
+ * @param d a Broker data value.
+ * @param type the expected type of the value to return.
+ * @return a pointer to a new Bro value or a nullptr if the conversion was not
+ * possible.
+ */
+Val* data_to_val(broker::data d, BroType* type);
+
+/**
+ * A Bro value which wraps a Broker data value.
+ */
+class DataVal : public OpaqueVal {
+public:
+
+	DataVal(broker::data arg_data)
+		: OpaqueVal(comm::opaque_of_data_type), data(std::move(arg_data))
+		{}
+
+	void ValDescribe(ODesc* d) const override
+		{
+		d->Add("broker::data{");
+		d->Add(broker::to_string(data));
+		d->Add("}");
+		}
+
+	DECLARE_SERIAL(DataVal);
+
+	broker::data data;
+
+protected:
+
+	DataVal()
+		{}
+};
+
+/**
+ * Visitor for retrieving type names a Broker data value.
+ */
+struct type_name_getter {
+	using result_type = const char*;
+
+	result_type operator()(bool a)
+		{ return "bool"; }
+
+	result_type operator()(uint64_t a)
+		{ return "uint64_t"; }
+
+	result_type operator()(int64_t a)
+		{ return "int64_t"; }
+
+	result_type operator()(double a)
+		{ return "double"; }
+
+	result_type operator()(const std::string& a)
+		{ return "string"; }
+
+	result_type operator()(const broker::address& a)
+		{ return "address"; }
+
+	result_type operator()(const broker::subnet& a)
+		{ return "subnet"; }
+
+	result_type operator()(const broker::port& a)
+		{ return "port"; }
+
+	result_type operator()(const broker::time_point& a)
+		{ return "time"; }
+
+	result_type operator()(const broker::time_duration& a)
+		{ return "interval"; }
+
+	result_type operator()(const broker::enum_value& a)
+		{ return "enum"; }
+
+	result_type operator()(const broker::set& a)
+		{ return "set"; }
+
+	result_type operator()(const broker::table& a)
+		{ return "table"; }
+
+	result_type operator()(const broker::vector& a)
+		{ return "vector"; }
+
+	result_type operator()(const broker::record& a)
+		{ return "record"; }
+};
+
+/**
+ * Retrieve Broker data value associated with a Comm::Data Bro value.
+ * @param v a Comm::Data value.
+ * @param f used to get location information on error.
+ * @return a reference to the wrapped Broker data value.  A runtime interpreter
+ * exception is thrown if the the optional opaque value of \a v is not set.
+ */
+broker::data& opaque_field_to_data(RecordVal* v, Frame* f);
+
+/**
+ * Retrieve variant data from a Broker data value.
+ * @tparam T a type that the variant may contain.
+ * @param d a Broker data value to get variant data out of.
+ * @param tag a Bro tag which corresponds to T (just used for error reporting).
+ * @param f used to get location information on error.
+ * @return a refrence to the requested type in the variant Broker data.
+ * A runtime interpret exception is thrown if trying to access a type which
+ * is not currently stored in the Broker data.
+ */
+template <typename T>
+T& require_data_type(broker::data& d, TypeTag tag, Frame* f)
+	{
+	auto ptr = broker::get<T>(d);
+
+	if ( ! ptr )
+		reporter->RuntimeError(f->GetCall()->GetLocationInfo(),
+		                       "data is of type '%s' not of type '%s'",
+		                       broker::visit(type_name_getter{}, d),
+		                       type_name(tag));
+
+	return *ptr;
+	}
+
+/**
+ * @see require_data_type() and opaque_field_to_data().
+ */
+template <typename T>
+inline T& require_data_type(RecordVal* v, TypeTag tag, Frame* f)
+	{
+	return require_data_type<T>(opaque_field_to_data(v, f), tag, f);
+	}
+
+/**
+ * Convert a Comm::Data Bro value to a Bro value of a given type.
+ * @tparam a type that a Broker data variant may contain.
+ * @param v a Comm::Data value.
+ * @param tag a Bro type to convert to.
+ * @param f used to get location information on error.
+ * A runtime interpret exception is thrown if trying to access a type which
+ * is not currently stored in the Broker data.
+ */
+template <typename T>
+inline Val* refine(RecordVal* v, TypeTag tag, Frame* f)
+	{
+	return new Val(require_data_type<T>(v, tag, f), tag);
+	}
+
+// Copying data in to iterator vals is not the fastest approach, but safer...
+
+class SetIterator : public OpaqueVal {
+public:
+
+	SetIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(comm::opaque_of_set_iterator),
+	      dat(require_data_type<broker::set>(v, TYPE_TABLE, f)),
+	      it(dat.begin())
+		{}
+
+	broker::set dat;
+	broker::set::iterator it;
+};
+
+class TableIterator : public OpaqueVal {
+public:
+
+	TableIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(comm::opaque_of_table_iterator),
+	      dat(require_data_type<broker::table>(v, TYPE_TABLE, f)),
+	      it(dat.begin())
+		{}
+
+	broker::table dat;
+	broker::table::iterator it;
+};
+
+class VectorIterator : public OpaqueVal {
+public:
+
+	VectorIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(comm::opaque_of_vector_iterator),
+	      dat(require_data_type<broker::vector>(v, TYPE_VECTOR, f)),
+	      it(dat.begin())
+		{}
+
+	broker::vector dat;
+	broker::vector::iterator it;
+};
+
+class RecordIterator : public OpaqueVal {
+public:
+
+	RecordIterator(RecordVal* v, TypeTag tag, Frame* f)
+	    : OpaqueVal(comm::opaque_of_record_iterator),
+	      dat(require_data_type<broker::record>(v, TYPE_VECTOR, f)),
+	      it(dat.fields.begin())
+		{}
+
+	broker::record dat;
+	decltype(broker::record::fields)::iterator it;
+};
+
+} // namespace comm
+
+#endif // BRO_COMM_DATA_H

src/comm/Manager.h

@@ -0,0 +1,364 @@
+#ifndef BRO_COMM_MANAGER_H
+#define BRO_COMM_MANAGER_H
+
+#include <broker/endpoint.hh>
+#include <broker/message_queue.hh>
+#include <memory>
+#include <string>
+#include <map>
+#include <unordered_set>
+#include "comm/Store.h"
+#include "Reporter.h"
+#include "iosource/IOSource.h"
+#include "Val.h"
+
+namespace comm {
+
+/**
+ * Communication statistics.  Some are tracked in relation to last
+ * sample (comm::Manager::ConsumeStatistics()).
+ */
+struct Stats {
+	// Number of outgoing peer connections (at time of sample).
+	size_t outgoing_peer_count = 0;
+	// Number of data stores (at time of sample).
+	size_t data_store_count = 0;
+	// Number of pending data store queries (at time of sample).
+	size_t pending_query_count = 0;
+	// Number of data store responses received (since last sample).
+	size_t response_count = 0;
+	// Number of outgoing connection updates received (since last sample).
+	size_t outgoing_conn_status_count = 0;
+	// Number of incoming connection updates received (since last sample).
+	size_t incoming_conn_status_count = 0;
+	// Number of broker report messages (e.g. debug, warning, errors) received
+	// (since last sample).
+	size_t report_count = 0;
+	// Number of print messages received per topic-prefix (since last sample).
+	std::map<std::string, size_t> print_count;
+	// Number of event messages received per topic-prefix (since last sample).
+	std::map<std::string, size_t> event_count;
+	// Number of log messages received per topic-prefix (since last sample).
+	std::map<std::string, size_t> log_count;
+};
+
+/**
+ * Manages various forms of communication between peer Bro processes
+ * or other external applications via use of the Broker messaging library.
+ */
+class Manager : public iosource::IOSource {
+friend class StoreHandleVal;
+public:
+
+	/**
+	 * Destructor.  Any still-pending data store queries are aborted.
+	 */
+	~Manager();
+
+	/**
+	 * Enable use of communication.
+	 * @param flags used to tune the local Broker endpoint's behavior.
+	 * See the Comm::EndpointFlags record type.
+	 * @return true if communication is successfully initialized.
+	 */
+	bool Enable(Val* flags);
+
+	/**
+	 * Changes endpoint flags originally supplied to comm::Manager::Enable().
+	 * @param flags the new behavior flags to use.
+	 * @return true if flags were changed.
+	 */
+	bool SetEndpointFlags(Val* flags);
+
+	/**
+	 * @return true if comm::Manager::Enable() has previously been called and
+	 * it succeeded.
+	 */
+	bool Enabled()
+		{ return endpoint != nullptr; }
+
+	/**
+	 * Listen for remote connections.
+	 * @param port the TCP port to listen on.
+	 * @param addr an address string on which to accept connections, e.g.
+	 * "127.0.0.1".  A nullptr refers to @p INADDR_ANY.
+	 * @param reuse_addr equivalent to behavior of SO_REUSEADDR.
+	 * @return true if the local endpoint is now listening for connections.
+	 */
+	bool Listen(uint16_t port, const char* addr = nullptr,
+	            bool reuse_addr = true);
+
+	/**
+	 * Initiate a remote connection.
+	 * @param addr an address to connect to, e.g. "localhost" or "127.0.0.1".
+	 * @param port the TCP port on which the remote side is listening.
+	 * @param retry_interval an interval at which to retry establishing the
+	 * connection with the remote peer.
+	 * @return true if it's possible to try connecting with the peer and
+	 * it's a new peer.  The actual connection may not be established until a
+	 * later point in time.
+	 */
+	bool Connect(std::string addr, uint16_t port,
+	             std::chrono::duration<double> retry_interval);
+
+	/**
+	 * Remove a remote connection.
+	 * @param addr the address used in comm::Manager::Connect().
+	 * @param port the port used in comm::Manager::Connect().
+	 * @return true if the arguments match a previously successful call to
+	 * comm::Manager::Connect().
+	 */
+	bool Disconnect(const std::string& addr, uint16_t port);
+
+	/**
+	 * Print a simple message to any interested peers.
+	 * @param topic a topic string associated with the print message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param msg the string to send to peers.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the Comm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Print(std::string topic, std::string msg, Val* flags);
+
+	/**
+	 * Send an event to any interested peers.
+	 * @param topic a topic string associated with the print message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param msg the event to send to peers, which is the name of the event
+	 * as a string followed by all of its arguments.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the Comm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Event(std::string topic, broker::message msg, int flags);
+
+	/**
+	 * Send an event to any interested peers.
+	 * @param topic a topic string associated with the print message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param args the event and its arguments to send to peers.  See the
+	 * Comm::EventArgs record type.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the Comm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Event(std::string topic, RecordVal* args, Val* flags);
+
+	/**
+	 * Send a log entry to any interested peers.  The topic name used is
+	 * implicitly "bro/log/<stream-name>".
+	 * @param stream_id the stream to which the log entry belongs.
+	 * @param columns the data which comprises the log entry.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the Comm::SendFlags record type.
+	 * @return true if the message is sent successfully.
+	 */
+	bool Log(EnumVal* stream_id, RecordVal* columns, int flags);
+
+	/**
+	 * Automatically send an event to any interested peers whenever it is
+	 * locally dispatched (e.g. using "event my_event(...);" in a script).
+	 * @param topic a topic string associated with the event message.
+	 * Peers advertise interest by registering a subscription to some prefix
+	 * of this topic name.
+	 * @param event a Bro event value.
+	 * @param flags tune the behavior of how the message is send.
+	 * See the Comm::SendFlags record type.
+	 * @return true if automatic event sending is now enabled.
+	 */
+	bool AutoEvent(std::string topic, Val* event, Val* flags);
+
+	/**
+	 * Stop automatically sending an event to peers upon local dispatch.
+	 * @param topic a topic originally given to comm::Manager::AutoEvent().
+	 * @param event an event originally given to comm::Manager::AutoEvent().
+	 * @return true if automatic events will no occur for the topic/event pair.
+	 */
+	bool AutoEventStop(const std::string& topic, Val* event);
+
+	/**
+	 * Create an EventArgs record value from an event and its arguments.
+	 * @param args the event and its arguments.  The event is always the first
+	 * elements in the list.
+	 * @return an EventArgs record value.  If an invalid event or arguments
+	 * were supplied the optional "name" field will not be set.
+	 */
+	RecordVal* MakeEventArgs(val_list* args);
+
+	/**
+	 * Register interest in peer print messages that use a certain topic prefix.
+	 * @param topic_prefix a prefix to match against remote message topics.
+	 * e.g. an empty prefix will match everything and "a" will match "alice"
+	 * and "amy" but not "bob".
+	 * @return true if it's a new print subscriptions and it is now registered.
+	 */
+	bool SubscribeToPrints(std::string topic_prefix);
+
+	/**
+	 * Unregister interest in peer print messages.
+	 * @param topic_prefix a prefix previously supplied to a successful call
+	 * to comm::Manager::SubscribeToPrints().
+	 * @return true if interest in topic prefix is no longer advertised.
+	 */
+	bool UnsubscribeToPrints(const std::string& topic_prefix);
+
+	/**
+	 * Register interest in peer event messages that use a certain topic prefix.
+	 * @param topic_prefix a prefix to match against remote message topics.
+	 * e.g. an empty prefix will match everything and "a" will match "alice"
+	 * and "amy" but not "bob".
+	 * @return true if it's a new event subscription and it is now registered.
+	 */
+	bool SubscribeToEvents(std::string topic_prefix);
+
+	/**
+	 * Unregister interest in peer event messages.
+	 * @param topic_prefix a prefix previously supplied to a successful call
+	 * to comm::Manager::SubscribeToEvents().
+	 * @return true if interest in topic prefix is no longer advertised.
+	 */
+	bool UnsubscribeToEvents(const std::string& topic_prefix);
+
+	/**
+	 * Register interest in peer log messages that use a certain topic prefix.
+	 * @param topic_prefix a prefix to match against remote message topics.
+	 * e.g. an empty prefix will match everything and "a" will match "alice"
+	 * and "amy" but not "bob".
+	 * @return true if it's a new log subscription and it is now registered.
+	 */
+	bool SubscribeToLogs(std::string topic_prefix);
+
+	/**
+	 * Unregister interest in peer log messages.
+	 * @param topic_prefix a prefix previously supplied to a successful call
+	 * to comm::Manager::SubscribeToLogs().
+	 * @return true if interest in topic prefix is no longer advertised.
+	 */
+	bool UnsubscribeToLogs(const std::string& topic_prefix);
+
+	/**
+	 * Allow sending messages to peers if associated with the given topic.
+	 * This has no effect if auto publication behavior is enabled via the flags
+	 * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags().
+	 * @param t a topic to allow messages to be published under.
+	 * @return true if successful.
+	 */
+	bool PublishTopic(broker::topic t);
+
+	/**
+	 * Disallow sending messages to peers if associated with the given topic.
+	 * This has no effect if auto publication behavior is enabled via the flags
+	 * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags().
+	 * @param t a topic to disallow messages to be published under.
+	 * @return true if successful.
+	 */
+	bool UnpublishTopic(broker::topic t);
+
+	/**
+	 * Allow advertising interest in the given topic to peers.
+	 * This has no effect if auto advertise behavior is enabled via the flags
+	 * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags().
+	 * @param t a topic to allow advertising interest/subscription to peers.
+	 * @return true if successful.
+	 */
+	bool AdvertiseTopic(broker::topic t);
+
+	/**
+	 * Disallow advertising interest in the given topic to peers.
+	 * This has no effect if auto advertise behavior is enabled via the flags
+	 * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags().
+	 * @param t a topic to disallow advertising interest/subscription to peers.
+	 * @return true if successful.
+	 */
+	bool UnadvertiseTopic(broker::topic t);
+
+	/**
+	 * Register the availability of a data store.
+	 * @param handle the data store.
+	 * @return true if the store was valid and not already away of it.
+	 */
+	bool AddStore(StoreHandleVal* handle);
+
+	/**
+	 * Lookup a data store by it's identifier name and type.
+	 * @param id the store's name.
+	 * @param type the type of data store.
+	 * @return a pointer to the store handle if it exists else nullptr.
+	 */
+	StoreHandleVal* LookupStore(const broker::store::identifier& id, StoreType type);
+
+	/**
+	 * Close and unregister a data store.  Any existing references to the
+	 * store handle will not be able to be used for any data store operations.
+	 * @param id the stores' name.
+	 * @param type the type of the data store.
+	 * @return true if such a store existed and is now closed.
+	 */
+	bool CloseStore(const broker::store::identifier& id, StoreType type);
+
+	/**
+	 * Register a data store query callback.
+	 * @param cb the callback info to use when the query completes or times out.
+	 * @return true if now tracking a data store query.
+	 */
+	bool TrackStoreQuery(StoreQueryCallback* cb);
+
+	/**
+	 * @return communication statistics.
+	 */
+	Stats ConsumeStatistics();
+
+	/**
+	 * Convert Comm::SendFlags to int flags for use with broker::send().
+	 */
+	static int send_flags_to_int(Val* flags);
+
+private:
+
+	// IOSource interface overrides:
+	void GetFds(iosource::FD_Set* read, iosource::FD_Set* write,
+	            iosource::FD_Set* except) override;
+
+	double NextTimestamp(double* local_network_time) override;
+
+	void Process() override;
+
+	const char* Tag() override
+		{ return "Comm::Manager"; }
+
+	broker::endpoint& Endpoint()
+		{ return *endpoint; }
+
+	struct QueueWithStats {
+		broker::message_queue q;
+		size_t received = 0;
+	};
+
+	std::unique_ptr<broker::endpoint> endpoint;
+	std::map<std::pair<std::string, uint16_t>, broker::peering> peers;
+	std::map<std::string, QueueWithStats> print_subscriptions;
+	std::map<std::string, QueueWithStats> event_subscriptions;
+	std::map<std::string, QueueWithStats> log_subscriptions;
+
+	std::map<std::pair<broker::store::identifier, StoreType>,
+	         StoreHandleVal*> data_stores;
+	std::unordered_set<StoreQueryCallback*> pending_queries;
+
+	Stats statistics;
+
+	static VectorType* vector_of_data_type;
+	static EnumType* log_id_type;
+	static int send_flags_self_idx;
+	static int send_flags_peers_idx;
+	static int send_flags_unsolicited_idx;
+};
+
+} // namespace comm
+
+extern comm::Manager* comm_mgr;
+
+#endif // BRO_COMM_MANAGER_H

src/comm/Store.cc

@@ -0,0 +1,204 @@
+#include "Store.h"
+#include "comm/Manager.h"
+
+#include <broker/store/master.hh>
+#include <broker/store/clone.hh>
+#include <broker/store/sqlite_backend.hh>
+
+#ifdef HAVE_ROCKSDB
+#include <broker/store/rocksdb_backend.hh>
+#include <rocksdb/db.h>
+#endif
+
+OpaqueType* comm::opaque_of_store_handle;
+
+comm::StoreHandleVal::StoreHandleVal(broker::store::identifier id,
+                                     comm::StoreType arg_type,
+                                     broker::util::optional<BifEnum::Store::BackendType> arg_back,
+                                     RecordVal* backend_options, std::chrono::duration<double> resync)
+	: OpaqueVal(opaque_of_store_handle),
+	  store(), store_type(arg_type), backend_type(arg_back)
+	{
+	using BifEnum::Store::BackendType;
+	std::unique_ptr<broker::store::backend> backend;
+
+	if ( backend_type )
+		switch ( *backend_type ) {
+		case BackendType::MEMORY:
+			backend.reset(new broker::store::memory_backend);
+			break;
+		case BackendType::SQLITE:
+			{
+			auto sqlite = new broker::store::sqlite_backend;
+			std::string path = backend_options->Lookup(0)->AsRecordVal()
+			                   ->Lookup(0)->AsStringVal()->CheckString();
+
+			if ( sqlite->open(path) )
+				backend.reset(sqlite);
+			else
+				{
+				reporter->Error("failed to open sqlite backend at path %s: %s",
+				                path.data(), sqlite->last_error().data());
+				delete sqlite;
+				}
+			}
+			break;
+		case BackendType::ROCKSDB:
+			{
+#ifdef HAVE_ROCKSDB
+			std::string path = backend_options->Lookup(1)->AsRecordVal()
+			                   ->Lookup(0)->AsStringVal()->CheckString();
+			rocksdb::Options rock_op;
+			rock_op.create_if_missing = true;
+
+			auto rocksdb = new broker::store::rocksdb_backend;
+
+			if ( rocksdb->open(path, options).ok() )
+				backend.reset(rocksdb);
+			else
+				{
+				reporter->Error("failed to open rocksdb backend at path %s: %s",
+				                path.data(), rocksdb->last_error().data());
+				delete rocksdb;
+				}
+#else
+			reporter->Error("rocksdb backend support is not enabled");
+#endif
+			}
+			break;
+		default:
+			reporter->FatalError("unknown data store backend: %d",
+			                     static_cast<int>(*backend_type));
+			}
+
+	switch ( store_type ) {
+	case StoreType::FRONTEND:
+		store = new broker::store::frontend(comm_mgr->Endpoint(), move(id));
+		break;
+	case StoreType::MASTER:
+		store = new broker::store::master(comm_mgr->Endpoint(), move(id),
+		                                  move(backend));
+		break;
+	case StoreType::CLONE:
+		store = new broker::store::clone(comm_mgr->Endpoint(), move(id), resync,
+		                                 move(backend));
+		break;
+	default:
+		reporter->FatalError("unknown data store type: %d",
+		                     static_cast<int>(store_type));
+		}
+	}
+
+void comm::StoreHandleVal::ValDescribe(ODesc* d) const
+	{
+	using BifEnum::Store::BackendType;
+	d->Add("broker::store::");
+
+	switch ( store_type ) {
+	case StoreType::FRONTEND:
+		d->Add("frontend");
+		break;
+	case StoreType::MASTER:
+		d->Add("master");
+		break;
+	case StoreType::CLONE:
+		d->Add("clone");
+		break;
+	default:
+		d->Add("unknown");
+		}
+
+	d->Add("{");
+	d->Add(store->id());
+
+	if ( backend_type )
+		{
+		d->Add(", ");
+
+		switch ( *backend_type ) {
+		case BackendType::MEMORY:
+			d->Add("memory");
+			break;
+		case BackendType::SQLITE:
+			d->Add("sqlite");
+			break;
+		case BackendType::ROCKSDB:
+			d->Add("rocksdb");
+			break;
+		default:
+			d->Add("unknown");
+			}
+		}
+
+	d->Add("}");
+	}
+
+IMPLEMENT_SERIAL(comm::StoreHandleVal, SER_COMM_STORE_HANDLE_VAL);
+
+bool comm::StoreHandleVal::DoSerialize(SerialInfo* info) const
+	{
+	DO_SERIALIZE(SER_COMM_STORE_HANDLE_VAL, OpaqueVal);
+
+	bool have_store = store != nullptr;
+
+	if ( ! SERIALIZE(have_store) )
+		return false;
+
+	if ( ! have_store )
+		return true;
+
+	if ( ! SERIALIZE(static_cast<int>(store_type)) )
+		return false;
+
+	if ( ! SERIALIZE_STR(store->id().data(), store->id().size()) )
+		return false;
+
+	return true;
+	}
+
+bool comm::StoreHandleVal::DoUnserialize(UnserialInfo* info)
+	{
+	DO_UNSERIALIZE(OpaqueVal);
+
+	bool have_store;
+
+	if ( ! UNSERIALIZE(&have_store) )
+		return false;
+
+	if ( ! have_store )
+		{
+		store = nullptr;
+		return true;
+		}
+
+	int type;
+
+	if ( ! UNSERIALIZE(&type) )
+		return false;
+
+	const char* id_str;
+	int len;
+
+	if ( ! UNSERIALIZE_STR(&id_str, &len) )
+		return false;
+
+	broker::store::identifier id(id_str, len);
+	delete [] id_str;
+
+	auto handle = comm_mgr->LookupStore(id, static_cast<comm::StoreType>(type));
+
+	if ( ! handle )
+		{
+		// Passing serialized version of store handles to other Bro processes
+		// doesn't make sense, only allow local clones of the handle val.
+		reporter->Error("failed to look up unserialized store handle %s, %d",
+		                id.data(), type);
+		store = nullptr;
+		return false;
+		}
+
+	store = handle->store;
+	store_type = handle->store_type;
+	backend_type = handle->backend_type;
+	return true;
+	}

src/comm/Store.h

@@ -0,0 +1,153 @@
+#ifndef BRO_COMM_STORE_H
+#define BRO_COMM_STORE_H
+
+#include "comm/store.bif.h"
+#include "comm/data.bif.h"
+#include "Reporter.h"
+#include "Type.h"
+#include "Val.h"
+#include "Trigger.h"
+
+#include <broker/store/frontend.hh>
+
+namespace comm {
+
+extern OpaqueType* opaque_of_store_handle;
+
+/**
+ * Enumerates the possible types of data stores.
+ */
+enum StoreType {
+	// Just a view in to a remote store, contains no data itself.
+	FRONTEND,
+	MASTER,
+	CLONE,
+};
+
+/**
+ * Create a Store::QueryStatus value.
+ * @param success whether the query status should be set to success or failure.
+ * @return a Store::QueryStatus value.
+ */
+inline EnumVal* query_status(bool success)
+	{
+	static EnumType* store_query_status = nullptr;
+	static int success_val;
+	static int failure_val;
+
+	if ( ! store_query_status )
+		{
+		store_query_status = internal_type("Store::QueryStatus")->AsEnumType();
+		success_val = store_query_status->Lookup("Store", "SUCCESS");
+		failure_val = store_query_status->Lookup("Store", "FAILURE");
+		}
+
+	return new EnumVal(success ? success_val : failure_val, store_query_status);
+	}
+
+/**
+ * @return a Store::QueryResult value that has a Store::QueryStatus indicating
+ * a failure.
+ */
+inline RecordVal* query_result()
+	{
+	auto rval = new RecordVal(BifType::Record::Store::QueryResult);
+	rval->Assign(0, query_status(false));
+	rval->Assign(1, new RecordVal(BifType::Record::Comm::Data));
+	return rval;
+	}
+
+/**
+ * @param data the result of the query.
+ * @return a Store::QueryResult value that has a Store::QueryStatus indicating
+ * a success.
+ */
+inline RecordVal* query_result(RecordVal* data)
+	{
+	auto rval = new RecordVal(BifType::Record::Store::QueryResult);
+	rval->Assign(0, query_status(true));
+	rval->Assign(1, data);
+	return rval;
+	}
+
+/**
+ * Used for asynchronous data store queries which use "when" statements.
+ */
+class StoreQueryCallback {
+public:
+
+	StoreQueryCallback(Trigger* arg_trigger, const CallExpr* arg_call,
+					   broker::store::identifier arg_store_id,
+	                   StoreType arg_store_type)
+		: trigger(arg_trigger), call(arg_call), store_id(move(arg_store_id)),
+	      store_type(arg_store_type)
+		{
+		Ref(trigger);
+		}
+
+	~StoreQueryCallback()
+		{
+		Unref(trigger);
+		}
+
+	void Result(RecordVal* result)
+		{
+		trigger->Cache(call, result);
+		trigger->Release();
+		Unref(result);
+		}
+
+	void Abort()
+		{
+		auto result = query_result();
+		trigger->Cache(call, result);
+		trigger->Release();
+		Unref(result);
+		}
+
+	bool Disabled() const
+		{ return trigger->Disabled(); }
+
+	const broker::store::identifier& StoreID() const
+		{ return store_id; }
+
+	StoreType GetStoreType() const
+		{ return store_type; }
+
+private:
+
+	Trigger* trigger;
+	const CallExpr* call;
+	broker::store::identifier store_id;
+	StoreType store_type;
+};
+
+/**
+ * An opaque handle which wraps a Broker data store.
+ */
+class StoreHandleVal : public OpaqueVal {
+public:
+
+	StoreHandleVal(broker::store::identifier id,
+		       comm::StoreType arg_type,
+		       broker::util::optional<BifEnum::Store::BackendType> arg_back,
+		       RecordVal* backend_options,
+		       std::chrono::duration<double> resync = std::chrono::seconds(1));
+
+	void ValDescribe(ODesc* d) const override;
+
+	DECLARE_SERIAL(StoreHandleVal);
+
+	broker::store::frontend* store;
+	comm::StoreType store_type;
+	broker::util::optional<BifEnum::Store::BackendType> backend_type;
+
+protected:
+
+	StoreHandleVal()
+		{}
+};
+
+} // namespace comm
+
+#endif // BRO_COMM_STORE_H

src/comm/comm.bif

@@ -0,0 +1,199 @@
+
+##! General functions regarding Bro's broker communication mechanisms.
+
+%%{
+#include "comm/Manager.h"
+%%}
+
+module Comm;
+
+type Comm::EndpointFlags: record;
+
+## Enable use of communication.
+##
+## flags: used to tune the local Broker endpoint behavior.
+##
+## Returns: true if communication is successfully initialized.
+function Comm::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool
+	%{
+	return new Val(comm_mgr->Enable(flags), TYPE_BOOL);
+	%}
+
+## Changes endpoint flags originally supplied to :bro:see:`Comm::enable`.
+##
+## flags: the new endpoint behavior flags to use.
+##
+## Returns: true of flags were changed.
+function Comm::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool
+	%{
+	return new Val(comm_mgr->SetEndpointFlags(flags), TYPE_BOOL);
+	%}
+
+## Allow sending messages to peers if associated with the given topic.
+## This has no effect if auto publication behavior is enabled via the flags
+## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`.
+##
+## topic: a topic to allow messages to be published under.
+##
+## Returns: true if successful.
+function Comm::publish_topic%(topic: string%): bool
+	%{
+	return new Val(comm_mgr->PublishTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Disallow sending messages to peers if associated with the given topic.
+## This has no effect if auto publication behavior is enabled via the flags
+## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`.
+##
+## topic: a topic to disallow messages to be published under.
+##
+## Returns: true if successful.
+function Comm::unpublish_topic%(topic: string%): bool
+	%{
+	return new Val(comm_mgr->UnpublishTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Allow advertising interest in the given topic to peers.
+## This has no effect if auto advertise behavior is enabled via the flags
+## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`.
+##
+## topic: a topic to allow advertising interest/subscription to peers.
+##
+## Returns: true if successful.
+function Comm::advertise_topic%(topic: string%): bool
+	%{
+	return new Val(comm_mgr->AdvertiseTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Disallow advertising interest in the given topic to peers.
+## This has no effect if auto advertise behavior is enabled via the flags
+## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`.
+##
+## topic: a topic to disallow advertising interest/subscription to peers.
+##
+## Returns: true if successful.
+function Comm::unadvertise_topic%(topic: string%): bool
+	%{
+	return new Val(comm_mgr->UnadvertiseTopic(topic->CheckString()), TYPE_BOOL);
+	%}
+
+## Generated when a connection has been established due to a previous call
+## to :bro:see:`Comm::connect`.
+##
+## peer_address: the address used to connect to the peer.
+##
+## peer_port: the port used to connect to the peer.
+##
+## peer_name: the name by which the peer identified itself.
+event Comm::outgoing_connection_established%(peer_address: string,
+                                             peer_port: port,
+                                             peer_name: string%);
+
+## Generated when a previously established connection becomes broken.
+## Reconnection will automatically be attempted at a frequency given
+## by the original call to :bro:see:`Comm::connect`.
+##
+## peer_address: the address used to connect to the peer.
+##
+## peer_port: the port used to connect to the peer.
+##
+## .. bro:see:: Comm::outgoing_connection_established
+event Comm::outgoing_connection_broken%(peer_address: string,
+                                        peer_port: port%);
+
+## Generated when a connection via :bro:see:`Comm::connect` has failed
+## because the remote side is incompatible.
+##
+## peer_address: the address used to connect to the peer.
+##
+## peer_port: the port used to connect to the peer.
+event Comm::outgoing_connection_incompatible%(peer_address: string,
+                                              peer_port: port%);
+
+## Generated when a peer has established a connection with this process
+## as a result of previously performing a :bro:see:`Comm::listen`.
+##
+## peer_name: the name by which the peer identified itself.
+event Comm::incoming_connection_established%(peer_name: string%);
+
+## Generated when a peer that previously established a connection with this
+## process becomes disconnected.
+##
+## peer_name: the name by which the peer identified itself.
+##
+## .. bro:see:: Comm::incoming_connection_established
+event Comm::incoming_connection_broken%(peer_name: string%);
+
+## Listen for remote connections.
+##
+## p: the TCP port to listen on.
+##
+## a: an address string on which to accept connections, e.g.
+##    "127.0.0.1".  An empty string refers to @p INADDR_ANY.
+##
+## reuse: equivalent to behavior of SO_REUSEADDR.
+##
+## Returns: true if the local endpoint is now listening for connections.
+##
+## .. bro:see:: Comm::incoming_connection_established
+function Comm::listen%(p: port, a: string &default = "",
+					   reuse: bool &default = T%): bool
+	%{
+	if ( ! p->IsTCP() )
+		{
+		reporter->Error("listen port must use tcp");
+		return new Val(false, TYPE_BOOL);
+		}
+
+	auto rval = comm_mgr->Listen(p->Port(), a->Len() ? a->CheckString() : 0,
+								 reuse);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Initiate a remote connection.
+##
+## a: an address to connect to, e.g. "localhost" or "127.0.0.1".
+##
+## p: the TCP port on which the remote side is listening.
+##
+## retry: an interval at which to retry establishing the
+##        connection with the remote peer if it cannot be made initially, or
+##        if it ever becomes disconnected.
+##
+## Returns: true if it's possible to try connecting with the peer and
+##          it's a new peer.  The actual connection may not be established
+##          a later point in time.
+##
+## .. bro:see:: Comm::outgoing_connection_established
+function Comm::connect%(a: string, p: port, retry: interval%): bool
+	%{
+	if ( ! p->IsTCP() )
+		{
+		reporter->Error("remote connection port must use tcp");
+		return new Val(false, TYPE_BOOL);
+		}
+
+	auto rval = comm_mgr->Connect(a->CheckString(), p->Port(),
+	                              std::chrono::duration<double>(retry));
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Remove a remote connection.
+##
+## a: the address used in previous successful call to :bro:see:`Comm::connect`.
+##
+## p: the port used in previous successful call to :bro:see:`Comm::connect`.
+##
+## Returns: true if the arguments match a previously successful call to
+##          :bro:see:`Comm::connect`.
+function Comm::disconnect%(a: string, p: port%): bool
+	%{
+	if ( ! p->IsTCP() )
+		{
+		reporter->Error("remote connection port must use tcp");
+		return new Val(false, TYPE_BOOL);
+		}
+
+	auto rval = comm_mgr->Disconnect(a->CheckString(), p->Port());
+	return new Val(rval, TYPE_BOOL);
+	%}

src/comm/data.bif

@@ -0,0 +1,834 @@
+
+##! Functions for inspecting and manipulating broker data.
+
+%%{
+#include "comm/Data.h"
+%%}
+
+module Comm;
+
+## Enumerates the possible types that :bro:see:`Comm::Data` may be in terms of
+## Bro data types.
+enum DataType %{
+	BOOL,
+	INT,
+	COUNT,
+	DOUBLE,
+	STRING,
+	ADDR,
+	SUBNET,
+	PORT,
+	TIME,
+	INTERVAL,
+	ENUM,
+	SET,
+	TABLE,
+	VECTOR,
+	RECORD,
+%}
+
+type Comm::Data: record;
+
+type Comm::TableItem: record;
+
+## Convert any Bro value in to communication data.
+##
+## d: any Bro value to attempt to convert (not all types are supported).
+##
+## Returns: the converted communication data which may not set its only
+##          opaque field of the the conversion was not possible (the Bro data
+##          type does not support being converted to communicaiton data).
+function Comm::data%(d: any%): Comm::Data
+	%{
+	return comm::make_data_val(d);
+	%}
+
+## Retrieve the type of data associated with communication data.
+##
+## d: the communication data.
+##
+## Returns: the data type associated with the communication data.
+function Comm::data_type%(d: Comm::Data%): Comm::DataType
+	%{
+	return comm::get_data_type(d->AsRecordVal(), frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::BOOL` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_bool%(d: Comm::Data%): bool
+	%{
+	return comm::refine<bool>(d->AsRecordVal(), TYPE_BOOL, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::INT` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_int%(d: Comm::Data%): int
+	%{
+	return comm::refine<int64_t>(d->AsRecordVal(), TYPE_INT, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::COUNT` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_count%(d: Comm::Data%): count
+	%{
+	return comm::refine<uint64_t>(d->AsRecordVal(), TYPE_COUNT, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::DOUBLE` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_double%(d: Comm::Data%): double
+	%{
+	return comm::refine<double>(d->AsRecordVal(), TYPE_DOUBLE, frame);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::STRING` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_string%(d: Comm::Data%): string
+	%{
+	return new StringVal(comm::require_data_type<std::string>(d->AsRecordVal(),
+												              TYPE_STRING,
+															  frame));
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::ADDR` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_addr%(d: Comm::Data%): addr
+	%{
+	auto& a = comm::require_data_type<broker::address>(d->AsRecordVal(),
+													   TYPE_ADDR, frame);
+	auto bits = reinterpret_cast<const in6_addr*>(&a.bytes());
+	return new AddrVal(IPAddr(*bits));
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::SUBNET` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_subnet%(d: Comm::Data%): subnet
+	%{
+	auto& a = comm::require_data_type<broker::subnet>(d->AsRecordVal(),
+													  TYPE_SUBNET, frame);
+	auto bits = reinterpret_cast<const in6_addr*>(&a.network().bytes());
+	return new SubNetVal(IPPrefix(IPAddr(*bits), a.length()));
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::PORT` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_port%(d: Comm::Data%): port
+	%{
+	auto& a = comm::require_data_type<broker::port>(d->AsRecordVal(),
+													TYPE_SUBNET, frame);
+	return new PortVal(a.number(), comm::to_bro_port_proto(a.type()));
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::TIME` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_time%(d: Comm::Data%): time
+	%{
+	auto v = comm::require_data_type<broker::time_point>(d->AsRecordVal(),
+														TYPE_TIME, frame).value;
+	return new Val(v, TYPE_TIME);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::INTERVAL` to
+## an actual Bro value.
+##
+## d: the communication data to convert.
+##
+## Returns: the value retrieved from the communication data.
+function Comm::refine_to_interval%(d: Comm::Data%): interval
+	%{
+	auto v = comm::require_data_type<broker::time_duration>(d->AsRecordVal(),
+														TYPE_TIME, frame).value;
+	return new Val(v, TYPE_INTERVAL);
+	%}
+
+## Convert communication data with a type of :bro:see:`Comm::ENUM` to
+## the name of the enum value.  :bro:see:`lookup_ID` may be used to convert
+## the name to the actual enum value.
+##
+## d: the communication data to convert.
+##
+## Returns: the enum name retrieved from the communication data.
+function Comm::refine_to_enum_name%(d: Comm::Data%): string
+	%{
+	auto& v = comm::require_data_type<broker::enum_value>(d->AsRecordVal(),
+														TYPE_ENUM, frame).name;
+	return new StringVal(v);
+	%}
+
+## Create communication data of type "set".
+function Comm::set_create%(%): Comm::Data
+	%{
+	return comm::make_data_val(broker::set());
+	%}
+
+## Remove all elements within a set.
+##
+## s: the set to clear.
+##
+## Returns: always true.
+function Comm::set_clear%(s: Comm::Data%): bool
+	%{
+	auto& v = comm::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	v.clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Get the number of elements within a set.
+##
+## s: the set to query.
+##
+## Returns: the number of elements in the set.
+function Comm::set_size%(s: Comm::Data%): count
+	%{
+	auto& v = comm::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	return new Val(static_cast<uint64_t>(v.size()), TYPE_COUNT);
+	%}
+
+## Check if a set contains a particular element.
+##
+## s: the set to query.
+##
+## key: the element to check for existence.
+##
+## Returns: true if the key exists in the set.
+function Comm::set_contains%(s: Comm::Data, key: Comm::Data%): bool
+	%{
+	auto& v = comm::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.find(k) != v.end(), TYPE_BOOL);
+	%}
+
+### Insert an element into a set.
+##
+## s: the set to modify.
+##
+## key: the element to insert.
+##
+## Returns: true if the key was inserted, or false if it already existed.
+function Comm::set_insert%(s: Comm::Data, key: Comm::Data%): bool
+	%{
+	auto& v = comm::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.insert(k).second, TYPE_BOOL);
+	%}
+
+## Remove an element from a set.
+##
+## s: the set to modify.
+##
+## key: the element to remove.
+##
+## Returns: true if the element existed in the set and is now removed.
+function Comm::set_remove%(s: Comm::Data, key: Comm::Data%): bool
+	%{
+	auto& v = comm::require_data_type<broker::set>(s->AsRecordVal(), TYPE_TABLE,
+												   frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.erase(k) > 0, TYPE_BOOL);
+	%}
+
+## Create an iterator for a set.  Note that this makes a copy of the set
+## internally to ensure the iterator is always valid.
+##
+## s: the set to iterate over.
+##
+## Returns: an iterator.
+function Comm::set_iterator%(s: Comm::Data%): opaque of Comm::SetIterator
+	%{
+	return new comm::SetIterator(s->AsRecordVal(), TYPE_TABLE, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function Comm::set_iterator_last%(it: opaque of Comm::SetIterator%): bool
+	%{
+	auto set_it = static_cast<comm::SetIterator*>(it);
+	return new Val(set_it->it == set_it->dat.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function Comm::set_iterator_next%(it: opaque of Comm::SetIterator%): bool
+	%{
+	auto set_it = static_cast<comm::SetIterator*>(it);
+
+	if ( set_it->it == set_it->dat.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++set_it->it;
+	return new Val(set_it->it != set_it->dat.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function Comm::set_iterator_value%(it: opaque of Comm::SetIterator%): Comm::Data
+	%{
+	auto set_it = static_cast<comm::SetIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::Comm::Data);
+
+	if ( set_it->it == set_it->dat.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid set iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	rval->Assign(0, new comm::DataVal(*set_it->it));
+	return rval;
+	%}
+
+## Create communication data of type "table".
+function Comm::table_create%(%): Comm::Data
+	%{
+	return comm::make_data_val(broker::table());
+	%}
+
+## Remove all elements within a table.
+##
+## t: the table to clear.
+##
+## Returns: always true.
+function Comm::table_clear%(t: Comm::Data%): bool
+	%{
+	auto& v = comm::require_data_type<broker::table>(t->AsRecordVal(),
+													 TYPE_TABLE, frame);
+	v.clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Get the number of elements within a table.
+##
+## t: the table to query.
+##
+## Returns: the number of elements in the table.
+function Comm::table_size%(t: Comm::Data%): count
+	%{
+	auto& v = comm::require_data_type<broker::table>(t->AsRecordVal(),
+													 TYPE_TABLE, frame);
+	return new Val(static_cast<uint64_t>(v.size()), TYPE_COUNT);
+	%}
+
+## Check if a table contains a particular key.
+##
+## t: the table to query.
+##
+## key: the key to check for existence.
+##
+## Returns: true if the key exists in the set.
+function Comm::table_contains%(t: Comm::Data, key: Comm::Data%): bool
+	%{
+	auto& v = comm::require_data_type<broker::table>(t->AsRecordVal(),
+													 TYPE_TABLE, frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	return new Val(v.find(k) != v.end(), TYPE_BOOL);
+	%}
+
+## Insert a key-value pair into a table.
+##
+## t: the table to modify.
+##
+## key: the key at which to insert the value.
+##
+## val: the value to insert.
+##
+## Returns: true if the key-value pair was inserted, or false if the key
+##          already existed in the table.
+function Comm::table_insert%(t: Comm::Data, key: Comm::Data, val: Comm::Data%): Comm::Data
+	%{
+	auto& table = comm::require_data_type<broker::table>(t->AsRecordVal(),
+													     TYPE_TABLE, frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	auto& v = comm::opaque_field_to_data(val->AsRecordVal(), frame);
+
+	try
+		{
+		auto& prev = table.at(k);
+		auto rval = comm::make_data_val(move(prev));
+		prev = v;
+		return rval;
+		}
+	catch (const std::out_of_range&)
+		{
+		table[k] = v;
+		return new RecordVal(BifType::Record::Comm::Data);
+		}
+	%}
+
+## Remove a key-value pair from a table.
+##
+## t: the table to modify.
+##
+## key: the key to remove from the table.
+##
+## Returns: the value associated with the key.  If the key did not exist, then
+##          the optional field of the returned record is not set.
+function Comm::table_remove%(t: Comm::Data, key: Comm::Data%): Comm::Data
+	%{
+	auto& table = comm::require_data_type<broker::table>(t->AsRecordVal(),
+													     TYPE_TABLE, frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	auto it = table.find(k);
+
+	if ( it == table.end() )
+		return new RecordVal(BifType::Record::Comm::Data);
+	else
+		{
+		auto rval = comm::make_data_val(move(it->second));
+		table.erase(it);
+		return rval;
+		}
+	%}
+
+## Retrieve a value from a table.
+##
+## t: the table to query.
+##
+## key: the key to lookup.
+##
+## Returns: the value associated with the key.  If the key did not exist, then
+##          the optional field of the returned record is not set.
+function Comm::table_lookup%(t: Comm::Data, key: Comm::Data%): Comm::Data
+	%{
+	auto& table = comm::require_data_type<broker::table>(t->AsRecordVal(),
+													     TYPE_TABLE, frame);
+	auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame);
+	auto it = table.find(k);
+
+	if ( it == table.end() )
+		return new RecordVal(BifType::Record::Comm::Data);
+	else
+		return comm::make_data_val(it->second);
+	%}
+
+## Create an iterator for a table.  Note that this makes a copy of the table
+## internally to ensure the iterator is always valid.
+##
+## t: the table to iterate over.
+##
+## Returns: an iterator.
+function Comm::table_iterator%(t: Comm::Data%): opaque of Comm::TableIterator
+	%{
+	return new comm::TableIterator(t->AsRecordVal(), TYPE_TABLE, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function Comm::table_iterator_last%(it: opaque of Comm::TableIterator%): bool
+	%{
+	auto ti = static_cast<comm::TableIterator*>(it);
+	return new Val(ti->it == ti->dat.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function Comm::table_iterator_next%(it: opaque of Comm::TableIterator%): bool
+	%{
+	auto ti = static_cast<comm::TableIterator*>(it);
+
+	if ( ti->it == ti->dat.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++ti->it;
+	return new Val(ti->it != ti->dat.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function Comm::table_iterator_value%(it: opaque of Comm::TableIterator%): Comm::TableItem
+	%{
+	auto ti = static_cast<comm::TableIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::Comm::TableItem);
+	auto key_val = new RecordVal(BifType::Record::Comm::Data);
+	auto val_val = new RecordVal(BifType::Record::Comm::Data);
+	rval->Assign(0, key_val);
+	rval->Assign(1, val_val);
+
+	if ( ti->it == ti->dat.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid table iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	key_val->Assign(0, new comm::DataVal(ti->it->first));
+	val_val->Assign(0, new comm::DataVal(ti->it->second));
+	return rval;
+	%}
+
+## Create communication data of type "vector".
+function Comm::vector_create%(%): Comm::Data
+	%{
+	return comm::make_data_val(broker::vector());
+	%}
+
+## Remove all elements within a vector.
+##
+## v: the vector to clear.
+##
+## Returns: always true.
+function Comm::vector_clear%(v: Comm::Data%): bool
+	%{
+	auto& vec = comm::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	vec.clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Get the number of elements within a vector.
+##
+## v: the vector to query.
+##
+## Returns: the number of elements in the vector.
+function Comm::vector_size%(v: Comm::Data%): count
+	%{
+	auto& vec = comm::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	return new Val(static_cast<uint64_t>(vec.size()), TYPE_COUNT);
+	%}
+
+## Insert an element into a vector at a particular position, possibly displacing
+## existing elements (insertion always grows the size of the vector by one).
+##
+## v: the vector to modify.
+##
+## d: the element to insert.
+##
+## idx: the index at which to insert the data.  If it is greater than the
+##      current size of the vector, the element is inserted at the end.
+##
+## Returns: always true.
+function Comm::vector_insert%(v: Comm::Data, d: Comm::Data, idx: count%): bool
+	%{
+	auto& vec = comm::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	auto& item = comm::opaque_field_to_data(d->AsRecordVal(), frame);
+	idx = min(idx, static_cast<uint64_t>(vec.size()));
+	vec.insert(vec.begin() + idx, item);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Replace an element in a vector at a particular position.
+##
+## v: the vector to modify.
+##
+## d: the element to insert.
+##
+## idx: the index to replace.
+##
+## Returns: the value that was just evicted.  If the index was larger than any
+##          valid index, the optional field of the returned record is not set.
+function Comm::vector_replace%(v: Comm::Data, d: Comm::Data, idx: count%): Comm::Data
+	%{
+	auto& vec = comm::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+	auto& item = comm::opaque_field_to_data(d->AsRecordVal(), frame);
+
+	if ( idx >= vec.size() )
+		return new RecordVal(BifType::Record::Comm::Data);
+
+	auto rval = comm::make_data_val(move(vec[idx]));
+	vec[idx] = item;
+	return rval;
+	%}
+
+## Remove an element from a vector at a particular position.
+##
+## v: the vector to modify.
+##
+## idx: the index to remove.
+##
+## Returns: the value that was just evicted.  If the index was larger than any
+##          valid index, the optional field of the returned record is not set.
+function Comm::vector_remove%(v: Comm::Data, idx: count%): Comm::Data
+	%{
+	auto& vec = comm::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+
+	if ( idx >= vec.size() )
+		return new RecordVal(BifType::Record::Comm::Data);
+
+	auto rval = comm::make_data_val(move(vec[idx]));
+	vec.erase(vec.begin() + idx);
+	return rval;
+	%}
+
+## Lookup an element in a vector at a particular position.
+##
+## v: the vector to query.
+##
+## idx: the index to lookup.
+##
+## Returns: the value at the index.  If the index was larger than any
+##          valid index, the optional field of the returned record is not set.
+function Comm::vector_lookup%(v: Comm::Data, idx: count%): Comm::Data
+	%{
+	auto& vec = comm::require_data_type<broker::vector>(v->AsRecordVal(),
+													    TYPE_VECTOR, frame);
+
+	if ( idx >= vec.size() )
+		return new RecordVal(BifType::Record::Comm::Data);
+
+	return comm::make_data_val(vec[idx]);
+	%}
+
+## Create an iterator for a vector.  Note that this makes a copy of the vector
+## internally to ensure the iterator is always valid.
+##
+## v: the vector to iterate over.
+##
+## Returns: an iterator.
+function Comm::vector_iterator%(v: Comm::Data%): opaque of Comm::VectorIterator
+	%{
+	return new comm::VectorIterator(v->AsRecordVal(), TYPE_VECTOR, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function Comm::vector_iterator_last%(it: opaque of Comm::VectorIterator%): bool
+	%{
+	auto vi = static_cast<comm::VectorIterator*>(it);
+	return new Val(vi->it == vi->dat.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function Comm::vector_iterator_next%(it: opaque of Comm::VectorIterator%): bool
+	%{
+	auto vi = static_cast<comm::VectorIterator*>(it);
+
+	if ( vi->it == vi->dat.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++vi->it;
+	return new Val(vi->it != vi->dat.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function Comm::vector_iterator_value%(it: opaque of Comm::VectorIterator%): Comm::Data
+	%{
+	auto vi = static_cast<comm::VectorIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::Comm::Data);
+
+	if ( vi->it == vi->dat.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid vector iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	rval->Assign(0, new comm::DataVal(*vi->it));
+	return rval;
+	%}
+
+## Create communication data of type "record".
+##
+## sz: the number of fields in the record.
+##
+## Returns: record data, with all fields uninitialized.
+function Comm::record_create%(sz: count%): Comm::Data
+	%{
+	return comm::make_data_val(broker::record(std::vector<broker::record::field>(sz)));
+	%}
+
+## Get the number of fields within a record.
+##
+## r: the record to query.
+##
+## Returns: the number of fields in the record.
+function Comm::record_size%(r: Comm::Data%): count
+	%{
+	auto& v = comm::require_data_type<broker::record>(r->AsRecordVal(),
+													  TYPE_RECORD, frame);
+	return new Val(static_cast<uint64_t>(v.fields.size()), TYPE_COUNT);
+	%}
+
+## Replace a field in a record at a particular position.
+##
+## t: the table to modify.
+##
+## d: the new field value to assign.
+##
+## idx: the index to replace.
+##
+## Returns: false if the index was larger than any valid index, else true.
+function Comm::record_assign%(r: Comm::Data, d: Comm::Data, idx: count%): bool
+	%{
+	auto& v = comm::require_data_type<broker::record>(r->AsRecordVal(),
+													  TYPE_RECORD, frame);
+	auto& item = comm::opaque_field_to_data(d->AsRecordVal(), frame);
+
+	if ( idx >= v.fields.size() )
+		return new Val(false, TYPE_BOOL);
+
+	v.fields[idx] = item;
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Lookup a field in a record at a particular position.
+##
+## r: the record to query.
+##
+## idx: the index to lookup.
+##
+## Returns: the value at the index.  The optional field of the returned record
+##          may not be set if the field of the record has no value or if the
+##          the index was not valid.
+function Comm::record_lookup%(r: Comm::Data, idx: count%): Comm::Data
+	%{
+	auto& v = comm::require_data_type<broker::record>(r->AsRecordVal(),
+													  TYPE_RECORD, frame);
+
+	if ( idx >= v.size() )
+		return new RecordVal(BifType::Record::Comm::Data);
+
+	if ( ! v.fields[idx] )
+		return new RecordVal(BifType::Record::Comm::Data);
+
+	return comm::make_data_val(*v.fields[idx]);
+	%}
+
+## Create an iterator for a record.  Note that this makes a copy of the record
+## internally to ensure the iterator is always valid.
+##
+## r: the record to iterate over.
+##
+## Returns: an iterator.
+function Comm::record_iterator%(r: Comm::Data%): opaque of Comm::RecordIterator
+	%{
+	return new comm::RecordIterator(r->AsRecordVal(), TYPE_RECORD, frame);
+	%}
+
+## Check if there are no more elements to iterate over.
+##
+## it: an iterator.
+##
+## Returns: true if there are no more elements to iterator over, i.e.
+##          the iterator is one-past-the-final-element.
+function Comm::record_iterator_last%(it: opaque of Comm::RecordIterator%): bool
+	%{
+	auto ri = static_cast<comm::RecordIterator*>(it);
+	return new Val(ri->it == ri->dat.fields.end(), TYPE_BOOL);
+	%}
+
+## Advance an iterator.
+##
+## it: an iterator.
+##
+## Returns: true if the iterator, after advancing, still references an element
+##          in the collection.  False if the iterator, after advancing, is
+##          one-past-the-final-element.
+function Comm::record_iterator_next%(it: opaque of Comm::RecordIterator%): bool
+	%{
+	auto ri = static_cast<comm::RecordIterator*>(it);
+
+	if ( ri->it == ri->dat.fields.end() )
+		return new Val(false, TYPE_BOOL);
+
+	++ri->it;
+	return new Val(ri->it != ri->dat.fields.end(), TYPE_BOOL);
+	%}
+
+## Retrieve the data at an iterator's current position.
+##
+## it: an iterator.
+##
+## Returns: element in the collection that the iterator currently references.
+function Comm::record_iterator_value%(it: opaque of Comm::RecordIterator%): Comm::Data
+	%{
+	auto ri = static_cast<comm::RecordIterator*>(it);
+	auto rval = new RecordVal(BifType::Record::Comm::Data);
+
+	if ( ri->it == ri->dat.fields.end() )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Warning("attempt to retrieve value of invalid record iterator");
+		reporter->PopLocation();
+		return rval;
+		}
+
+	if ( ! *ri->it )
+		return rval; // field isn't set
+
+	rval->Assign(0, new comm::DataVal(**ri->it));
+	return rval;
+	%}

src/comm/messaging.bif

@@ -0,0 +1,211 @@
+
+##! Functions for peering and various messaging patterns (e.g. print/log/event).
+
+%%{
+#include "comm/Manager.h"
+#include "logging/Manager.h"
+%%}
+
+module Comm;
+
+type Comm::SendFlags: record;
+
+type Comm::EventArgs: record;
+
+## Used to handle remote print messages from peers that call
+## :bro:see:`Comm::print`.
+event Comm::print_handler%(msg: string%);
+
+## Print a simple message to any interested peers.  The receiver can use
+## :bro:see:`Comm::print_handler` to handle messages.
+##
+## topic: a topic associated with the printed message.
+##
+## msg: the print message to send to peers.
+##
+## flags: tune the behavior of how the message is sent.
+##
+## Returns: true if the message is sent.
+function Comm::print%(topic: string, msg: string,
+                      flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = comm_mgr->Print(topic->CheckString(), msg->CheckString(),
+	                            flags);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Register interest in all peer print messages that use a certain topic prefix.
+## use :bro:see:`Comm::print_handler` to handle received messages.
+##
+## topic_prefix: a prefix to match against remote message topics.
+##               e.g. an empty prefix matches everything and "a" matches
+##               "alice" and "amy" but not "bob".
+##
+## Returns: true if it's a new print subscription and it is now registered.
+function Comm::subscribe_to_prints%(topic_prefix: string%): bool
+	%{
+	auto rval = comm_mgr->SubscribeToPrints(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Unregister interest in all peer print messages that use a topic prefix.
+##
+## topic_prefix: a prefix previously supplied to a successful call to
+##               :bro:see:`Comm::subscribe_to_prints`.
+##
+## Returns: true if interest in the topic prefix is no longer advertised.
+function Comm::unsubscribe_to_prints%(topic_prefix: string%): bool
+	%{
+	auto rval = comm_mgr->UnsubscribeToPrints(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Create a data structure that may be used to send a remote event via
+## :bro:see:`Comm::event`.
+##
+## args: an event, followed by a list of argument values that may be used
+##       to call it.
+##
+## Returns: opaque communication data that may be used to send a remote event.
+function Comm::event_args%(...%): Comm::EventArgs
+	%{
+	auto rval = comm_mgr->MakeEventArgs(@ARGS@);
+	return rval;
+	%}
+
+## Send an event to any interested peers.
+##
+## topic: a topic associated with the event message.
+##
+## args: event arguments as made by :bro:see:`Comm::event_args`.
+##
+## flags: tune the behavior of how the message is sent.
+##
+## Returns: true if the message is sent.
+function Comm::event%(topic: string, args: Comm::EventArgs,
+                      flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = comm_mgr->Event(topic->CheckString(), args->AsRecordVal(),
+	                            flags);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Automatically send an event to any interested peers whenever it is
+## locally dispatched (e.g. using "event my_event(...);" in a script).
+##
+## topic: a topic string associated with the event message.
+##        Peers advertise interest by registering a subscription to some prefix
+##        of this topic name.
+##
+## ev: a Bro event value.
+##
+## flags: tune the behavior of how the message is send.
+##
+## Returns: true if automatic event sending is now enabled.
+function Comm::auto_event%(topic: string, ev: any,
+                           flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = comm_mgr->AutoEvent(topic->CheckString(), ev, flags);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Stop automatically sending an event to peers upon local dispatch.
+##
+## topic: a topic originally given to :bro:see:`Comm::auto_event`.
+##
+## ev: an event originally given to :bro:see:`Comm::auto_event`.
+##
+## Returns: true if automatic events will no occur for the topic/event pair.
+function Comm::auto_event_stop%(topic: string, ev: any%): bool
+	%{
+	auto rval = comm_mgr->AutoEventStop(topic->CheckString(), ev);
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Register interest in all peer event messages that use a certain topic prefix.
+##
+## topic_prefix: a prefix to match against remote message topics.
+##               e.g. an empty prefix matches everything and "a" matches
+##               "alice" and "amy" but not "bob".
+##
+## Returns: true if it's a new event subscription and it is now registered.
+function Comm::subscribe_to_events%(topic_prefix: string%): bool
+	%{
+	auto rval = comm_mgr->SubscribeToEvents(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Unregister interest in all peer event messages that use a topic prefix.
+##
+## topic_prefix: a prefix previously supplied to a successful call to
+##               :bro:see:`Comm::subscribe_to_events`.
+##
+## Returns: true if interest in the topic prefix is no longer advertised.
+function Comm::unsubscribe_to_events%(topic_prefix: string%): bool
+	%{
+	auto rval = comm_mgr->UnsubscribeToEvents(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Enable remote logs for a given log stream.
+##
+## id: the log stream to enable remote logs for.
+##
+## flags: tune the behavior of how log entry messages are sent.
+##
+## Returns: true if remote logs are enabled for the stream.
+function
+Comm::enable_remote_logs%(id: Log::ID,
+                          flags: SendFlags &default = SendFlags()%): bool
+	%{
+	auto rval = log_mgr->EnableRemoteLogs(id->AsEnumVal(),
+	                                   comm::Manager::send_flags_to_int(flags));
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Disable remote logs for a given log stream.
+##
+## id: the log stream to disable remote logs for.
+##
+## Returns: true if remote logs are disabled for the stream.
+function Comm::disable_remote_logs%(id: Log::ID%): bool
+	%{
+	auto rval = log_mgr->DisableRemoteLogs(id->AsEnumVal());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Returns: true if remote logs are enabled for the given stream.
+function Comm::remote_logs_enabled%(id: Log::ID%): bool
+	%{
+	auto rval = log_mgr->RemoteLogsAreEnabled(id->AsEnumVal());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Register interest in all peer log messages that use a certain topic prefix.
+## Logs are implicitly sent with topic "bro/log/<stream-name>" and the
+## receiving side processes them through the logging framework as usual.
+##
+## topic_prefix: a prefix to match against remote message topics.
+##               e.g. an empty prefix matches everything and "a" matches
+##               "alice" and "amy" but not "bob".
+##
+## Returns: true if it's a new log subscription and it is now registered.
+function Comm::subscribe_to_logs%(topic_prefix: string%): bool
+	%{
+	auto rval = comm_mgr->SubscribeToLogs(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}
+
+## Unregister interest in all peer log messages that use a topic prefix.
+## Logs are implicitly sent with topic "bro/log/<stream-name>" and the
+## receiving side processes them through the logging framework as usual.
+##
+## topic_prefix: a prefix previously supplied to a successful call to
+##               :bro:see:`Comm::subscribe_to_logs`.
+##
+## Returns: true if interest in the topic prefix is no longer advertised.
+function Comm::unsubscribe_to_logs%(topic_prefix: string%): bool
+	%{
+	auto rval = comm_mgr->UnsubscribeToLogs(topic_prefix->CheckString());
+	return new Val(rval, TYPE_BOOL);
+	%}

src/comm/store.bif

@@ -0,0 +1,590 @@
+
+##! Functions to interface with broker's distributed data store.
+
+%%{
+#include "comm/Manager.h"
+#include "comm/Store.h"
+#include "comm/Data.h"
+#include "Trigger.h"
+%%}
+
+module Store;
+
+type Store::ExpiryTime: record;
+
+type Store::QueryResult: record;
+
+type Store::BackendOptions: record;
+
+## Enumerates the possible storage backends.
+enum BackendType %{
+	MEMORY,
+	SQLITE,
+	ROCKSDB,
+%}
+
+## Create a master data store which contains key-value pairs.
+##
+## id: a unique name for the data store.
+##
+## b: the storage backend to use.
+##
+## options: tunes how some storage backends operate.
+##
+## Returns: a handle to the data store.
+function Store::create_master%(id: string, b: BackendType &default = MEMORY,
+                               options: BackendOptions &default = BackendOptions()%): opaque of Store::Handle
+	%{
+	auto id_str = id->CheckString();
+	auto type = comm::StoreType::MASTER;
+	auto rval = comm_mgr->LookupStore(id_str, type);
+
+	if ( rval )
+		{
+		Ref(rval);
+		return rval;
+		}
+
+	rval = new comm::StoreHandleVal(id_str, type,
+	                                static_cast<BifEnum::Store::BackendType>(b->AsEnum()),
+	                                options->AsRecordVal());
+	assert(comm_mgr->AddStore(rval));
+	return rval;
+	%}
+
+## Create a clone of a master data store which may live with a remote peer.
+## A clone automatically synchronizes to the master by automatically receiving
+## modifications and applying them locally.  Direct modifications are not
+## possible, they must be sent through the master store, which then
+## automatically broadcasts the changes out to clones.  But queries may be made
+## directly against the local cloned copy, which may be resolved quicker than
+## reaching out to a remote master store.
+##
+## id: the unique name which identifies the master data store.
+##
+## b: the storage backend to use.
+##
+## options: tunes how some storage backends operate.
+##
+## resync: the interval at which to re-attempt synchronizing with the master
+##         store should the connection be lost.  If the clone has not yet
+##         synchronized for the first time, updates and queries queue up until
+##         the synchronization completes.  After, if the connection to the
+##         master store is lost, queries continue to use the clone's version,
+##         but updates will be lost until the master is once again available.
+##
+## Returns: a handle to the data store.
+function Store::create_clone%(id: string, b: BackendType &default = MEMORY,
+                              options: BackendOptions &default = BackendOptions(),
+                              resync: interval &default = 1sec%): opaque of Store::Handle
+	%{
+	auto id_str = id->CheckString();
+	auto type = comm::StoreType::CLONE;
+	auto rval = comm_mgr->LookupStore(id_str, type);
+
+	if ( rval )
+		{
+		Ref(rval);
+		return rval;
+		}
+
+	rval = new comm::StoreHandleVal(id_str, type,
+	                                static_cast<BifEnum::Store::BackendType>(b->AsEnum()),
+	                                options->AsRecordVal(),
+	                                std::chrono::duration<double>(resync));
+	assert(comm_mgr->AddStore(rval));
+	return rval;
+	%}
+
+## Create a frontend interface to an existing master data store that allows
+## querying and updating its contents.
+##
+## id: the unique name which identifies the master data store.
+##
+## Returns: a handle to the data store.
+function Store::create_frontend%(id: string%): opaque of Store::Handle
+	%{
+	auto id_str = id->CheckString();
+	auto type = comm::StoreType::FRONTEND;
+	auto rval = comm_mgr->LookupStore(id_str, type);
+
+	if ( rval )
+		{
+		Ref(rval);
+		return rval;
+		}
+
+	rval = new comm::StoreHandleVal(id_str, type, {}, nullptr);
+	assert(comm_mgr->AddStore(rval));
+	return rval;
+	%}
+
+## Close a data store.
+##
+## h: a data store handle.
+##
+## Returns: true if store was valid and is now closed.  The handle can no
+##          longer be used for data store operations.
+function Store::close_by_handle%(h: opaque of Store::Handle%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	return new Val(comm_mgr->CloseStore(handle->store->id(),
+										handle->store_type), TYPE_BOOL);
+	%}
+
+###########################
+# non-blocking update API #
+###########################
+
+## Insert a key-value pair in to the store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key to insert.
+##
+## v: the value to insert.
+##
+## e: the expiration time of the key-value pair.
+##
+## Returns: false if the store handle was not valid.
+function Store::insert%(h: opaque of Store::Handle,
+                        k: Comm::Data, v: Comm::Data,
+                        e: Store::ExpiryTime &default = Store::ExpiryTime()%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	auto& val = comm::opaque_field_to_data(v->AsRecordVal(), frame);
+
+	using broker::store::expiration_time;
+	broker::util::optional<expiration_time> expiry;
+
+	auto abs_expiry_val = e->AsRecordVal()->Lookup(0);
+
+	if ( abs_expiry_val )
+		{
+		expiry = expiration_time(abs_expiry_val->AsTime());
+		handle->store->insert(key, val, expiry);
+		return new Val(true, TYPE_BOOL);
+		}
+
+	auto rel_expiry_val = e->AsRecordVal()->Lookup(1);
+
+	if ( rel_expiry_val )
+		{
+		auto ct = broker::time_point::now().value;
+		expiry = expiration_time(rel_expiry_val->AsInterval(), ct);
+		handle->store->insert(key, val, expiry);
+		return new Val(true, TYPE_BOOL);
+		}
+
+	handle->store->insert(key, val, expiry);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Remove a key-value pair from the store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key to remove.
+##
+## Returns: false if the store handle was not valid.
+function Store::erase%(h: opaque of Store::Handle, k: Comm::Data%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	handle->store->erase(key);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Remove all key-value pairs from the store.
+##
+## h: the handle of the store to modify.
+##
+## Returns: false if the store handle was not valid.
+function Store::clear%(h: opaque of Store::Handle%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	handle->store->clear();
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Increment an integer value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## by: the amount to increment the value by.  A non-existent key will first
+##     create it with an implicit value of zero before incrementing.
+##
+## Returns: false if the store handle was not valid.
+function Store::increment%(h: opaque of Store::Handle,
+                           k: Comm::Data, by: int &default = +1%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	handle->store->increment(key, by);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Decrement an integer value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## by: the amount to decrement the value by.  A non-existent key will first
+##     create it with an implicit value of zero before decrementing.
+##
+## Returns: false if the store handle was not valid.
+function Store::decrement%(h: opaque of Store::Handle,
+                           k: Comm::Data, by: int &default = +1%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	handle->store->decrement(key, by);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Add an element to a set value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## element: the element to add to the set.  A non-existent key will first
+##          create it with an implicit empty set value before modifying.
+##
+## Returns: false if the store handle was not valid.
+function Store::add_to_set%(h: opaque of Store::Handle,
+                            k: Comm::Data, element: Comm::Data%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	auto& ele = comm::opaque_field_to_data(element->AsRecordVal(), frame);
+	handle->store->add_to_set(key, ele);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Remove an element from a set value in a data store.
+##
+## h: the handle of the store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## element: the element to remove from the set.  A non-existent key will
+##          implicitly create an empty set value associated with the key.
+##
+## Returns: false if the store handle was not valid.
+function Store::remove_from_set%(h: opaque of Store::Handle,
+                                 k: Comm::Data, element: Comm::Data%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	auto& ele = comm::opaque_field_to_data(element->AsRecordVal(), frame);
+	handle->store->remove_from_set(key, ele);
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Add a new item to the head of a vector value in a data store.
+##
+## h: the handle of store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## item: the element to insert in to the vector.  A non-existent key will first
+##       create empty vector value before modifying.
+##
+## Returns: the handle of store to modify.
+function Store::push_left%(h: opaque of Store::Handle, k: Comm::Data,
+                           items: Comm::DataVector%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	broker::vector items_vector;
+	auto items_vv = items->AsVector();
+
+	for ( auto i = 0u; i < items_vv->size(); ++i )
+		{
+		auto& item = comm::opaque_field_to_data((*items_vv)[i]->AsRecordVal(),
+		                                        frame);
+		items_vector.emplace_back(item);
+		}
+
+	handle->store->push_left(key, move(items_vector));
+	return new Val(true, TYPE_BOOL);
+	%}
+
+## Add a new item to the tail of a vector value in a data store.
+##
+## h: the handle of store to modify.
+##
+## k: the key whose associated value is to be modified.
+##
+## item: the element to insert in to the vector.  A non-existent key will first
+##       create empty vector value before modifying.
+##
+## Returns: the handle of store to modify.
+function Store::push_right%(h: opaque of Store::Handle, k: Comm::Data,
+                            items: Comm::DataVector%): bool
+	%{
+	auto handle = static_cast<comm::StoreHandleVal*>(h);
+
+	if ( ! handle->store )
+		return new Val(false, TYPE_BOOL);
+
+	auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame);
+	broker::vector items_vector;
+	auto items_vv = items->AsVector();
+
+	for ( auto i = 0u; i < items_vv->size(); ++i )
+		{
+		auto& item = comm::opaque_field_to_data((*items_vv)[i]->AsRecordVal(),
+		                                        frame);
+		items_vector.emplace_back(item);
+		}
+
+	handle->store->push_right(key, move(items_vector));
+	return new Val(true, TYPE_BOOL);
+	%}
+
+##########################
+# non-blocking query API #
+##########################
+
+%%{
+static bool prepare_for_query(Val* opaque, Frame* frame,
+			      comm::StoreHandleVal** handle,
+			      double* timeout,
+			      comm::StoreQueryCallback** cb)
+	{
+	*handle = static_cast<comm::StoreHandleVal*>(opaque);
+
+	if ( ! (*handle)->store )
+		return false;
+
+	Trigger* trigger = frame->GetTrigger();
+
+	if ( ! trigger )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Error("Store queries can only be called inside when-condition");
+		reporter->PopLocation();
+		return false;
+		}
+
+	*timeout = trigger->TimeoutValue();
+
+	if ( *timeout < 0 )
+		{
+		reporter->PushLocation(frame->GetCall()->GetLocationInfo());
+		reporter->Error("Store queries must specify a timeout block");
+		reporter->PopLocation();
+		return false;
+		}
+
+	frame->SetDelayed();
+	trigger->Hold();
+	*cb = new comm::StoreQueryCallback(trigger, frame->GetCall(),
+					   (*handle)->store->id(),
+					   (*handle)->store_type);
+	comm_mgr->TrackStoreQuery(*cb);
+	return true;
+	}
+
+%%}
+
+## Pop the head of a data store vector value.
+##
+## h: the handle of the store to query.
+##
+## k: the key associated with the vector to modify.
+##
+## Returns: the result of the query.
+function Store::pop_left%(h: opaque of Store::Handle,
+                          k: Comm::Data%): Store::QueryResult
+	%{
+	if ( ! comm_mgr->Enabled() )
+		return comm::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return comm::query_result();
+
+	double timeout;
+	comm::StoreQueryCallback* cb;
+	comm::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return comm::query_result();
+
+	handle->store->pop_left(static_cast<comm::DataVal*>(key)->data,
+	                         std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Pop the tail of a data store vector value.
+##
+## h: the handle of the store to query.
+##
+## k: the key associated with the vector to modify.
+##
+## Returns: the result of the query.
+function Store::pop_right%(h: opaque of Store::Handle,
+                           k: Comm::Data%): Store::QueryResult
+	%{
+	if ( ! comm_mgr->Enabled() )
+		return comm::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return comm::query_result();
+
+	double timeout;
+	comm::StoreQueryCallback* cb;
+	comm::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return comm::query_result();
+
+	handle->store->pop_right(static_cast<comm::DataVal*>(key)->data,
+	                         std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Lookup the value associated with a key in a data store.
+##
+## h: the handle of the store to query.
+##
+## k: the key to lookup.
+##
+## Returns: the result of the query.
+function Store::lookup%(h: opaque of Store::Handle,
+                       k: Comm::Data%): Store::QueryResult
+	%{
+	if ( ! comm_mgr->Enabled() )
+		return comm::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return comm::query_result();
+
+	double timeout;
+	comm::StoreQueryCallback* cb;
+	comm::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return comm::query_result();
+
+	handle->store->lookup(static_cast<comm::DataVal*>(key)->data,
+	                      std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Check if a data store contains a given key.
+##
+## h: the handle of the store to query.
+##
+## k: the key to check for existence.
+##
+## Returns: the result of the query (uses :bro:see:`Comm::BOOL`).
+function Store::exists%(h: opaque of Store::Handle,
+                        k: Comm::Data%): Store::QueryResult
+	%{
+	if ( ! comm_mgr->Enabled() )
+		return comm::query_result();
+
+	Val* key = k->AsRecordVal()->Lookup(0);
+
+	if ( ! key )
+		return comm::query_result();
+
+	double timeout;
+	comm::StoreQueryCallback* cb;
+	comm::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return comm::query_result();
+
+	handle->store->exists(static_cast<comm::DataVal*>(key)->data,
+	                      std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Retrieve all keys in a data store.
+##
+## h: the handle of the store to query.
+##
+## Returns: the result of the query (uses :bro:see:`Comm::VECTOR`).
+function Store::keys%(h: opaque of Store::Handle%): Store::QueryResult
+	%{
+	double timeout;
+	comm::StoreQueryCallback* cb;
+	comm::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return comm::query_result();
+
+	handle->store->keys(std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}
+
+## Get the number of key-value pairs in a data store.
+##
+## h: the handle of the store to query.
+##
+## Returns: the result of the query (uses :bro:see:`Comm::COUNT`).
+function Store::size%(h: opaque of Store::Handle%): Store::QueryResult
+	%{
+	if ( ! comm_mgr->Enabled() )
+		return comm::query_result();
+
+	double timeout;
+	comm::StoreQueryCallback* cb;
+	comm::StoreHandleVal* handle;
+
+	if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) )
+		return comm::query_result();
+
+	handle->store->size(std::chrono::duration<double>(timeout), cb);
+	return 0;
+	%}

src/logging/Manager.cc

@@ -16,6 +16,10 @@
 #include "WriterBackend.h"
 #include "logging.bif.h"
 
+#ifdef ENABLE_BROKER
+#include "comm/Manager.h"
+#endif
+
 using namespace logging;
 
 struct Manager::Filter {
@@ -69,6 +73,11 @@ struct Manager::Stream {
 
 	WriterMap writers;	// Writers indexed by id/path pair.
 
+#ifdef ENABLE_BROKER
+	bool enable_remote;
+	int remote_flags;
+#endif
+
 	~Stream();
 	};
 
@@ -287,6 +296,11 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval)
 	streams[idx]->event = event ? event_registry->Lookup(event->Name()) : 0;
 	streams[idx]->columns = columns->Ref()->AsRecordType();
 
+#ifdef ENABLE_BROKER
+	streams[idx]->enable_remote = internal_val("Log::enable_remote_logging")->AsBool();
+	streams[idx]->remote_flags = broker::PEERS;
+#endif
+
 	DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s",
 		streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : "<none>");
 
@@ -828,6 +842,11 @@ bool Manager::Write(EnumVal* id, RecordVal* columns)
 #endif
 		}
 
+#ifdef ENABLE_BROKER
+	if ( stream->enable_remote )
+		comm_mgr->Log(id, columns, stream->remote_flags);
+#endif
+
 	Unref(columns);
 
 	if ( error )
@@ -1206,6 +1225,53 @@ void Manager::Terminate()
 		}
 	}
 
+#ifdef ENABLE_BROKER
+
+bool Manager::EnableRemoteLogs(EnumVal* stream_id, int flags)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return false;
+
+	stream->enable_remote = true;
+	stream->remote_flags = flags;
+	return true;
+	}
+
+bool Manager::DisableRemoteLogs(EnumVal* stream_id)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return false;
+
+	stream->enable_remote = false;
+	return true;
+	}
+
+bool Manager::RemoteLogsAreEnabled(EnumVal* stream_id)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return false;
+
+	return stream->enable_remote;
+	}
+
+RecordType* Manager::StreamColumns(EnumVal* stream_id)
+	{
+	auto stream = FindStream(stream_id);
+
+	if ( ! stream )
+		return nullptr;
+
+	return stream->columns;
+	}
+
+#endif
+
 // Timer which on dispatching rotates the filter.
 class RotationTimer : public Timer {
 public:

src/logging/Manager.h

@@ -157,6 +157,34 @@ public:
 	 */
 	void Terminate();
 
+#ifdef ENABLE_BROKER
+	/**
+	 * Enable remote logs for a given stream.
+	 * @param stream_id the stream to enable remote logs for.
+	 * @param flags tune behavior of how log entries are sent to peer endpoints.
+	 * @return true if remote logs are enabled.
+	 */
+	bool EnableRemoteLogs(EnumVal* stream_id, int flags);
+
+	/**
+	 * Disable remote logs for a given stream.
+	 * @param stream_id the stream to disable remote logs for.
+	 * @return true if remote logs are disabled.
+	 */
+	bool DisableRemoteLogs(EnumVal* stream_id);
+
+	/**
+	 * @return true if remote logs are enabled for a given stream.
+	 */
+	bool RemoteLogsAreEnabled(EnumVal* stream_id);
+
+	/**
+	 * @return the type which corresponds to the columns in a log entry for
+	 * a given log stream.
+	 */
+	RecordType* StreamColumns(EnumVal* stream_id);
+#endif
+
 protected:
 	friend class WriterFrontend;
 	friend class RotationFinishedMessage;

src/main.cc

@@ -63,6 +63,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void);
 
 #include "3rdparty/sqlite3.h"
 
+#ifdef ENABLE_BROKER
+#include <comm/Manager.h>
+#endif
+
 Brofiler brofiler;
 
 #ifndef HAVE_STRSEP
@@ -81,9 +85,6 @@ int perftools_leaks = 0;
 int perftools_profile = 0;
 #endif
 
-const char* prog;
-char* writefile = 0;
-name_list prefixes;
 DNS_Mgr* dns_mgr;
 TimerMgr* timer_mgr;
 logging::Manager* log_mgr = 0;
@@ -94,6 +95,13 @@ analyzer::Manager* analyzer_mgr = 0;
 file_analysis::Manager* file_mgr = 0;
 broxygen::Manager* broxygen_mgr = 0;
 iosource::Manager* iosource_mgr = 0;
+#ifdef ENABLE_BROKER
+comm::Manager* comm_mgr = 0;
+#endif
+
+const char* prog;
+char* writefile = 0;
+name_list prefixes;
 Stmt* stmts;
 EventHandlerPtr net_done = 0;
 RuleMatcher* rule_matcher = 0;
@@ -851,6 +859,10 @@ int main(int argc, char** argv)
 	input_mgr = new input::Manager();
 	file_mgr = new file_analysis::Manager();
 
+#ifdef ENABLE_BROKER
+	comm_mgr = new comm::Manager();
+#endif
+
 	plugin_mgr->InitPreScript();
 	analyzer_mgr->InitPreScript();
 	file_mgr->InitPreScript();

testing/btest/Baseline/comm.clone_store/clone.clone.out

@@ -0,0 +1,5 @@
+clone keys, [status=Store::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]]
+lookup, one, [status=Store::SUCCESS, result=[d=broker::data{111}]]
+lookup, two, [status=Store::SUCCESS, result=[d=broker::data{222}]]
+lookup, myset, [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]]
+lookup, myvec, [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]]

testing/btest/Baseline/comm.connection_updates/recv.recv.out

@@ -0,0 +1,2 @@
+Comm::incoming_connection_established, connector
+Comm::incoming_connection_broken, connector

testing/btest/Baseline/comm.connection_updates/send.send.out

@@ -0,0 +1 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp, listener

testing/btest/Baseline/comm.data/out

@@ -0,0 +1,99 @@
+Comm::BOOL
+Comm::INT
+Comm::COUNT
+Comm::DOUBLE
+Comm::STRING
+Comm::ADDR
+Comm::SUBNET
+Comm::PORT
+Comm::TIME
+Comm::INTERVAL
+Comm::ENUM
+Comm::SET
+Comm::TABLE
+Comm::VECTOR
+Comm::RECORD
+***************************
+T
+F
+1
+0
+-1
+1
+0
+1.1
+-11.1
+hello
+1.2.3.4
+192.168.0.0/16
+22/tcp
+42.0
+180.0
+Comm::BOOL
+***************************
+{
+two,
+one,
+three
+}
+0
+T
+1
+T
+F
+T
+2
+T
+1
+F
+{
+bye
+}
+0
+***************************
+{
+[two] = 2,
+[one] = 1,
+[three] = 3
+}
+0
+[d=<uninitialized>]
+1
+T
+42
+F
+[d=<uninitialized>]
+2
+[d=broker::data{7}]
+2
+37
+[d=broker::data{42}]
+1
+***************************
+[zero, one, two]
+0
+T
+T
+T
+T
+[hi, salutations, hello, greetings]
+4
+[d=broker::data{hello}]
+[d=broker::data{bah}]
+[d=broker::data{hi}]
+[hi, salutations, bah, greetings]
+[d=broker::data{bah}]
+[hi, salutations, greetings]
+3
+***************************
+[a=<uninitialized>, b=bee, c=1]
+[a=test, b=bee, c=1]
+[a=test, b=testagain, c=1]
+3
+T
+T
+T
+[d=broker::data{hi}]
+[d=broker::data{hello}]
+[d=broker::data{37}]
+3

testing/btest/Baseline/comm.master_store/out

@@ -0,0 +1,14 @@
+lookup(two): [status=Store::SUCCESS, result=[d=broker::data{222}]]
+lookup(four): [status=Store::SUCCESS, result=[d=<uninitialized>]]
+lookup(myset): [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]]
+lookup(one): [status=Store::SUCCESS, result=[d=broker::data{111}]]
+lookup(myvec): [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]]
+exists(one): [status=Store::SUCCESS, result=[d=broker::data{1}]]
+exists(two): [status=Store::SUCCESS, result=[d=broker::data{0}]]
+exists(myset): [status=Store::SUCCESS, result=[d=broker::data{1}]]
+exists(four): [status=Store::SUCCESS, result=[d=broker::data{0}]]
+pop_right(myvec): [status=Store::SUCCESS, result=[d=broker::data{omega}]]
+pop_left(myvec): [status=Store::SUCCESS, result=[d=broker::data{delta}]]
+keys: [status=Store::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]]
+size: [status=Store::SUCCESS, result=[d=broker::data{3}]]
+size (after clear): [status=Store::SUCCESS, result=[d=broker::data{0}]]

testing/btest/Baseline/comm.remote_event/recv.recv.out

@@ -0,0 +1,6 @@
+got event msg, ping, 0
+got event msg, ping, 1
+got event msg, ping, 2
+got event msg, ping, 3
+got event msg, ping, 4
+got event msg, ping, 5

testing/btest/Baseline/comm.remote_event/send.send.out

@@ -0,0 +1,11 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp
+got event msg, pong, 0
+got auto event msg, ping, 0
+got event msg, pong, 1
+got auto event msg, ping, 1
+got event msg, pong, 2
+got auto event msg, ping, 2
+got event msg, pong, 3
+got auto event msg, ping, 3
+got event msg, pong, 4
+got auto event msg, ping, 4

testing/btest/Baseline/comm.remote_log/recv.recv.out

@@ -0,0 +1,6 @@
+wrote log, [msg=ping, num=0]
+wrote log, [msg=ping, num=1]
+wrote log, [msg=ping, num=2]
+wrote log, [msg=ping, num=3]
+wrote log, [msg=ping, num=4]
+wrote log, [msg=ping, num=5]

testing/btest/Baseline/comm.remote_log/recv.test.log

@@ -0,0 +1,15 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#open	2015-01-26-22-47-11
+#fields	msg	num
+#types	string	count
+ping	0
+ping	1
+ping	2
+ping	3
+ping	4
+ping	5
+#close	2015-01-26-22-47-11

testing/btest/Baseline/comm.remote_log/send.send.out

@@ -0,0 +1 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp

testing/btest/Baseline/comm.remote_log/send.test.log

@@ -0,0 +1,15 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#open	2015-01-26-22-47-11
+#fields	msg	num
+#types	string	count
+ping	0
+ping	1
+ping	2
+ping	3
+ping	4
+ping	5
+#close	2015-01-26-22-47-11

testing/btest/Baseline/comm.remote_print/recv.recv.out

@@ -0,0 +1,6 @@
+got print msg, ping 0
+got print msg, ping 1
+got print msg, ping 2
+got print msg, ping 3
+got print msg, ping 4
+got print msg, ping 5

testing/btest/Baseline/comm.remote_print/send.send.out

@@ -0,0 +1,6 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp
+got print msg, pong 0
+got print msg, pong 1
+got print msg, pong 2
+got print msg, pong 3
+got print msg, pong 4

testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out

@@ -0,0 +1,5 @@
+clone keys, [status=Store::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]]
+lookup, one, [status=Store::SUCCESS, result=[d=broker::data{111}]]
+lookup, two, [status=Store::SUCCESS, result=[d=broker::data{222}]]
+lookup, myset, [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]]
+lookup, myvec, [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]]

testing/btest/Baseline/core.leaks.comm.data/bro..stdout

@@ -0,0 +1,99 @@
+Comm::BOOL
+Comm::INT
+Comm::COUNT
+Comm::DOUBLE
+Comm::STRING
+Comm::ADDR
+Comm::SUBNET
+Comm::PORT
+Comm::TIME
+Comm::INTERVAL
+Comm::ENUM
+Comm::SET
+Comm::TABLE
+Comm::VECTOR
+Comm::RECORD
+***************************
+T
+F
+1
+0
+-1
+1
+0
+1.1
+-11.1
+hello
+1.2.3.4
+192.168.0.0/16
+22/tcp
+42.0
+180.0
+Comm::BOOL
+***************************
+{
+two,
+one,
+three
+}
+0
+T
+1
+T
+F
+T
+2
+T
+1
+F
+{
+bye
+}
+0
+***************************
+{
+[two] = 2,
+[one] = 1,
+[three] = 3
+}
+0
+[d=<uninitialized>]
+1
+T
+42
+F
+[d=<uninitialized>]
+2
+[d=broker::data{7}]
+2
+37
+[d=broker::data{42}]
+1
+***************************
+[zero, one, two]
+0
+T
+T
+T
+T
+[hi, salutations, hello, greetings]
+4
+[d=broker::data{hello}]
+[d=broker::data{bah}]
+[d=broker::data{hi}]
+[hi, salutations, bah, greetings]
+[d=broker::data{bah}]
+[hi, salutations, greetings]
+3
+***************************
+[a=<uninitialized>, b=bee, c=1]
+[a=test, b=bee, c=1]
+[a=test, b=testagain, c=1]
+3
+T
+T
+T
+[d=broker::data{hi}]
+[d=broker::data{hello}]
+[d=broker::data{37}]
+3

testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout

@@ -0,0 +1,14 @@
+lookup(two): [status=Store::SUCCESS, result=[d=broker::data{222}]]
+lookup(four): [status=Store::SUCCESS, result=[d=<uninitialized>]]
+lookup(myset): [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]]
+lookup(one): [status=Store::SUCCESS, result=[d=broker::data{111}]]
+lookup(myvec): [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]]
+exists(one): [status=Store::SUCCESS, result=[d=broker::data{1}]]
+exists(two): [status=Store::SUCCESS, result=[d=broker::data{0}]]
+exists(myset): [status=Store::SUCCESS, result=[d=broker::data{1}]]
+exists(four): [status=Store::SUCCESS, result=[d=broker::data{0}]]
+pop_right(myvec): [status=Store::SUCCESS, result=[d=broker::data{omega}]]
+pop_left(myvec): [status=Store::SUCCESS, result=[d=broker::data{delta}]]
+keys: [status=Store::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]]
+size: [status=Store::SUCCESS, result=[d=broker::data{3}]]
+size (after clear): [status=Store::SUCCESS, result=[d=broker::data{0}]]

testing/btest/Baseline/core.leaks.comm.remote_event/recv.recv.out

@@ -0,0 +1,6 @@
+got event msg, ping, 0
+got event msg, ping, 1
+got event msg, ping, 2
+got event msg, ping, 3
+got event msg, ping, 4
+got event msg, ping, 5

testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out

@@ -0,0 +1,11 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp
+got event msg, pong, 0
+got auto event msg, ping, 0
+got event msg, pong, 1
+got auto event msg, ping, 1
+got event msg, pong, 2
+got auto event msg, ping, 2
+got event msg, pong, 3
+got auto event msg, ping, 3
+got event msg, pong, 4
+got auto event msg, ping, 4

testing/btest/Baseline/core.leaks.comm.remote_log/recv.recv.out

@@ -0,0 +1,6 @@
+wrote log, [msg=ping, num=0]
+wrote log, [msg=ping, num=1]
+wrote log, [msg=ping, num=2]
+wrote log, [msg=ping, num=3]
+wrote log, [msg=ping, num=4]
+wrote log, [msg=ping, num=5]

testing/btest/Baseline/core.leaks.comm.remote_log/recv.test.log

@@ -0,0 +1,15 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#open	2015-02-12-17-33-13
+#fields	msg	num
+#types	string	count
+ping	0
+ping	1
+ping	2
+ping	3
+ping	4
+ping	5
+#close	2015-02-12-17-33-14

testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out

@@ -0,0 +1 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp

testing/btest/Baseline/core.leaks.comm.remote_log/send.test.log

@@ -0,0 +1,15 @@
+#separator \x09
+#set_separator	,
+#empty_field	(empty)
+#unset_field	-
+#path	test
+#open	2015-02-12-17-33-13
+#fields	msg	num
+#types	string	count
+ping	0
+ping	1
+ping	2
+ping	3
+ping	4
+ping	5
+#close	2015-02-12-17-33-15

testing/btest/Baseline/core.leaks.comm.remote_print/recv.recv.out

@@ -0,0 +1,6 @@
+got print msg, ping 0
+got print msg, ping 1
+got print msg, ping 2
+got print msg, ping 3
+got print msg, ping 4
+got print msg, ping 5

testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out

@@ -0,0 +1,6 @@
+Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp
+got print msg, pong 0
+got print msg, pong 1
+got print msg, pong 2
+got print msg, pong 3
+got print msg, pong 4

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-10-31-20-38-14
+#open	2015-02-03-22-47-13
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -14,6 +14,8 @@ scripts/base/init-bare.bro
   build/scripts/base/bif/reporter.bif.bro
   build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro
   build/scripts/base/bif/event.bif.bro
+  scripts/base/frameworks/comm/__load__.bro
+    scripts/base/frameworks/comm/main.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
       build/scripts/base/bif/logging.bif.bro
@@ -47,6 +49,10 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
+    build/scripts/base/bif/comm.bif.bro
+    build/scripts/base/bif/data.bif.bro
+    build/scripts/base/bif/messaging.bif.bro
+    build/scripts/base/bif/store.bif.bro
   build/scripts/base/bif/plugins/__load__.bro
     build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro
@@ -115,4 +121,4 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro
 scripts/policy/misc/loaded-scripts.bro
   scripts/base/utils/paths.bro
-#close	2014-10-31-20-38-14
+#close	2015-02-03-22-47-13

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -3,7 +3,7 @@
 #empty_field	(empty)
 #unset_field	-
 #path	loaded_scripts
-#open	2014-10-31-20-38-48
+#open	2015-02-03-22-47-15
 #fields	name
 #types	string
 scripts/base/init-bare.bro
@@ -14,6 +14,8 @@ scripts/base/init-bare.bro
   build/scripts/base/bif/reporter.bif.bro
   build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro
   build/scripts/base/bif/event.bif.bro
+  scripts/base/frameworks/comm/__load__.bro
+    scripts/base/frameworks/comm/main.bro
   scripts/base/frameworks/logging/__load__.bro
     scripts/base/frameworks/logging/main.bro
       build/scripts/base/bif/logging.bif.bro
@@ -47,6 +49,10 @@ scripts/base/init-bare.bro
     build/scripts/base/bif/bloom-filter.bif.bro
     build/scripts/base/bif/cardinality-counter.bif.bro
     build/scripts/base/bif/top-k.bif.bro
+    build/scripts/base/bif/comm.bif.bro
+    build/scripts/base/bif/data.bif.bro
+    build/scripts/base/bif/messaging.bif.bro
+    build/scripts/base/bif/store.bif.bro
   build/scripts/base/bif/plugins/__load__.bro
     build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro
     build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro
@@ -247,4 +253,4 @@ scripts/base/init-default.bro
   scripts/base/misc/find-checksum-offloading.bro
   scripts/base/misc/find-filtered-trace.bro
 scripts/policy/misc/loaded-scripts.bro
-#close	2014-10-31-20-38-48
+#close	2015-02-03-22-47-15

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-connector_bro/output

@@ -0,0 +1,23 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+connecting-connector.bro
+
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+		  peer_address, peer_port, peer_name;
+	terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-listener_bro/output

@@ -0,0 +1,25 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+connecting-listener.bro
+
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event Comm::incoming_connection_broken(peer_name: string)
+	{
+	print "Comm::incoming_connection_broken", peer_name;
+	terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-connector_bro/output

@@ -0,0 +1,35 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+events-connector.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	Comm::auto_event("bro/event/my_auto_event", my_auto_event);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	Comm::event("bro/event/my_event", Comm::event_args(my_event, "hi", 0));
+	event my_auto_event("stuff", 88);
+	Comm::event("bro/event/my_event", Comm::event_args(my_event, "...", 1));
+	event my_auto_event("more stuff", 51);
+	Comm::event("bro/event/my_event", Comm::event_args(my_event, "bye", 2));
+	}
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-listener_bro/output

@@ -0,0 +1,41 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+events-listener.bro
+
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+global msg_count = 0;
+global my_event: event(msg: string, c: count);
+global my_auto_event: event(msg: string, c: count);
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_events("bro/event/");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event my_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}
+
+event my_auto_event(msg: string, c: count)
+	{
+	++msg_count;
+	print "got my_auto_event", msg, c;
+
+	if ( msg_count == 5 )
+		terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-connector_bro/output

@@ -0,0 +1,44 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+logs-connector.bro
+
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+redef Log::enable_local_logging = F;
+redef Log::enable_remote_logging = F;
+global n = 0;
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::enable_remote_logs(Test::LOG);
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event do_write()
+	{
+	if ( n == 6 )
+		return;
+
+	Log::write(Test::LOG, [$msg = "ping", $num = n]);
+	++n;
+	event do_write();
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	event do_write();
+	}
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-listener_bro/output

@@ -0,0 +1,29 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+logs-listener.bro
+
+@load ./testlog
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_logs("bro/log/Test::LOG");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event Test::log_test(rec: Test::Info)
+	{
+	print "wrote log", rec;
+
+	if ( rec$num == 5 )
+		terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-connector_bro/output

@@ -0,0 +1,30 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+printing-connector.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;
+	Comm::print("bro/print/hi", "hello");
+	Comm::print("bro/print/stuff", "...");
+	Comm::print("bro/print/bye", "goodbye");
+	}
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-listener_bro/output

@@ -0,0 +1,30 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+printing-listener.bro
+
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+global msg_count = 0;
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_prints("bro/print/");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;
+	}
+
+event Comm::print_handler(msg: string)
+	{
+	++msg_count;
+	print "got print message", msg;
+
+	if ( msg_count == 3 )
+		terminate();
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-connector_bro/output

@@ -0,0 +1,57 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+stores-connector.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of Store::Handle;
+
+function dv(d: Comm::Data): Comm::DataVector
+	{
+	local rval: Comm::DataVector;
+	rval[0] = d;
+	return rval;
+	}
+
+global ready: event();
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	local myset: set[string] = {"a", "b", "c"};
+	local myvec: vector of string = {"alpha", "beta", "gamma"};
+	h = Store::create_master("mystore");
+	Store::insert(h, Comm::data("one"), Comm::data(110));
+	Store::insert(h, Comm::data("two"), Comm::data(223));
+	Store::insert(h, Comm::data("myset"), Comm::data(myset));
+	Store::insert(h, Comm::data("myvec"), Comm::data(myvec));
+	Store::increment(h, Comm::data("one"));
+	Store::decrement(h, Comm::data("two"));
+	Store::add_to_set(h, Comm::data("myset"), Comm::data("d"));
+	Store::remove_from_set(h, Comm::data("myset"), Comm::data("b"));
+	Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta")));
+	Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega")));
+
+	when ( local res = Store::size(h) )
+		{
+		print "master size", res;
+		event ready();
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1secs);
+	Comm::auto_event("bro/event/ready", ready);
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-listener_bro/output

@@ -0,0 +1,47 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+stores-listener.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of Store::Handle;
+global expected_key_count = 4;
+global key_count = 0;
+
+function do_lookup(key: string)
+	{
+	when ( local res = Store::lookup(h, Comm::data(key)) )
+		{
+		++key_count;
+		print "lookup", key, res;
+
+		if ( key_count == expected_key_count )
+			terminate();
+		}
+	timeout 10sec
+		{ print "timeout", key; }
+	}
+
+event ready()
+	{
+	h = Store::create_clone("mystore");
+
+	when ( local res = Store::keys(h) )
+		{
+		print "clone keys", res;
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3)));
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_events("bro/event/ready");
+	Comm::listen(broker_port, "127.0.0.1");
+	}

testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_testlog_bro/output

@@ -0,0 +1,23 @@
+# @TEST-EXEC: cat %INPUT >output && btest-diff output
+
+testlog.bro
+
+
+module Test;
+
+export {
+	redef enum Log::ID += { LOG };
+
+	type Info: record {
+		msg: string &log;
+		num: count &log;
+	};
+
+	global log_test: event(rec: Test::Info);
+}
+
+event bro_init() &priority=5
+	{
+	Comm::enable();
+	Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]);
+	}

testing/btest/Baseline/plugins.hooks/output

@@ -191,7 +191,7 @@
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::__create_stream, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])) -> <null>
-0.000000   MetaHookPost  CallFunction(Log::__write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
+0.000000   MetaHookPost  CallFunction(Log::__write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, (Cluster::LOG)) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, (Communication::LOG)) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::add_default_filter, (Conn::LOG)) -> <null>
@@ -285,8 +285,8 @@
 0.000000   MetaHookPost  CallFunction(Log::create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509])) -> <null>
 0.000000   MetaHookPost  CallFunction(Log::create_stream, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])) -> <null>
-0.000000   MetaHookPost  CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
+0.000000   MetaHookPost  CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
-0.000000   MetaHookPost  CallFunction(Log::write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
+0.000000   MetaHookPost  CallFunction(Log::write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) -> <null>
 0.000000   MetaHookPost  CallFunction(Notice::want_pp, ()) -> <null>
 0.000000   MetaHookPost  CallFunction(PacketFilter::build, ()) -> <null>
 0.000000   MetaHookPost  CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> <null>
@@ -401,10 +401,12 @@
 0.000000   MetaHookPost  LoadFile(./bro.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./broxygen.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./cardinality-counter.bif.bro) -> -1
+0.000000   MetaHookPost  LoadFile(./comm.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./const.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./consts) -> -1
 0.000000   MetaHookPost  LoadFile(./consts.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./contents) -> -1
+0.000000   MetaHookPost  LoadFile(./data.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./dcc-send) -> -1
 0.000000   MetaHookPost  LoadFile(./entities) -> -1
 0.000000   MetaHookPost  LoadFile(./event.bif.bro) -> -1
@@ -425,6 +427,7 @@
 0.000000   MetaHookPost  LoadFile(./main) -> -1
 0.000000   MetaHookPost  LoadFile(./main.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./max) -> -1
+0.000000   MetaHookPost  LoadFile(./messaging.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./min) -> -1
 0.000000   MetaHookPost  LoadFile(./mozilla-ca-list) -> -1
 0.000000   MetaHookPost  LoadFile(./netstats) -> -1
@@ -440,6 +443,7 @@
 0.000000   MetaHookPost  LoadFile(./sftp) -> -1
 0.000000   MetaHookPost  LoadFile(./site) -> -1
 0.000000   MetaHookPost  LoadFile(./std-dev) -> -1
+0.000000   MetaHookPost  LoadFile(./store.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./strings.bif.bro) -> -1
 0.000000   MetaHookPost  LoadFile(./sum) -> -1
 0.000000   MetaHookPost  LoadFile(./top-k.bif.bro) -> -1
@@ -474,6 +478,7 @@
 0.000000   MetaHookPost  LoadFile(base<...>/analyzer.bif) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/bro.bif) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/cluster) -> -1
+0.000000   MetaHookPost  LoadFile(base<...>/comm) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/communication) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/conn) -> -1
 0.000000   MetaHookPost  LoadFile(base<...>/conn-ids) -> -1
@@ -730,7 +735,7 @@
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509]))
 0.000000   MetaHookPre   CallFunction(Log::__create_stream, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql]))
-0.000000   MetaHookPre   CallFunction(Log::__write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::__write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, (Cluster::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, (Communication::LOG))
 0.000000   MetaHookPre   CallFunction(Log::add_default_filter, (Conn::LOG))
@@ -824,8 +829,8 @@
 0.000000   MetaHookPre   CallFunction(Log::create_stream, (Weird::LOG, [columns=<no value description>, ev=Weird::log_weird]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, (X509::LOG, [columns=<no value description>, ev=X509::log_x509]))
 0.000000   MetaHookPre   CallFunction(Log::create_stream, (mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql]))
-0.000000   MetaHookPre   CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T]))
-0.000000   MetaHookPre   CallFunction(Log::write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T]))
+0.000000   MetaHookPre   CallFunction(Log::write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T]))
 0.000000   MetaHookPre   CallFunction(Notice::want_pp, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::build, ())
 0.000000   MetaHookPre   CallFunction(PacketFilter::combine_filters, (ip or not ip, and, ))
@@ -940,10 +945,12 @@
 0.000000   MetaHookPre   LoadFile(./bro.bif.bro)
 0.000000   MetaHookPre   LoadFile(./broxygen.bif.bro)
 0.000000   MetaHookPre   LoadFile(./cardinality-counter.bif.bro)
+0.000000   MetaHookPre   LoadFile(./comm.bif.bro)
 0.000000   MetaHookPre   LoadFile(./const.bif.bro)
 0.000000   MetaHookPre   LoadFile(./consts)
 0.000000   MetaHookPre   LoadFile(./consts.bro)
 0.000000   MetaHookPre   LoadFile(./contents)
+0.000000   MetaHookPre   LoadFile(./data.bif.bro)
 0.000000   MetaHookPre   LoadFile(./dcc-send)
 0.000000   MetaHookPre   LoadFile(./entities)
 0.000000   MetaHookPre   LoadFile(./event.bif.bro)
@@ -964,6 +971,7 @@
 0.000000   MetaHookPre   LoadFile(./main)
 0.000000   MetaHookPre   LoadFile(./main.bro)
 0.000000   MetaHookPre   LoadFile(./max)
+0.000000   MetaHookPre   LoadFile(./messaging.bif.bro)
 0.000000   MetaHookPre   LoadFile(./min)
 0.000000   MetaHookPre   LoadFile(./mozilla-ca-list)
 0.000000   MetaHookPre   LoadFile(./netstats)
@@ -979,6 +987,7 @@
 0.000000   MetaHookPre   LoadFile(./sftp)
 0.000000   MetaHookPre   LoadFile(./site)
 0.000000   MetaHookPre   LoadFile(./std-dev)
+0.000000   MetaHookPre   LoadFile(./store.bif.bro)
 0.000000   MetaHookPre   LoadFile(./strings.bif.bro)
 0.000000   MetaHookPre   LoadFile(./sum)
 0.000000   MetaHookPre   LoadFile(./top-k.bif.bro)
@@ -1013,6 +1022,7 @@
 0.000000   MetaHookPre   LoadFile(base<...>/analyzer.bif)
 0.000000   MetaHookPre   LoadFile(base<...>/bro.bif)
 0.000000   MetaHookPre   LoadFile(base<...>/cluster)
+0.000000   MetaHookPre   LoadFile(base<...>/comm)
 0.000000   MetaHookPre   LoadFile(base<...>/communication)
 0.000000   MetaHookPre   LoadFile(base<...>/conn)
 0.000000   MetaHookPre   LoadFile(base<...>/conn-ids)
@@ -1269,7 +1279,7 @@
 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])
 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509])
 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])
-0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG)
 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG)
@@ -1363,8 +1373,8 @@
 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=<no value description>, ev=Weird::log_weird])
 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=<no value description>, ev=X509::log_x509])
 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=<no value description>, ev=MySQL::log_mysql])
-0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])
-0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])
+0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])
 0.000000 | HookCallFunction Notice::want_pp()
 0.000000 | HookCallFunction PacketFilter::build()
 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, )

testing/btest/btest.cfg

@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts istate coverage signatures plugins
+TestDirs    = doc bifs language core scripts istate coverage signatures plugins comm
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
@@ -25,3 +25,4 @@ TMPDIR=%(testbase)s/.tmp
 BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX
 BTEST_RST_FILTER=$SCRIPTS/rst-filter
 BRO_DNS_FAKE=1
+BROKER_PORT=9999/tcp

testing/btest/comm/clone_store.bro

@@ -0,0 +1,112 @@
+# @TEST-SERIALIZE: brokercomm
+# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt
+
+# @TEST-EXEC: btest-bg-run clone "bro -b ../clone.bro broker_port=$BROKER_PORT >clone.out"
+# @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out"
+
+# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone/clone.out
+# @TEST-EXEC: btest-diff master/master.out
+
+@TEST-START-FILE clone.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of Store::Handle;
+global expected_key_count = 4;
+global key_count = 0;
+
+function do_lookup(key: string)
+	{
+	when ( local res = Store::lookup(h, Comm::data(key)) )
+		{
+		++key_count;
+		print "lookup", key, res;
+
+		if ( key_count == expected_key_count )
+			terminate();
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event ready()
+	{
+	h = Store::create_clone("mystore");
+
+	when ( local res = Store::keys(h) )
+		{
+		print "clone keys", res;
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2)));
+		do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3)));
+		}
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::subscribe_to_events("bro/event/ready");
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+@TEST-END-FILE
+
+@TEST-START-FILE master.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+
+global h: opaque of Store::Handle;
+
+function dv(d: Comm::Data): Comm::DataVector
+	{
+	local rval: Comm::DataVector;
+	rval[0] = d;
+	return rval;
+	}
+
+global ready: event();
+
+event Comm::outgoing_connection_broken(peer_address: string,
+                                       peer_port: port)
+	{
+	terminate();
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	local myset: set[string] = {"a", "b", "c"};
+	local myvec: vector of string = {"alpha", "beta", "gamma"};
+	h = Store::create_master("mystore");
+	Store::insert(h, Comm::data("one"), Comm::data(110));
+	Store::insert(h, Comm::data("two"), Comm::data(223));
+	Store::insert(h, Comm::data("myset"), Comm::data(myset));
+	Store::insert(h, Comm::data("myvec"), Comm::data(myvec));
+	Store::increment(h, Comm::data("one"));
+	Store::decrement(h, Comm::data("two"));
+	Store::add_to_set(h, Comm::data("myset"), Comm::data("d"));
+	Store::remove_from_set(h, Comm::data("myset"), Comm::data("b"));
+	Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta")));
+	Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega")));
+
+	when ( local res = Store::size(h) )
+		{ event ready(); }
+	timeout 10sec
+		{ print "timeout"; }
+	}
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1secs);
+	Comm::auto_event("bro/event/ready", ready);
+	}
+
+@TEST-END-FILE

testing/btest/comm/connection_updates.bro

@@ -0,0 +1,57 @@
+# @TEST-SERIALIZE: brokercomm
+# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt
+
+# @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out"
+# @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out"
+
+# @TEST-EXEC: btest-bg-wait 20
+# @TEST-EXEC: btest-diff recv/recv.out
+# @TEST-EXEC: btest-diff send/send.out
+
+@TEST-START-FILE recv.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "listener";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::listen(broker_port, "127.0.0.1");
+	}
+
+event Comm::incoming_connection_established(peer_name: string)
+	{
+	print "Comm::incoming_connection_established", peer_name;;
+	}
+
+event Comm::incoming_connection_broken(peer_name: string)
+	{
+	print "Comm::incoming_connection_broken", peer_name;;
+	terminate();
+	}
+
+@TEST-END-FILE
+
+@TEST-START-FILE send.bro
+
+const broker_port: port &redef;
+redef exit_only_after_terminate = T;
+redef Comm::endpoint_name = "connector";
+
+event bro_init()
+	{
+	Comm::enable();
+	Comm::connect("127.0.0.1", broker_port, 1sec);
+	}
+
+event Comm::outgoing_connection_established(peer_address: string,
+                                            peer_port: port,
+                                            peer_name: string)
+	{
+	print "Comm::outgoing_connection_established",
+	      peer_address, peer_port, peer_name;;
+	terminate();
+	}
+
+@TEST-END-FILE