diff --git a/.gitmodules b/.gitmodules index 24375ce23d..91f39e3d04 100644 --- a/.gitmodules +++ b/.gitmodules @@ -22,3 +22,6 @@ [submodule "aux/plugins"] path = aux/plugins url = git://git.bro.org/bro-plugins +[submodule "aux/broker"] + path = aux/broker + url = git://git.bro.org/broker diff --git a/CHANGES b/CHANGES index cad5e26988..6fd14a2dd7 100644 --- a/CHANGES +++ b/CHANGES @@ -1,4 +1,21 @@ +2.3-489 | 2015-03-02 15:29:30 -0800 + + * Integrate Broker, Bro's new communication library. (Jon Siwek) + + See aux/broker/README for more information on Broker, and + doc/frameworks/comm.rst for the corresponding Bro script API. + + Broker support is by default off for now; it can be enabled at + configure time with --enable-broker. It requires CAF + (https://github.com/actor-framework/actor-framework); for now + needs the "develop" branch. Broker also requires a C++11 compiler. + + Broker will become a mandatory dependency in future Bro versions. + + * Add --enable-c++11 configure flag to compile Bro's source code in + C++11 mode with a corresponding compiler. (Jon Siwek) + 2.3-451 | 2015-02-24 16:37:08 -0800 * Updating submodule(s). diff --git a/CMakeLists.txt b/CMakeLists.txt index 04ac197f74..8f60ab95ad 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -177,6 +177,17 @@ include_directories(${CMAKE_CURRENT_BINARY_DIR}) ######################################################################## ## Recurse on sub-directories +if ( ENABLE_CXX11 ) + include(RequireCXX11) +endif () + +if ( ENABLE_BROKER ) + add_subdirectory(aux/broker) + set(brodeps ${brodeps} broker) + add_definitions(-DENABLE_BROKER) + include_directories(BEFORE ${CMAKE_CURRENT_SOURCE_DIR}/aux/broker) +endif () + add_subdirectory(src) add_subdirectory(scripts) add_subdirectory(doc) @@ -224,6 +235,7 @@ message( "\nCXXFLAGS: ${CMAKE_CXX_FLAGS} ${CMAKE_CXX_FLAGS_${BuildType}}" "\nCPP: ${CMAKE_CXX_COMPILER}" "\n" + "\nBroker: ${ENABLE_BROKER}" "\nBroccoli: ${INSTALL_BROCCOLI}" "\nBroctl: ${INSTALL_BROCTL}" "\nAux. Tools: ${INSTALL_AUX_TOOLS}" diff --git a/NEWS b/NEWS index 8af6c38201..61f16d145d 100644 --- a/NEWS +++ b/NEWS @@ -39,6 +39,25 @@ New Functionality ``next`` and ``break`` can be used inside the loop's body just like with ``for`` loops. +- Bro now integrates Broker, it's new communication library. See + aux/broker/README for more information on Broker, and + doc/frameworks/comm.rst for the corresponding Bro script API. + + TODO: Extend with some more information on Broker. + + Broker support is by default off for now; it can be enabled at + configure time with --enable-broker. It requires CAF + (https://github.com/actor-framework/actor-framework) as well as a + C++11 compiler. + + TODO: Add minumim version for CAF. + + Broker will become a mandatory dependency in future Bro versions. + +- Add --enable-c++11 configure flag to compile Bro's source code in + C++11 mode with a corresponding compiler. Note that 2.4 will be the + last version of Bro that compiles without C++11 support. + Changed Functionality --------------------- diff --git a/VERSION b/VERSION index a8a700226f..3155627498 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.3-451 +2.3-489 diff --git a/aux/broker b/aux/broker new file mode 160000 index 0000000000..8f7b32e773 --- /dev/null +++ b/aux/broker @@ -0,0 +1 @@ +Subproject commit 8f7b32e773976fec513ce15237a7e0fdb05b0ebd diff --git a/cmake b/cmake index ff08be5aa1..2fd35ab6a6 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit ff08be5aa1b8eaadbe2775cbc11b499c5f93349e +Subproject commit 2fd35ab6a6245a005828c32f0aa87eb21698c054 diff --git a/configure b/configure index 2b1c568b26..3f7295711c 100755 --- a/configure +++ b/configure @@ -41,6 +41,9 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --enable-perftools-debug use Google's perftools for debugging --enable-jemalloc link against jemalloc --enable-ruby build ruby bindings for broccoli (deprecated) + --enable-c++11 build using the C++11 standard + --enable-broker enable use of the Broker communication library + (requires C++ Actor Framework and C++11) --disable-broccoli don't build or install the Broccoli library --disable-broctl don't install Broctl --disable-auxtools don't build or install auxiliary tools @@ -55,6 +58,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-flex=PATH path to flex executable --with-bison=PATH path to bison executable --with-perl=PATH path to perl executable + --with-libcaf=PATH path to C++ Actor Framework installation + (a required Broker dependency) Optional Packages in Non-Standard Locations: --with-geoip=PATH path to the libGeoIP install root @@ -67,6 +72,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-ruby-lib=PATH path to ruby library --with-ruby-inc=PATH path to ruby headers --with-swig=PATH path to SWIG executable + --with-rocksdb=PATH path to RocksDB installation + (an optional Broker dependency) Packaging Options (for developers): --binary-package toggle special logic for binary packaging @@ -176,6 +183,13 @@ while [ $# -ne 0 ]; do --enable-jemalloc) append_cache_entry ENABLE_JEMALLOC BOOL true ;; + --enable-c++11) + append_cache_entry ENABLE_CXX11 BOOL true + ;; + --enable-broker) + append_cache_entry ENABLE_CXX11 BOOL true + append_cache_entry ENABLE_BROKER BOOL true + ;; --disable-broccoli) append_cache_entry INSTALL_BROCCOLI BOOL false ;; @@ -248,6 +262,12 @@ while [ $# -ne 0 ]; do --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; + --with-libcaf=*) + append_cache_entry LIBCAF_ROOT_DIR PATH $optarg + ;; + --with-rocksdb=*) + append_cache_entry ROCKSDB_ROOT_DIR PATH $optarg + ;; --binary-package) append_cache_entry BINARY_PACKAGING_MODE BOOL true ;; diff --git a/doc/components/broker/README.rst b/doc/components/broker/README.rst new file mode 120000 index 0000000000..eafa3b8e77 --- /dev/null +++ b/doc/components/broker/README.rst @@ -0,0 +1 @@ +../../../aux/broker/README \ No newline at end of file diff --git a/doc/components/broker/broker-manual.rst b/doc/components/broker/broker-manual.rst new file mode 120000 index 0000000000..90bf8f0833 --- /dev/null +++ b/doc/components/broker/broker-manual.rst @@ -0,0 +1 @@ +../../../aux/broker/broker-manual.rst \ No newline at end of file diff --git a/doc/components/index.rst b/doc/components/index.rst index fe05f13683..c1feda4a61 100644 --- a/doc/components/index.rst +++ b/doc/components/index.rst @@ -17,6 +17,8 @@ current, independent component releases. Broccoli - User Manual Broccoli Python Bindings Broccoli Ruby Bindings + Broker - Bro's (New) Messaging Library (README) + Broker - User Manual BroControl - Interactive Bro management shell Bro-Aux - Small auxiliary tools for Bro BTest - A unit testing framework diff --git a/doc/frameworks/comm.rst b/doc/frameworks/comm.rst new file mode 100644 index 0000000000..38f1f5b644 --- /dev/null +++ b/doc/frameworks/comm.rst @@ -0,0 +1,202 @@ + +.. _comm-framework: + +====================================== +Broker-Enabled Communication Framework +====================================== + +.. rst-class:: opening + + Bro can now use the `Broker Library + <../components/broker/README.html>`_ to exchange information with + other Bro processes. To enable it run Bro's ``configure`` script + with the ``--enable-broker`` option. Note that a C++11 compatible + compiler is required as well as the `C++ Actor Framework + `_. + +.. contents:: + +Connecting to Peers +=================== + +Communication via Broker must first be turned on via +:bro:see:`Comm::enable`. + +Bro can accept incoming connections by calling :bro:see:`Comm::listen` +and then monitor connection status updates via +:bro:see:`Comm::incoming_connection_established` and +:bro:see:`Comm::incoming_connection_broken`. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/connecting-listener.bro + +Bro can initiate outgoing connections by calling :bro:see:`Comm::connect` +and then monitor connection status updates via +:bro:see:`Comm::outgoing_connection_established`, +:bro:see:`Comm::outgoing_connection_broken`, and +:bro:see:`Comm::outgoing_connection_incompatible`. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/connecting-connector.bro + +Remote Printing +=============== + +To receive remote print messages, first use +:bro:see:`Comm::subscribe_to_prints` to advertise to peers a topic +prefix of interest and then create an event handler for +:bro:see:`Comm::print_handler` to handle any print messages that are +received. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/printing-listener.bro + +To send remote print messages, just call :bro:see:`Comm::print`. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/printing-connector.bro + +Notice that the subscriber only used the prefix "bro/print/", but is +able to receive messages with full topics of "bro/print/hi", +"bro/print/stuff", and "bro/print/bye". The model here is that the +publisher of a message checks for all subscribers who advertised +interest in a prefix of that message's topic and sends it to them. + +Message Format +-------------- + +For other applications that want to exchange print messages with Bro, +the Broker message format is simply: + +.. code:: c++ + + broker::message{std::string{}}; + +Remote Events +============= + +Receiving remote events is similar to remote prints. Just use +:bro:see:`Comm::subscribe_to_events` and possibly define any new events +along with handlers that peers may want to send. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/events-listener.bro + +To send events, there are two choices. The first is to use call +:bro:see:`Comm::event` directly. The second option is to use +:bro:see:`Comm::auto_event` to make it so a particular event is +automatically sent to peers whenever it is called locally via the normal +event invocation syntax. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/events-connector.bro + +Again, the subscription model is prefix-based. + +Message Format +-------------- + +For other applications that want to exchange event messages with Bro, +the Broker message format is: + +.. code:: c++ + + broker::message{std::string{}, ...}; + +The first parameter is the name of the event and the remaining ``...`` +are its arguments, which are any of the support Broker data types as +they correspond to the Bro types for the event named in the first +parameter of the message. + +Remote Logging +============== + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/testlog.bro + +Use :bro:see:`Comm::subscribe_to_logs` to advertise interest in logs +written by peers. The topic names that Bro uses are implicitly of the +form "bro/log/". + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/logs-listener.bro + +To send remote logs either use :bro:see:`Log::enable_remote_logging` or +:bro:see:`Comm::enable_remote_logs`. The former allows any log stream +to be sent to peers while the later toggles remote logging for +particular streams. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/logs-connector.bro + +Message Format +-------------- + +For other applications that want to exchange logs messages with Bro, +the Broker message format is: + +.. code:: c++ + + broker::message{broker::enum_value{}, broker::record{}}; + +The enum value corresponds to the stream's :bro:see:`Log::ID` value, and +the record corresponds to a single entry of that log's columns record, +in this case a ``Test::INFO`` value. + +Tuning Access Control +===================== + +By default, endpoints do not restrict the message topics that it sends +to peers and do not restrict what message topics and data store +identifiers get advertised to peers. These are the default +:bro:see:`Comm::EndpointFlags` supplied to :bro:see:`Comm::enable`. + +If not using the ``auto_publish`` flag, one can use the +:bro:see:`Comm::publish_topic` and :bro:see:`Comm::unpublish_topic` +functions to manipulate the set of message topics (must match exactly) +that are allowed to be sent to peer endpoints. These settings take +precedence over the per-message ``peers`` flag supplied to functions +that take a :bro:see:`Comm::SendFlags` such as :bro:see:`Comm::print`, +:bro:see:`Comm::event`, :bro:see:`Comm::auto_event` or +:bro:see:`Comm::enable_remote_logs`. + +If not using the ``auto_advertise`` flag, one can use the +:bro:see:`Comm::advertise_topic` and :bro:see:`Comm::unadvertise_topic` +to manupulate the set of topic prefixes that are allowed to be +advertised to peers. If an endpoint does not advertise a topic prefix, +the only way a peers can send messages to it is via the ``unsolicited`` +flag of :bro:see:`Comm::SendFlags` and choosing a topic with a matching +prefix (i.e. full topic may be longer than receivers prefix, just the +prefix needs to match). + +Distributed Data Stores +======================= + +There are three flavors of key-value data store interfaces: master, +clone, and frontend. + +A frontend is the common interface to query and modify data stores. +That is, a clone is a specific type of frontend and a master is also a +specific type of frontend, but a standalone frontend can also exist to +e.g. query and modify the contents of a remote master store without +actually "owning" any of the contents itself. + +A master data store can be be cloned from remote peers which may then +perform lightweight, local queries against the clone, which +automatically stays synchronized with the master store. Clones cannot +modify their content directly, instead they send modifications to the +centralized master store which applies them and then broadcasts them to +all clones. + +Master and clone stores get to choose what type of storage backend to +use. E.g. In-memory versus SQLite for persistence. Note that if clones +are used, data store sizes should still be able to fit within memory +regardless of the storage backend as a single snapshot of the master +store is sent in a single chunk to initialize the clone. + +Data stores also support expiration on a per-key basis either using an +absolute point in time or a relative amount of time since the entry's +last modification time. + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/stores-listener.bro + +.. btest-include:: ${DOC_ROOT}/frameworks/comm/stores-connector.bro + +In the above example, if a local copy of the store contents isn't +needed, just replace the :bro:see:`Store::create_clone` call with +:bro:see:`Store::create_frontend`. Queries will then be made against +the remote master store instead of the local clone. + +Note that all queries are made within Bro's asynchrounous ``when`` +statements and must specify a timeout block. diff --git a/doc/frameworks/comm/connecting-connector.bro b/doc/frameworks/comm/connecting-connector.bro new file mode 100644 index 0000000000..d5b191ad38 --- /dev/null +++ b/doc/frameworks/comm/connecting-connector.bro @@ -0,0 +1,19 @@ + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + terminate(); + } diff --git a/doc/frameworks/comm/connecting-listener.bro b/doc/frameworks/comm/connecting-listener.bro new file mode 100644 index 0000000000..4e5c3ad86f --- /dev/null +++ b/doc/frameworks/comm/connecting-listener.bro @@ -0,0 +1,21 @@ + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Comm::incoming_connection_broken(peer_name: string) + { + print "Comm::incoming_connection_broken", peer_name; + terminate(); + } diff --git a/doc/frameworks/comm/events-connector.bro b/doc/frameworks/comm/events-connector.bro new file mode 100644 index 0000000000..28a94f356e --- /dev/null +++ b/doc/frameworks/comm/events-connector.bro @@ -0,0 +1,31 @@ +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + Comm::auto_event("bro/event/my_auto_event", my_auto_event); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + Comm::event("bro/event/my_event", Comm::event_args(my_event, "hi", 0)); + event my_auto_event("stuff", 88); + Comm::event("bro/event/my_event", Comm::event_args(my_event, "...", 1)); + event my_auto_event("more stuff", 51); + Comm::event("bro/event/my_event", Comm::event_args(my_event, "bye", 2)); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/doc/frameworks/comm/events-listener.bro b/doc/frameworks/comm/events-listener.bro new file mode 100644 index 0000000000..4d8985d09a --- /dev/null +++ b/doc/frameworks/comm/events-listener.bro @@ -0,0 +1,37 @@ + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; +global msg_count = 0; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event my_event(msg: string, c: count) + { + ++msg_count; + print "got my_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } + +event my_auto_event(msg: string, c: count) + { + ++msg_count; + print "got my_auto_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } diff --git a/doc/frameworks/comm/logs-connector.bro b/doc/frameworks/comm/logs-connector.bro new file mode 100644 index 0000000000..2c02acb188 --- /dev/null +++ b/doc/frameworks/comm/logs-connector.bro @@ -0,0 +1,40 @@ +@load ./testlog + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; +redef Log::enable_local_logging = F; +redef Log::enable_remote_logging = F; +global n = 0; + +event bro_init() + { + Comm::enable(); + Comm::enable_remote_logs(Test::LOG); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event do_write() + { + if ( n == 6 ) + return; + + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + event do_write(); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/doc/frameworks/comm/logs-listener.bro b/doc/frameworks/comm/logs-listener.bro new file mode 100644 index 0000000000..5f763be7ee --- /dev/null +++ b/doc/frameworks/comm/logs-listener.bro @@ -0,0 +1,25 @@ +@load ./testlog + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_logs("bro/log/Test::LOG"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + + if ( rec$num == 5 ) + terminate(); + } diff --git a/doc/frameworks/comm/printing-connector.bro b/doc/frameworks/comm/printing-connector.bro new file mode 100644 index 0000000000..76567dbb97 --- /dev/null +++ b/doc/frameworks/comm/printing-connector.bro @@ -0,0 +1,26 @@ +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + Comm::print("bro/print/hi", "hello"); + Comm::print("bro/print/stuff", "..."); + Comm::print("bro/print/bye", "goodbye"); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/doc/frameworks/comm/printing-listener.bro b/doc/frameworks/comm/printing-listener.bro new file mode 100644 index 0000000000..9bd3844502 --- /dev/null +++ b/doc/frameworks/comm/printing-listener.bro @@ -0,0 +1,26 @@ + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; +global msg_count = 0; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_prints("bro/print/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Comm::print_handler(msg: string) + { + ++msg_count; + print "got print message", msg; + + if ( msg_count == 3 ) + terminate(); + } diff --git a/doc/frameworks/comm/stores-connector.bro b/doc/frameworks/comm/stores-connector.bro new file mode 100644 index 0000000000..61e863e835 --- /dev/null +++ b/doc/frameworks/comm/stores-connector.bro @@ -0,0 +1,53 @@ +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + h = Store::create_master("mystore"); + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + + when ( local res = Store::size(h) ) + { + print "master size", res; + event ready(); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1secs); + Comm::auto_event("bro/event/ready", ready); + } diff --git a/doc/frameworks/comm/stores-listener.bro b/doc/frameworks/comm/stores-listener.bro new file mode 100644 index 0000000000..89384c2e9d --- /dev/null +++ b/doc/frameworks/comm/stores-listener.bro @@ -0,0 +1,43 @@ +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout", key; } + } + +event ready() + { + h = Store::create_clone("mystore"); + + when ( local res = Store::keys(h) ) + { + print "clone keys", res; + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/ready"); + Comm::listen(broker_port, "127.0.0.1"); + } diff --git a/doc/frameworks/comm/testlog.bro b/doc/frameworks/comm/testlog.bro new file mode 100644 index 0000000000..b5f449ae3d --- /dev/null +++ b/doc/frameworks/comm/testlog.bro @@ -0,0 +1,19 @@ + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event bro_init() &priority=5 + { + Comm::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); + } diff --git a/doc/frameworks/index.rst b/doc/frameworks/index.rst index f8c681d795..9819b803f0 100644 --- a/doc/frameworks/index.rst +++ b/doc/frameworks/index.rst @@ -14,4 +14,4 @@ Frameworks notice signatures sumstats - + comm diff --git a/scripts/base/frameworks/comm/__load__.bro b/scripts/base/frameworks/comm/__load__.bro new file mode 100644 index 0000000000..a10fe855df --- /dev/null +++ b/scripts/base/frameworks/comm/__load__.bro @@ -0,0 +1 @@ +@load ./main diff --git a/scripts/base/frameworks/comm/main.bro b/scripts/base/frameworks/comm/main.bro new file mode 100644 index 0000000000..d0f58f585a --- /dev/null +++ b/scripts/base/frameworks/comm/main.bro @@ -0,0 +1,103 @@ +##! Various data structure definitions for use with Bro's communication system. + +module Comm; + +export { + + ## A name used to identify this endpoint to peers. + ## .. bro:see:: Comm::connect Comm::listen + const endpoint_name = "" &redef; + + ## Change communication behavior. + type EndpointFlags: record { + ## Whether to restrict message topics that can be published to peers. + auto_publish: bool &default = T; + ## Whether to restrict what message topics or data store identifiers + ## the local endpoint advertises to peers (e.g. subscribing to + ## events or making a master data store available). + auto_advertise: bool &default = T; + }; + + ## Fine-grained tuning of communication behavior for a particular message. + type SendFlags: record { + ## Send the message to the local endpoint. + self: bool &default = F; + ## Send the message to peer endpoints that advertise interest in + ## the topic associated with the message. + peers: bool &default = T; + ## Send the message to peer endpoints even if they don't advertise + ## interest in the topic associated with the message. + unsolicited: bool &default = F; + }; + + ## Opaque communication data. + type Data: record { + d: opaque of Comm::Data &optional; + }; + + ## Opaque communication data. + type DataVector: vector of Comm::Data; + + ## Opaque event communication data. + type EventArgs: record { + ## The name of the event. Not set if invalid event or arguments. + name: string &optional; + ## The arguments to the event. + args: DataVector; + }; + + ## Opaque communication data used as a convenient way to wrap key-value + ## pairs that comprise table entries. + type Comm::TableItem : record { + key: Comm::Data; + val: Comm::Data; + }; +} + +module Store; + +export { + + ## Whether a data store query could be completed or not. + type QueryStatus: enum { + SUCCESS, + FAILURE, + }; + + ## An expiry time for a key-value pair inserted in to a data store. + type ExpiryTime: record { + ## Absolute point in time at which to expire the entry. + absolute: time &optional; + ## A point in time relative to the last modification time at which + ## to expire the entry. New modifications will delay the expiration. + since_last_modification: interval &optional; + }; + + ## The result of a data store query. + type QueryResult: record { + ## Whether the query completed or not. + status: Store::QueryStatus; + ## The result of the query. Certain queries may use a particular + ## data type (e.g. querying store size always returns a count, but + ## a lookup may return various data types). + result: Comm::Data; + }; + + ## Options to tune the SQLite storage backend. + type SQLiteOptions: record { + ## File system path of the database. + path: string &default = "store.sqlite"; + }; + + ## Options to tune the RocksDB storage backend. + type RocksDBOptions: record { + ## File system path of the database. + path: string &default = "store.rocksdb"; + }; + + ## Options to tune the particular storage backends. + type BackendOptions: record { + sqlite: SQLiteOptions &default = SQLiteOptions(); + rocksdb: RocksDBOptions &default = RocksDBOptions(); + }; +} diff --git a/scripts/base/init-bare.bro b/scripts/base/init-bare.bro index 4a1bcfbe72..9d790e1e09 100644 --- a/scripts/base/init-bare.bro +++ b/scripts/base/init-bare.bro @@ -3358,6 +3358,7 @@ const bits_per_uid: count = 96 &redef; # Load these frameworks here because they use fairly deep integration with # BiFs and script-land defined types. +@load base/frameworks/comm @load base/frameworks/logging @load base/frameworks/input @load base/frameworks/analyzer diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 6d24172b97..974c23c3a3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -161,6 +161,14 @@ add_subdirectory(iosource) add_subdirectory(logging) add_subdirectory(probabilistic) +if ( ENABLE_BROKER ) + add_subdirectory(comm) +else () + # Just to satisfy coverage unit tests until new Broker-based + # communication is enabled by default. + add_subdirectory(comm-dummy) +endif () + set(bro_SUBDIRS # Order is important here. ${bro_PLUGIN_LIBS} diff --git a/src/DebugLogger.cc b/src/DebugLogger.cc index 6f025e3c2b..3ce5d92888 100644 --- a/src/DebugLogger.cc +++ b/src/DebugLogger.cc @@ -19,7 +19,7 @@ DebugLogger::Stream DebugLogger::streams[NUM_DBGS] = { { "logging", 0, false }, {"input", 0, false }, { "threading", 0, false }, { "file_analysis", 0, false }, { "plugins", 0, false }, { "broxygen", 0, false }, - { "pktio", 0, false} + { "pktio", 0, false }, { "broker", 0, false } }; DebugLogger::DebugLogger(const char* filename) diff --git a/src/DebugLogger.h b/src/DebugLogger.h index 9cd09dada1..13124657e7 100644 --- a/src/DebugLogger.h +++ b/src/DebugLogger.h @@ -32,6 +32,7 @@ enum DebugStream { DBG_PLUGINS, // Plugin system DBG_BROXYGEN, // Broxygen DBG_PKTIO, // Packet sources and dumpers. + DBG_BROKER, // Broker communication NUM_DBGS // Has to be last }; diff --git a/src/EventHandler.cc b/src/EventHandler.cc index 0f25d63ba8..f063026d9a 100644 --- a/src/EventHandler.cc +++ b/src/EventHandler.cc @@ -5,6 +5,11 @@ #include "RemoteSerializer.h" #include "NetVar.h" +#ifdef ENABLE_BROKER +#include "comm/Manager.h" +#include "comm/Data.h" +#endif + EventHandler::EventHandler(const char* arg_name) { name = copy_string(arg_name); @@ -26,7 +31,12 @@ EventHandler::operator bool() const { return enabled && ((local && local->HasBodies()) || receivers.length() - || generate_always); + || generate_always +#ifdef ENABLE_BROKER + || ! auto_remote_send.empty() + // TODO: and require a subscriber interested in a topic or unsolicited flags? +#endif + ); } FuncType* EventHandler::FType() @@ -73,6 +83,46 @@ void EventHandler::Call(val_list* vl, bool no_remote) SerialInfo info(remote_serializer); remote_serializer->SendCall(&info, receivers[i], name, vl); } + +#ifdef ENABLE_BROKER + + if ( ! auto_remote_send.empty() ) + { + // TODO: also short-circuit based on interested subscribers/flags? + broker::message msg; + msg.reserve(vl->length() + 1); + msg.emplace_back(Name()); + bool valid_args = true; + + for ( auto i = 0; i < vl->length(); ++i ) + { + auto opt_data = comm::val_to_data((*vl)[i]); + + if ( opt_data ) + msg.emplace_back(move(*opt_data)); + else + { + valid_args = false; + auto_remote_send.clear(); + reporter->Error("failed auto-remote event '%s', disabled", + Name()); + break; + } + } + + if ( valid_args ) + { + for ( auto it = auto_remote_send.begin(); + it != auto_remote_send.end(); ++it ) + { + if ( std::next(it) == auto_remote_send.end() ) + comm_mgr->Event(it->first, move(msg), it->second); + else + comm_mgr->Event(it->first, msg, it->second); + } + } + } +#endif } if ( local ) diff --git a/src/EventHandler.h b/src/EventHandler.h index 55ac33cffd..2acd2569a6 100644 --- a/src/EventHandler.h +++ b/src/EventHandler.h @@ -4,7 +4,8 @@ #define EVENTHANDLER #include - +#include +#include #include "List.h" #include "BroList.h" @@ -28,6 +29,18 @@ public: void AddRemoteHandler(SourceID peer); void RemoveRemoteHandler(SourceID peer); +#ifdef ENABLE_BROKER + void AutoRemote(std::string topic, int flags) + { + auto_remote_send[std::move(topic)] = flags; + } + + void AutoRemoteStop(const std::string& topic) + { + auto_remote_send.erase(topic); + } +#endif + void Call(val_list* vl, bool no_remote = false); // Returns true if there is at least one local or remote handler. @@ -67,6 +80,10 @@ private: declare(List, SourceID); typedef List(SourceID) receiver_list; receiver_list receivers; + +#ifdef ENABLE_BROKER + std::map auto_remote_send; // topic -> flags +#endif }; // Encapsulates a ptr to an event handler to overload the boolean operator. diff --git a/src/Net.cc b/src/Net.cc index adac9c02fd..3acd4bce9d 100644 --- a/src/Net.cc +++ b/src/Net.cc @@ -34,6 +34,10 @@ #include "iosource/PktDumper.h" #include "plugin/Manager.h" +#ifdef ENABLE_BROKER +#include "comm/Manager.h" +#endif + extern "C" { #include "setsignal.h" }; @@ -315,6 +319,11 @@ void net_run() } #endif current_iosrc = src; + bool communication_enabled = using_communication; + +#ifdef ENABLE_BROKER + communication_enabled |= comm_mgr->Enabled(); +#endif if ( src ) src->Process(); // which will call net_packet_dispatch() @@ -332,7 +341,7 @@ void net_run() } } - else if ( (have_pending_timers || using_communication) && + else if ( (have_pending_timers || communication_enabled) && ! pseudo_realtime ) { // Take advantage of the lull to get up to @@ -347,7 +356,7 @@ void net_run() // us a lot of idle time, but doesn't delay near-term // timers too much. (Delaying them somewhat is okay, // since Bro timers are not high-precision anyway.) - if ( ! using_communication ) + if ( ! communication_enabled ) usleep(100000); else usleep(1000); diff --git a/src/Reporter.cc b/src/Reporter.cc index 9002633b10..cd1aa09d4c 100644 --- a/src/Reporter.cc +++ b/src/Reporter.cc @@ -123,6 +123,19 @@ void Reporter::ExprRuntimeError(const Expr* expr, const char* fmt, ...) throw InterpreterException(); } +void Reporter::RuntimeError(const Location* location, const char* fmt, ...) + { + ++errors; + PushLocation(location); + va_list ap; + va_start(ap, fmt); + FILE* out = errors_to_stderr ? stderr : 0; + DoLog("runtime error", reporter_error, out, 0, 0, true, true, "", fmt, ap); + va_end(ap); + PopLocation(); + throw InterpreterException(); + } + void Reporter::InternalError(const char* fmt, ...) { va_list ap; diff --git a/src/Reporter.h b/src/Reporter.h index e477ad8934..52bcd7d02a 100644 --- a/src/Reporter.h +++ b/src/Reporter.h @@ -73,6 +73,10 @@ public: // function will not return but raise an InterpreterException. void ExprRuntimeError(const Expr* expr, const char* fmt, ...); + // Report a runtime error in evaluating a Bro script expression. This + // function will not return but raise an InterpreterException. + void RuntimeError(const Location* location, const char* fmt, ...); + // Report a traffic weirdness, i.e., an unexpected protocol situation // that may lead to incorrectly processing a connnection. void Weird(const char* name); // Raises net_weird(). diff --git a/src/SerialTypes.h b/src/SerialTypes.h index e50ec3889f..cf2c52a08b 100644 --- a/src/SerialTypes.h +++ b/src/SerialTypes.h @@ -113,6 +113,8 @@ SERIAL_VAL(TOPK_VAL, 20) SERIAL_VAL(BLOOMFILTER_VAL, 21) SERIAL_VAL(CARDINALITY_VAL, 22) SERIAL_VAL(X509_VAL, 23) +SERIAL_VAL(COMM_STORE_HANDLE_VAL, 24) +SERIAL_VAL(COMM_DATA_VAL, 25) #define SERIAL_EXPR(name, val) SERIAL_CONST(name, val, EXPR) SERIAL_EXPR(EXPR, 1) diff --git a/src/Stats.cc b/src/Stats.cc index 01ca0a41d3..111af52598 100644 --- a/src/Stats.cc +++ b/src/Stats.cc @@ -10,6 +10,10 @@ #include "Trigger.h" #include "threading/Manager.h" +#ifdef ENABLE_BROKER +#include "comm/Manager.h" +#endif + int killed_by_inactivity = 0; uint64 tot_ack_events = 0; @@ -222,6 +226,26 @@ void ProfileLogger::Log() )); } +#ifdef ENABLE_BROKER + auto cs = comm_mgr->ConsumeStatistics(); + + file->Write(fmt("%0.6f Comm: peers=%zu stores=%zu " + "store_queries=%zu store_responses=%zu " + "outgoing_conn_status=%zu incoming_conn_status=%zu " + "reports=%zu\n", + network_time, cs.outgoing_peer_count, cs.data_store_count, + cs.pending_query_count, cs.response_count, + cs.outgoing_conn_status_count, cs.incoming_conn_status_count, + cs.report_count)); + + for ( const auto& s : cs.print_count ) + file->Write(fmt(" %-25s prints dequeued=%zu\n", s.first.data(), s.second)); + for ( const auto& s : cs.event_count ) + file->Write(fmt(" %-25s events dequeued=%zu\n", s.first.data(), s.second)); + for ( const auto& s : cs.log_count ) + file->Write(fmt(" %-25s logs dequeued=%zu\n", s.first.data(), s.second)); +#endif + // Script-level state. unsigned int size, mem = 0; PDict(ID)* globals = global_scope()->Vars(); diff --git a/src/Trigger.cc b/src/Trigger.cc index 099027f4e0..772a991791 100644 --- a/src/Trigger.cc +++ b/src/Trigger.cc @@ -112,6 +112,7 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts, attached = 0; is_return = arg_is_return; location = arg_location; + timeout_value = -1; ++total_triggers; @@ -133,17 +134,22 @@ Trigger::Trigger(Expr* arg_cond, Stmt* arg_body, Stmt* arg_timeout_stmts, Val* timeout_val = arg_timeout ? arg_timeout->Eval(arg_frame) : 0; + if ( timeout_val ) + { + Unref(timeout_val); + timeout_value = timeout_val->AsInterval(); + } + // Make sure we don't get deleted if somebody calls a method like // Timeout() while evaluating the trigger. Ref(this); - if ( ! Eval() && timeout_val ) + if ( ! Eval() && timeout_value >= 0 ) { - timer = new TriggerTimer(timeout_val->AsInterval(), this); + timer = new TriggerTimer(timeout_value, this); timer_mgr->Add(timer); } - Unref(timeout_val); Unref(this); } diff --git a/src/Trigger.h b/src/Trigger.h index b752ea8ada..3af9ddf1b0 100644 --- a/src/Trigger.h +++ b/src/Trigger.h @@ -32,6 +32,10 @@ public: // Executes timeout code and deletes the object. void Timeout(); + // Return the timeout interval (negative if none was specified). + double TimeoutValue() const + { return timeout_value; } + // Called if another entity needs to complete its operations first // in any case before this trigger can proceed. void Hold() { delayed = true; } @@ -51,6 +55,8 @@ public: // may not immediately delete it as other references may still exist. void Disable(); + bool Disabled() const { return disabled; } + virtual void Describe(ODesc* d) const { d->Add(""); } // Overidden from Notifier. We queue the trigger and evaluate it @@ -87,6 +93,7 @@ private: Stmt* body; Stmt* timeout_stmts; Expr* timeout; + double timeout_value; Frame* frame; bool is_return; const Location* location; diff --git a/src/comm-dummy/CMakeLists.txt b/src/comm-dummy/CMakeLists.txt new file mode 100644 index 0000000000..cddea1342d --- /dev/null +++ b/src/comm-dummy/CMakeLists.txt @@ -0,0 +1,13 @@ +# Placeholder for Broker-based communication functionality, not enabled +# by default. This helps satisfy coverage unit tests pass regardless of +# whether Broker is enabled or not. + +include(BroSubdir) + +bif_target(comm.bif) +bif_target(data.bif) +bif_target(messaging.bif) +bif_target(store.bif) + +bro_add_subdir_library(comm_dummy ${BIF_OUTPUT_CC}) +add_dependencies(bro_comm_dummy generate_outputs) diff --git a/src/comm-dummy/comm.bif b/src/comm-dummy/comm.bif new file mode 100644 index 0000000000..b030a4cc73 --- /dev/null +++ b/src/comm-dummy/comm.bif @@ -0,0 +1,3 @@ + +##! Placeholder for Broker-based communication functionality, not enabled +##! by default. diff --git a/src/comm-dummy/data.bif b/src/comm-dummy/data.bif new file mode 100644 index 0000000000..e9b9950474 --- /dev/null +++ b/src/comm-dummy/data.bif @@ -0,0 +1,3 @@ + +##! Placeholder for Broker-based communication functionality, not enabled +##! by default diff --git a/src/comm-dummy/messaging.bif b/src/comm-dummy/messaging.bif new file mode 100644 index 0000000000..e9b9950474 --- /dev/null +++ b/src/comm-dummy/messaging.bif @@ -0,0 +1,3 @@ + +##! Placeholder for Broker-based communication functionality, not enabled +##! by default diff --git a/src/comm-dummy/store.bif b/src/comm-dummy/store.bif new file mode 100644 index 0000000000..e9b9950474 --- /dev/null +++ b/src/comm-dummy/store.bif @@ -0,0 +1,3 @@ + +##! Placeholder for Broker-based communication functionality, not enabled +##! by default diff --git a/src/comm/CMakeLists.txt b/src/comm/CMakeLists.txt new file mode 100644 index 0000000000..ef41c605c7 --- /dev/null +++ b/src/comm/CMakeLists.txt @@ -0,0 +1,28 @@ +include(BroSubdir) + +include_directories(BEFORE + ${CMAKE_CURRENT_SOURCE_DIR} + ${CMAKE_CURRENT_BINARY_DIR} +) + +if ( ROCKSDB_INCLUDE_DIR ) + add_definitions(-DHAVE_ROCKSDB) + include_directories(BEFORE ${ROCKSDB_INCLUDE_DIR}) +endif () + +include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_CORE}) +include_directories(BEFORE ${LIBCAF_INCLUDE_DIR_IO}) + +set(comm_SRCS + Data.cc + Manager.cc + Store.cc +) + +bif_target(comm.bif) +bif_target(data.bif) +bif_target(messaging.bif) +bif_target(store.bif) + +bro_add_subdir_library(comm ${comm_SRCS} ${BIF_OUTPUT_CC}) +add_dependencies(bro_comm generate_outputs) diff --git a/src/comm/Data.cc b/src/comm/Data.cc new file mode 100644 index 0000000000..46fc8bc8eb --- /dev/null +++ b/src/comm/Data.cc @@ -0,0 +1,702 @@ +#include "Data.h" +#include "comm/data.bif.h" +#include +#include + +using namespace std; + +OpaqueType* comm::opaque_of_data_type; +OpaqueType* comm::opaque_of_set_iterator; +OpaqueType* comm::opaque_of_table_iterator; +OpaqueType* comm::opaque_of_vector_iterator; +OpaqueType* comm::opaque_of_record_iterator; + +static broker::port::protocol to_broker_port_proto(TransportProto tp) + { + switch ( tp ) { + case TRANSPORT_TCP: + return broker::port::protocol::tcp; + case TRANSPORT_UDP: + return broker::port::protocol::udp; + case TRANSPORT_ICMP: + return broker::port::protocol::icmp; + case TRANSPORT_UNKNOWN: + default: + return broker::port::protocol::unknown; + } + } + +TransportProto comm::to_bro_port_proto(broker::port::protocol tp) + { + switch ( tp ) { + case broker::port::protocol::tcp: + return TRANSPORT_TCP; + case broker::port::protocol::udp: + return TRANSPORT_UDP; + case broker::port::protocol::icmp: + return TRANSPORT_ICMP; + case broker::port::protocol::unknown: + default: + return TRANSPORT_UNKNOWN; + } + } + +struct val_converter { + using result_type = Val*; + + BroType* type; + + result_type operator()(bool a) + { + if ( type->Tag() == TYPE_BOOL ) + return new Val(a, TYPE_BOOL); + return nullptr; + } + + result_type operator()(uint64_t a) + { + if ( type->Tag() == TYPE_COUNT ) + return new Val(a, TYPE_COUNT); + if ( type->Tag() == TYPE_COUNTER ) + return new Val(a, TYPE_COUNTER); + return nullptr; + } + + result_type operator()(int64_t a) + { + if ( type->Tag() == TYPE_INT ) + return new Val(a, TYPE_INT); + return nullptr; + } + + result_type operator()(double a) + { + if ( type->Tag() == TYPE_DOUBLE ) + return new Val(a, TYPE_DOUBLE); + return nullptr; + } + + result_type operator()(std::string& a) + { + switch ( type->Tag() ) { + case TYPE_STRING: + return new StringVal(a.size(), a.data()); + case TYPE_FILE: + { + auto file = BroFile::GetFile(a.data()); + + if ( file ) + { + Ref(file); + return new Val(file); + } + + return nullptr; + } + case TYPE_FUNC: + { + auto id = lookup_ID(a.data(), GLOBAL_MODULE_NAME); + auto rval = id ? id->ID_Val() : nullptr; + Unref(id); + + if ( rval && rval->Type()->Tag() == TYPE_FUNC ) + return rval; + + return nullptr; + } + default: + return nullptr; + } + } + + result_type operator()(broker::address& a) + { + if ( type->Tag() == TYPE_ADDR ) + { + auto bits = reinterpret_cast(&a.bytes()); + return new AddrVal(IPAddr(*bits)); + } + + return nullptr; + } + + result_type operator()(broker::subnet& a) + { + if ( type->Tag() == TYPE_SUBNET ) + { + auto bits = reinterpret_cast(&a.network().bytes()); + return new SubNetVal(IPPrefix(IPAddr(*bits), a.length())); + } + + return nullptr; + } + + result_type operator()(broker::port& a) + { + if ( type->Tag() == TYPE_PORT ) + return new PortVal(a.number(), comm::to_bro_port_proto(a.type())); + + return nullptr; + } + + result_type operator()(broker::time_point& a) + { + if ( type->Tag() == TYPE_TIME ) + return new Val(a.value, TYPE_TIME); + + return nullptr; + } + + result_type operator()(broker::time_duration& a) + { + if ( type->Tag() == TYPE_INTERVAL ) + return new Val(a.value, TYPE_INTERVAL); + + return nullptr; + } + + result_type operator()(broker::enum_value& a) + { + if ( type->Tag() == TYPE_ENUM ) + { + auto etype = type->AsEnumType(); + auto i = etype->Lookup(GLOBAL_MODULE_NAME, a.name.data()); + + if ( i == -1 ) + return nullptr; + + return new EnumVal(i, etype); + } + + return nullptr; + } + + result_type operator()(broker::set& a) + { + if ( ! type->IsSet() ) + return nullptr; + + auto tt = type->AsTableType(); + auto rval = new TableVal(tt); + + for ( auto& item : a ) + { + broker::vector composite_key; + auto indices = broker::get(item); + + if ( ! indices ) + { + composite_key.emplace_back(move(item)); + indices = &composite_key; + } + + auto expected_index_types = tt->Indices()->Types(); + + if ( static_cast(expected_index_types->length()) != + indices->size() ) + { + Unref(rval); + return nullptr; + } + + auto list_val = new ListVal(TYPE_ANY); + + for ( auto i = 0u; i < indices->size(); ++i ) + { + auto index_val = comm::data_to_val(move((*indices)[i]), + (*expected_index_types)[i]); + + if ( ! index_val ) + { + Unref(rval); + Unref(list_val); + return nullptr; + } + + list_val->Append(index_val); + } + + + rval->Assign(list_val, nullptr); + Unref(list_val); + } + + return rval; + } + + result_type operator()(broker::table& a) + { + if ( ! type->IsTable() ) + return nullptr; + + auto tt = type->AsTableType(); + auto rval = new TableVal(tt); + + for ( auto& item : a ) + { + broker::vector composite_key; + auto indices = broker::get(item.first); + + if ( ! indices ) + { + composite_key.emplace_back(move(item.first)); + indices = &composite_key; + } + + auto expected_index_types = tt->Indices()->Types(); + + if ( static_cast(expected_index_types->length()) != + indices->size() ) + { + Unref(rval); + return nullptr; + } + + auto list_val = new ListVal(TYPE_ANY); + + for ( auto i = 0u; i < indices->size(); ++i ) + { + auto index_val = comm::data_to_val(move((*indices)[i]), + (*expected_index_types)[i]); + + if ( ! index_val ) + { + Unref(rval); + Unref(list_val); + return nullptr; + } + + list_val->Append(index_val); + } + + auto value_val = comm::data_to_val(move(item.second), + tt->YieldType()); + + if ( ! value_val ) + { + Unref(rval); + Unref(list_val); + return nullptr; + } + + rval->Assign(list_val, value_val); + Unref(list_val); + } + + return rval; + } + + result_type operator()(broker::vector& a) + { + if ( type->Tag() != TYPE_VECTOR ) + return nullptr; + + auto vt = type->AsVectorType(); + auto rval = new VectorVal(vt); + + for ( auto& item : a ) + { + auto item_val = comm::data_to_val(move(item), vt->YieldType()); + + if ( ! item_val ) + { + Unref(rval); + return nullptr; + } + + rval->Assign(rval->Size(), item_val); + } + + return rval; + } + + result_type operator()(broker::record& a) + { + if ( type->Tag() != TYPE_RECORD ) + return nullptr; + + auto rt = type->AsRecordType(); + + if ( a.fields.size() != static_cast(rt->NumFields()) ) + return nullptr; + + auto rval = new RecordVal(rt); + + for ( auto i = 0u; i < a.fields.size(); ++i ) + { + if ( ! a.fields[i] ) + { + rval->Assign(i, nullptr); + continue; + } + + auto item_val = comm::data_to_val(move(*a.fields[i]), + rt->FieldType(i)); + + if ( ! item_val ) + { + Unref(rval); + return nullptr; + } + + rval->Assign(i, item_val); + } + + return rval; + } +}; + +Val* comm::data_to_val(broker::data d, BroType* type) + { + return broker::visit(val_converter{type}, d); + } + +broker::util::optional comm::val_to_data(Val* v) + { + switch ( v->Type()->Tag() ) { + case TYPE_BOOL: + return {v->AsBool()}; + case TYPE_INT: + return {v->AsInt()}; + case TYPE_COUNT: + return {v->AsCount()}; + case TYPE_COUNTER: + return {v->AsCounter()}; + case TYPE_PORT: + { + auto p = v->AsPortVal(); + return {broker::port(p->Port(), to_broker_port_proto(p->PortType()))}; + } + case TYPE_ADDR: + { + auto a = v->AsAddr(); + in6_addr tmp; + a.CopyIPv6(&tmp); + return {broker::address(reinterpret_cast(&tmp), + broker::address::family::ipv6, + broker::address::byte_order::network)}; + } + break; + case TYPE_SUBNET: + { + auto s = v->AsSubNet(); + in6_addr tmp; + s.Prefix().CopyIPv6(&tmp); + auto a = broker::address(reinterpret_cast(&tmp), + broker::address::family::ipv6, + broker::address::byte_order::network); + return {broker::subnet(a, s.Length())}; + } + break; + case TYPE_DOUBLE: + return {v->AsDouble()}; + case TYPE_TIME: + return {broker::time_point(v->AsTime())}; + case TYPE_INTERVAL: + return {broker::time_duration(v->AsInterval())}; + case TYPE_ENUM: + { + auto enum_type = v->Type()->AsEnumType(); + auto enum_name = enum_type->Lookup(v->AsEnum()); + return {broker::enum_value(enum_name ? enum_name : "")}; + } + case TYPE_STRING: + { + auto s = v->AsString(); + return {string(reinterpret_cast(s->Bytes()), s->Len())}; + } + case TYPE_FILE: + return {string(v->AsFile()->Name())}; + case TYPE_FUNC: + return {string(v->AsFunc()->Name())}; + case TYPE_TABLE: + { + auto is_set = v->Type()->IsSet(); + auto table = v->AsTable(); + auto table_val = v->AsTableVal(); + broker::data rval; + + if ( is_set ) + rval = broker::set(); + else + rval = broker::table(); + + struct iter_guard { + iter_guard(HashKey* arg_k, ListVal* arg_lv) + : k(arg_k), lv(arg_lv) + {} + + ~iter_guard() + { + delete k; + Unref(lv); + } + + HashKey* k; + ListVal* lv; + }; + + HashKey* k; + TableEntryVal* entry; + auto c = table->InitForIteration(); + + while ( (entry = table->NextEntry(k, c)) ) + { + auto vl = table_val->RecoverIndex(k); + iter_guard ig(k, vl); + + broker::vector composite_key; + composite_key.reserve(vl->Length()); + + for ( auto k = 0; k < vl->Length(); ++k ) + { + auto key_part = val_to_data((*vl->Vals())[k]); + + if ( ! key_part ) + return {}; + + composite_key.emplace_back(move(*key_part)); + } + + broker::data key; + + if ( composite_key.size() == 1 ) + key = move(composite_key[0]); + else + key = move(composite_key); + + if ( is_set ) + broker::get(rval)->emplace(move(key)); + else + { + auto val = val_to_data(entry->Value()); + + if ( ! val ) + return {}; + + broker::get(rval)->emplace(move(key), + move(*val)); + } + } + + return {rval}; + } + case TYPE_VECTOR: + { + auto vec = v->AsVectorVal(); + broker::vector rval; + rval.reserve(vec->Size()); + + for ( auto i = 0u; i < vec->Size(); ++i ) + { + auto item_val = vec->Lookup(i); + + if ( ! item_val ) + continue; + + auto item = val_to_data(item_val); + + if ( ! item ) + return {}; + + rval.emplace_back(move(*item)); + } + + return {rval}; + } + case TYPE_RECORD: + { + auto rec = v->AsRecordVal(); + broker::record rval; + size_t num_fields = v->Type()->AsRecordType()->NumFields(); + rval.fields.reserve(num_fields); + + for ( auto i = 0u; i < num_fields; ++i ) + { + auto item_val = rec->LookupWithDefault(i); + + if ( ! item_val ) + { + rval.fields.emplace_back(broker::record::field{}); + continue; + } + + auto item = val_to_data(item_val); + Unref(item_val); + + if ( ! item ) + return {}; + + rval.fields.emplace_back(broker::record::field{move(*item)}); + } + + return {rval}; + } + default: + reporter->Error("unsupported Comm::Data type: %s", + type_name(v->Type()->Tag())); + break; + } + + return {}; + } + +RecordVal* comm::make_data_val(Val* v) + { + auto rval = new RecordVal(BifType::Record::Comm::Data); + auto data = val_to_data(v); + + if ( data ) + rval->Assign(0, new DataVal(move(*data))); + + return rval; + } + +RecordVal* comm::make_data_val(broker::data d) + { + auto rval = new RecordVal(BifType::Record::Comm::Data); + rval->Assign(0, new DataVal(move(d))); + return rval; + } + +struct data_type_getter { + using result_type = EnumVal*; + + result_type operator()(bool a) + { + return new EnumVal(BifEnum::Comm::BOOL, + BifType::Enum::Comm::DataType); + } + + result_type operator()(uint64_t a) + { + return new EnumVal(BifEnum::Comm::COUNT, + BifType::Enum::Comm::DataType); + } + + result_type operator()(int64_t a) + { + return new EnumVal(BifEnum::Comm::INT, + BifType::Enum::Comm::DataType); + } + + result_type operator()(double a) + { + return new EnumVal(BifEnum::Comm::DOUBLE, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const std::string& a) + { + return new EnumVal(BifEnum::Comm::STRING, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::address& a) + { + return new EnumVal(BifEnum::Comm::ADDR, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::subnet& a) + { + return new EnumVal(BifEnum::Comm::SUBNET, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::port& a) + { + return new EnumVal(BifEnum::Comm::PORT, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::time_point& a) + { + return new EnumVal(BifEnum::Comm::TIME, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::time_duration& a) + { + return new EnumVal(BifEnum::Comm::INTERVAL, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::enum_value& a) + { + return new EnumVal(BifEnum::Comm::ENUM, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::set& a) + { + return new EnumVal(BifEnum::Comm::SET, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::table& a) + { + return new EnumVal(BifEnum::Comm::TABLE, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::vector& a) + { + return new EnumVal(BifEnum::Comm::VECTOR, + BifType::Enum::Comm::DataType); + } + + result_type operator()(const broker::record& a) + { + return new EnumVal(BifEnum::Comm::RECORD, + BifType::Enum::Comm::DataType); + } +}; + +EnumVal* comm::get_data_type(RecordVal* v, Frame* frame) + { + return broker::visit(data_type_getter{}, opaque_field_to_data(v, frame)); + } + +broker::data& comm::opaque_field_to_data(RecordVal* v, Frame* f) + { + Val* d = v->Lookup(0); + + if ( ! d ) + reporter->RuntimeError(f->GetCall()->GetLocationInfo(), + "Comm::Data's opaque field is not set"); + + return static_cast(d)->data; + } + +IMPLEMENT_SERIAL(comm::DataVal, SER_COMM_DATA_VAL); + +bool comm::DataVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_COMM_DATA_VAL, OpaqueVal); + + std::string serial; + caf::binary_serializer bs(std::back_inserter(serial)); + bs << data; + + if ( ! SERIALIZE_STR(serial.data(), serial.size()) ) + return false; + + return true; + } + +bool comm::DataVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(OpaqueVal); + + const char* serial; + int len; + + if ( ! UNSERIALIZE_STR(&serial, &len) ) + return false; + + caf::binary_deserializer bd(serial, len); + caf::uniform_typeid()->deserialize(&data, &bd); + delete [] serial; + return true; + } diff --git a/src/comm/Data.h b/src/comm/Data.h new file mode 100644 index 0000000000..ef7b15110d --- /dev/null +++ b/src/comm/Data.h @@ -0,0 +1,254 @@ +#ifndef BRO_COMM_DATA_H +#define BRO_COMM_DATA_H + +#include +#include "Val.h" +#include "Reporter.h" +#include "Frame.h" +#include "Expr.h" + +namespace comm { + +extern OpaqueType* opaque_of_data_type; +extern OpaqueType* opaque_of_set_iterator; +extern OpaqueType* opaque_of_table_iterator; +extern OpaqueType* opaque_of_vector_iterator; +extern OpaqueType* opaque_of_record_iterator; + +/** + * Convert a broker port protocol to a bro port protocol. + */ +TransportProto to_bro_port_proto(broker::port::protocol tp); + +/** + * Create a Comm::Data value from a Bro value. + * @param v the Bro value to convert to a Broker data value. + * @return a Comm::Data value, where the optional field is set if the conversion + * was possible, else it is unset. + */ +RecordVal* make_data_val(Val* v); + +/** + * Create a Comm::Data value from a Broker data value. + * @param d the Broker value to wrap in an opaque type. + * @return a Comm::Data value that wraps the Broker value. + */ +RecordVal* make_data_val(broker::data d); + +/** + * Get the type of Broker data that Comm::Data wraps. + * @param v a Comm::Data value. + * @param frame used to get location info upon error. + * @return a Comm::DataType value. + */ +EnumVal* get_data_type(RecordVal* v, Frame* frame); + +/** + * Convert a Bro value to a Broker data value. + * @param v a Bro value. + * @return a Broker data value if the Bro value could be converted to one. + */ +broker::util::optional val_to_data(Val* v); + +/** + * Convert a Broker data value to a Bro value. + * @param d a Broker data value. + * @param type the expected type of the value to return. + * @return a pointer to a new Bro value or a nullptr if the conversion was not + * possible. + */ +Val* data_to_val(broker::data d, BroType* type); + +/** + * A Bro value which wraps a Broker data value. + */ +class DataVal : public OpaqueVal { +public: + + DataVal(broker::data arg_data) + : OpaqueVal(comm::opaque_of_data_type), data(std::move(arg_data)) + {} + + void ValDescribe(ODesc* d) const override + { + d->Add("broker::data{"); + d->Add(broker::to_string(data)); + d->Add("}"); + } + + DECLARE_SERIAL(DataVal); + + broker::data data; + +protected: + + DataVal() + {} +}; + +/** + * Visitor for retrieving type names a Broker data value. + */ +struct type_name_getter { + using result_type = const char*; + + result_type operator()(bool a) + { return "bool"; } + + result_type operator()(uint64_t a) + { return "uint64_t"; } + + result_type operator()(int64_t a) + { return "int64_t"; } + + result_type operator()(double a) + { return "double"; } + + result_type operator()(const std::string& a) + { return "string"; } + + result_type operator()(const broker::address& a) + { return "address"; } + + result_type operator()(const broker::subnet& a) + { return "subnet"; } + + result_type operator()(const broker::port& a) + { return "port"; } + + result_type operator()(const broker::time_point& a) + { return "time"; } + + result_type operator()(const broker::time_duration& a) + { return "interval"; } + + result_type operator()(const broker::enum_value& a) + { return "enum"; } + + result_type operator()(const broker::set& a) + { return "set"; } + + result_type operator()(const broker::table& a) + { return "table"; } + + result_type operator()(const broker::vector& a) + { return "vector"; } + + result_type operator()(const broker::record& a) + { return "record"; } +}; + +/** + * Retrieve Broker data value associated with a Comm::Data Bro value. + * @param v a Comm::Data value. + * @param f used to get location information on error. + * @return a reference to the wrapped Broker data value. A runtime interpreter + * exception is thrown if the the optional opaque value of \a v is not set. + */ +broker::data& opaque_field_to_data(RecordVal* v, Frame* f); + +/** + * Retrieve variant data from a Broker data value. + * @tparam T a type that the variant may contain. + * @param d a Broker data value to get variant data out of. + * @param tag a Bro tag which corresponds to T (just used for error reporting). + * @param f used to get location information on error. + * @return a refrence to the requested type in the variant Broker data. + * A runtime interpret exception is thrown if trying to access a type which + * is not currently stored in the Broker data. + */ +template +T& require_data_type(broker::data& d, TypeTag tag, Frame* f) + { + auto ptr = broker::get(d); + + if ( ! ptr ) + reporter->RuntimeError(f->GetCall()->GetLocationInfo(), + "data is of type '%s' not of type '%s'", + broker::visit(type_name_getter{}, d), + type_name(tag)); + + return *ptr; + } + +/** + * @see require_data_type() and opaque_field_to_data(). + */ +template +inline T& require_data_type(RecordVal* v, TypeTag tag, Frame* f) + { + return require_data_type(opaque_field_to_data(v, f), tag, f); + } + +/** + * Convert a Comm::Data Bro value to a Bro value of a given type. + * @tparam a type that a Broker data variant may contain. + * @param v a Comm::Data value. + * @param tag a Bro type to convert to. + * @param f used to get location information on error. + * A runtime interpret exception is thrown if trying to access a type which + * is not currently stored in the Broker data. + */ +template +inline Val* refine(RecordVal* v, TypeTag tag, Frame* f) + { + return new Val(require_data_type(v, tag, f), tag); + } + +// Copying data in to iterator vals is not the fastest approach, but safer... + +class SetIterator : public OpaqueVal { +public: + + SetIterator(RecordVal* v, TypeTag tag, Frame* f) + : OpaqueVal(comm::opaque_of_set_iterator), + dat(require_data_type(v, TYPE_TABLE, f)), + it(dat.begin()) + {} + + broker::set dat; + broker::set::iterator it; +}; + +class TableIterator : public OpaqueVal { +public: + + TableIterator(RecordVal* v, TypeTag tag, Frame* f) + : OpaqueVal(comm::opaque_of_table_iterator), + dat(require_data_type(v, TYPE_TABLE, f)), + it(dat.begin()) + {} + + broker::table dat; + broker::table::iterator it; +}; + +class VectorIterator : public OpaqueVal { +public: + + VectorIterator(RecordVal* v, TypeTag tag, Frame* f) + : OpaqueVal(comm::opaque_of_vector_iterator), + dat(require_data_type(v, TYPE_VECTOR, f)), + it(dat.begin()) + {} + + broker::vector dat; + broker::vector::iterator it; +}; + +class RecordIterator : public OpaqueVal { +public: + + RecordIterator(RecordVal* v, TypeTag tag, Frame* f) + : OpaqueVal(comm::opaque_of_record_iterator), + dat(require_data_type(v, TYPE_VECTOR, f)), + it(dat.fields.begin()) + {} + + broker::record dat; + decltype(broker::record::fields)::iterator it; +}; + +} // namespace comm + +#endif // BRO_COMM_DATA_H diff --git a/src/comm/Manager.cc b/src/comm/Manager.cc new file mode 100644 index 0000000000..420f67f711 --- /dev/null +++ b/src/comm/Manager.cc @@ -0,0 +1,1055 @@ +#include "Manager.h" +#include "Data.h" +#include "Store.h" +#include +#include +#include +#include +#include "util.h" +#include "Var.h" +#include "Reporter.h" +#include "comm/comm.bif.h" +#include "comm/data.bif.h" +#include "comm/messaging.bif.h" +#include "comm/store.bif.h" +#include "logging/Manager.h" +#include "DebugLogger.h" +#include "iosource/Manager.h" + +using namespace std; + +VectorType* comm::Manager::vector_of_data_type; +EnumType* comm::Manager::log_id_type; +int comm::Manager::send_flags_self_idx; +int comm::Manager::send_flags_peers_idx; +int comm::Manager::send_flags_unsolicited_idx; + +comm::Manager::~Manager() + { + for ( auto& s : data_stores ) + CloseStore(s.first.first, s.first.second); + } + +static int require_field(RecordType* rt, const char* name) + { + auto rval = rt->FieldOffset(name); + + if ( rval < 0 ) + reporter->InternalError("no field named '%s' in record type '%s'", name, + rt->GetName().data()); + + return rval; + } + +static int endpoint_flags_to_int(Val* broker_endpoint_flags) + { + int rval = 0; + auto r = broker_endpoint_flags->AsRecordVal(); + Val* auto_publish_flag = r->Lookup("auto_publish", true); + Val* auto_advertise_flag = r->Lookup("auto_advertise", true); + + if ( auto_publish_flag->AsBool() ) + rval |= broker::AUTO_PUBLISH; + + if ( auto_advertise_flag->AsBool() ) + rval |= broker::AUTO_ADVERTISE; + + Unref(auto_publish_flag); + Unref(auto_advertise_flag); + return rval; + } + +bool comm::Manager::Enable(Val* broker_endpoint_flags) + { + if ( endpoint != nullptr ) + return true; + + auto send_flags_type = internal_type("Comm::SendFlags")->AsRecordType(); + send_flags_self_idx = require_field(send_flags_type, "self"); + send_flags_peers_idx = require_field(send_flags_type, "peers"); + send_flags_unsolicited_idx = require_field(send_flags_type, "unsolicited"); + + log_id_type = internal_type("Log::ID")->AsEnumType(); + + comm::opaque_of_data_type = new OpaqueType("Comm::Data"); + comm::opaque_of_set_iterator = new OpaqueType("Comm::SetIterator"); + comm::opaque_of_table_iterator = new OpaqueType("Comm::TableIterator"); + comm::opaque_of_vector_iterator = new OpaqueType("Comm::VectorIterator"); + comm::opaque_of_record_iterator = new OpaqueType("Comm::RecordIterator"); + comm::opaque_of_store_handle = new OpaqueType("Store::Handle"); + vector_of_data_type = new VectorType(internal_type("Comm::Data")->Ref()); + + auto res = broker::init(); + + if ( res ) + { + fprintf(stderr, "broker::init failed: %s\n", broker::strerror(res)); + return false; + } + + res = broker::report::init(true); + + if ( res ) + { + fprintf(stderr, "broker::report::init failed: %s\n", + broker::strerror(res)); + return false; + } + + const char* name; + auto name_from_script = internal_val("Comm::endpoint_name")->AsString(); + + if ( name_from_script->Len() ) + name = name_from_script->CheckString(); + else + { + char host[256]; + + if ( gethostname(host, sizeof(host)) == 0 ) + name = fmt("bro@%s.%ld", host, static_cast(getpid())); + else + name = fmt("bro@.%ld", static_cast(getpid())); + } + + int flags = endpoint_flags_to_int(broker_endpoint_flags); + endpoint = unique_ptr(new broker::endpoint(name, flags)); + iosource_mgr->Register(this, true); + return true; + } + +bool comm::Manager::SetEndpointFlags(Val* broker_endpoint_flags) + { + if ( ! Enabled() ) + return false; + + int flags = endpoint_flags_to_int(broker_endpoint_flags); + endpoint->set_flags(flags); + return true; + } + +bool comm::Manager::Listen(uint16_t port, const char* addr, bool reuse_addr) + { + if ( ! Enabled() ) + return false; + + auto rval = endpoint->listen(port, addr, reuse_addr); + + if ( ! rval ) + { + reporter->Error("Failed to listen on %s:%" PRIu16 " : %s", + addr ? addr : "INADDR_ANY", port, + endpoint->last_error().data()); + } + + return rval; + } + +bool comm::Manager::Connect(string addr, uint16_t port, + chrono::duration retry_interval) + { + if ( ! Enabled() ) + return false; + + auto& peer = peers[make_pair(addr, port)]; + + if ( peer ) + return false; + + peer = endpoint->peer(move(addr), port, retry_interval); + return true; + } + +bool comm::Manager::Disconnect(const string& addr, uint16_t port) + { + if ( ! Enabled() ) + return false; + + auto it = peers.find(make_pair(addr, port)); + + if ( it == peers.end() ) + return false; + + auto rval = endpoint->unpeer(it->second); + peers.erase(it); + return rval; + } + +bool comm::Manager::Print(string topic, string msg, Val* flags) + { + if ( ! Enabled() ) + return false; + + endpoint->send(move(topic), broker::message{move(msg)}, + send_flags_to_int(flags)); + return true; + } + +bool comm::Manager::Event(std::string topic, broker::message msg, int flags) + { + if ( ! Enabled() ) + return false; + + endpoint->send(move(topic), move(msg), flags); + return true; + } + +bool comm::Manager::Log(EnumVal* stream, RecordVal* columns, int flags) + { + if ( ! Enabled() ) + return false; + + auto stream_name = stream->Type()->AsEnumType()->Lookup(stream->AsEnum()); + + if ( ! stream_name ) + { + reporter->Error("Failed to remotely log: stream %d doesn't have name", + stream->AsEnum()); + return false; + } + + auto opt_column_data = val_to_data(columns); + + if ( ! opt_column_data ) + { + reporter->Error("Failed to remotely log stream %s: unsupported types", + stream_name); + return false; + } + + broker::message msg{broker::enum_value{stream_name}, + move(*opt_column_data)}; + std::string topic = std::string("bro/log/") + stream_name; + endpoint->send(move(topic), move(msg), flags); + return true; + } + +bool comm::Manager::Event(std::string topic, RecordVal* args, Val* flags) + { + if ( ! Enabled() ) + return false; + + if ( ! args->Lookup(0) ) + return false; + + auto event_name = args->Lookup(0)->AsString()->CheckString(); + auto vv = args->Lookup(1)->AsVectorVal(); + broker::message msg; + msg.reserve(vv->Size() + 1); + msg.emplace_back(event_name); + + for ( auto i = 0u; i < vv->Size(); ++i ) + { + auto val = vv->Lookup(i)->AsRecordVal()->Lookup(0); + auto data_val = static_cast(val); + msg.emplace_back(data_val->data); + } + + endpoint->send(move(topic), move(msg), send_flags_to_int(flags)); + return true; + } + +bool comm::Manager::AutoEvent(string topic, Val* event, Val* flags) + { + if ( ! Enabled() ) + return false; + + if ( event->Type()->Tag() != TYPE_FUNC ) + { + reporter->Error("Comm::auto_event must operate on an event"); + return false; + } + + auto event_val = event->AsFunc(); + + if ( event_val->Flavor() != FUNC_FLAVOR_EVENT ) + { + reporter->Error("Comm::auto_event must operate on an event"); + return false; + } + + auto handler = event_registry->Lookup(event_val->Name()); + + if ( ! handler ) + { + reporter->Error("Comm::auto_event failed to lookup event '%s'", + event_val->Name()); + return false; + } + + handler->AutoRemote(move(topic), send_flags_to_int(flags)); + return true; + } + +bool comm::Manager::AutoEventStop(const string& topic, Val* event) + { + if ( ! Enabled() ) + return false; + + if ( event->Type()->Tag() != TYPE_FUNC ) + { + reporter->Error("Comm::auto_event_stop must operate on an event"); + return false; + } + + auto event_val = event->AsFunc(); + + if ( event_val->Flavor() != FUNC_FLAVOR_EVENT ) + { + reporter->Error("Comm::auto_event_stop must operate on an event"); + return false; + } + + auto handler = event_registry->Lookup(event_val->Name()); + + if ( ! handler ) + { + reporter->Error("Comm::auto_event_stop failed to lookup event '%s'", + event_val->Name()); + return false; + } + + + handler->AutoRemoteStop(topic); + return true; + } + +RecordVal* comm::Manager::MakeEventArgs(val_list* args) + { + if ( ! Enabled() ) + return nullptr; + + auto rval = new RecordVal(BifType::Record::Comm::EventArgs); + auto arg_vec = new VectorVal(vector_of_data_type); + rval->Assign(1, arg_vec); + Func* func = 0; + + for ( auto i = 0; i < args->length(); ++i ) + { + auto arg_val = (*args)[i]; + + if ( i == 0 ) + { + // Event val must come first. + + if ( arg_val->Type()->Tag() != TYPE_FUNC ) + { + reporter->Error("1st param of Comm::event_args must be event"); + return rval; + } + + func = arg_val->AsFunc(); + + if ( func->Flavor() != FUNC_FLAVOR_EVENT ) + { + reporter->Error("1st param of Comm::event_args must be event"); + return rval; + } + + auto num_args = func->FType()->Args()->NumFields(); + + if ( num_args != args->length() - 1 ) + { + reporter->Error("bad # of Comm::event_args: got %d, expect %d", + args->length(), num_args + 1); + return rval; + } + + rval->Assign(0, new StringVal(func->Name())); + continue; + } + + auto expected_type = (*func->FType()->ArgTypes()->Types())[i - 1]; + + if ( ! same_type((*args)[i]->Type(), expected_type) ) + { + rval->Assign(0, 0); + reporter->Error("Comm::event_args param %d type mismatch", i); + return rval; + } + + auto data_val = make_data_val((*args)[i]); + + if ( ! data_val->Lookup(0) ) + { + Unref(data_val); + rval->Assign(0, 0); + reporter->Error("Comm::event_args unsupported event/params"); + return rval; + } + + arg_vec->Assign(i - 1, data_val); + } + + return rval; + } + +bool comm::Manager::SubscribeToPrints(string topic_prefix) + { + if ( ! Enabled() ) + return false; + + auto& q = print_subscriptions[topic_prefix].q; + + if ( q ) + return false; + + q = broker::message_queue(move(topic_prefix), *endpoint); + return true; + } + +bool comm::Manager::UnsubscribeToPrints(const string& topic_prefix) + { + if ( ! Enabled() ) + return false; + + return print_subscriptions.erase(topic_prefix); + } + +bool comm::Manager::SubscribeToEvents(string topic_prefix) + { + if ( ! Enabled() ) + return false; + + auto& q = event_subscriptions[topic_prefix].q; + + if ( q ) + return false; + + q = broker::message_queue(move(topic_prefix), *endpoint); + return true; + } + +bool comm::Manager::UnsubscribeToEvents(const string& topic_prefix) + { + if ( ! Enabled() ) + return false; + + return event_subscriptions.erase(topic_prefix); + } + +bool comm::Manager::SubscribeToLogs(string topic_prefix) + { + if ( ! Enabled() ) + return false; + + auto& q = log_subscriptions[topic_prefix].q; + + if ( q ) + return false; + + q = broker::message_queue(move(topic_prefix), *endpoint); + return true; + } + +bool comm::Manager::UnsubscribeToLogs(const string& topic_prefix) + { + if ( ! Enabled() ) + return false; + + return log_subscriptions.erase(topic_prefix); + } + +bool comm::Manager::PublishTopic(broker::topic t) + { + if ( ! Enabled() ) + return false; + + endpoint->publish(move(t)); + return true; + } + +bool comm::Manager::UnpublishTopic(broker::topic t) + { + if ( ! Enabled() ) + return false; + + endpoint->unpublish(move(t)); + return true; + } + +bool comm::Manager::AdvertiseTopic(broker::topic t) + { + if ( ! Enabled() ) + return false; + + endpoint->advertise(move(t)); + return true; + } + +bool comm::Manager::UnadvertiseTopic(broker::topic t) + { + if ( ! Enabled() ) + return false; + + endpoint->unadvertise(move(t)); + return true; + } + +int comm::Manager::send_flags_to_int(Val* flags) + { + auto r = flags->AsRecordVal(); + int rval = 0; + Val* self_flag = r->LookupWithDefault(send_flags_self_idx); + Val* peers_flag = r->LookupWithDefault(send_flags_peers_idx); + Val* unsolicited_flag = r->LookupWithDefault(send_flags_unsolicited_idx); + + if ( self_flag->AsBool() ) + rval |= broker::SELF; + + if ( peers_flag->AsBool() ) + rval |= broker::PEERS; + + if ( unsolicited_flag->AsBool() ) + rval |= broker::UNSOLICITED; + + Unref(self_flag); + Unref(peers_flag); + Unref(unsolicited_flag); + return rval; + } + +void comm::Manager::GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) + { + read->Insert(endpoint->outgoing_connection_status().fd()); + read->Insert(endpoint->incoming_connection_status().fd()); + + for ( const auto& ps : print_subscriptions ) + read->Insert(ps.second.q.fd()); + + for ( const auto& ps : event_subscriptions ) + read->Insert(ps.second.q.fd()); + + for ( const auto& ps : log_subscriptions ) + read->Insert(ps.second.q.fd()); + + for ( const auto& s : data_stores ) + read->Insert(s.second->store->responses().fd()); + + read->Insert(broker::report::default_queue->fd()); + } + +double comm::Manager::NextTimestamp(double* local_network_time) + { + // TODO: do something better? + return timer_mgr->Time(); + } + +struct response_converter { + using result_type = RecordVal*; + broker::store::query::tag query_tag; + + result_type operator()(bool d) + { + switch ( query_tag ) { + case broker::store::query::tag::pop_left: + case broker::store::query::tag::pop_right: + case broker::store::query::tag::lookup: + // A boolean result means the key doesn't exist (if it did, then + // the result would contain the broker::data value, not a bool). + return new RecordVal(BifType::Record::Comm::Data); + default: + return comm::make_data_val(broker::data{d}); + } + } + + result_type operator()(uint64_t d) + { + return comm::make_data_val(broker::data{d}); + } + + result_type operator()(broker::data& d) + { + return comm::make_data_val(move(d)); + } + + result_type operator()(std::vector& d) + { + return comm::make_data_val(broker::data{move(d)}); + } + + result_type operator()(broker::store::snapshot& d) + { + broker::table table; + + for ( auto& item : d.entries ) + { + auto& key = item.first; + auto& val = item.second.item; + table[move(key)] = move(val); + } + + return comm::make_data_val(broker::data{move(table)}); + } +}; + +static RecordVal* response_to_val(broker::store::response r) + { + return broker::visit(response_converter{r.request.type}, r.reply.value); + } + +void comm::Manager::Process() + { + bool idle = true; + auto outgoing_connection_updates = + endpoint->outgoing_connection_status().want_pop(); + auto incoming_connection_updates = + endpoint->incoming_connection_status().want_pop(); + + statistics.outgoing_conn_status_count += outgoing_connection_updates.size(); + statistics.incoming_conn_status_count += incoming_connection_updates.size(); + + for ( auto& u : outgoing_connection_updates ) + { + idle = false; + + switch ( u.status ) { + case broker::outgoing_connection_status::tag::established: + if ( Comm::outgoing_connection_established ) + { + val_list* vl = new val_list; + vl->append(new StringVal(u.relation.remote_tuple().first)); + vl->append(new PortVal(u.relation.remote_tuple().second, + TRANSPORT_TCP)); + vl->append(new StringVal(u.peer_name)); + mgr.QueueEvent(Comm::outgoing_connection_established, vl); + } + break; + + case broker::outgoing_connection_status::tag::disconnected: + if ( Comm::outgoing_connection_broken ) + { + val_list* vl = new val_list; + vl->append(new StringVal(u.relation.remote_tuple().first)); + vl->append(new PortVal(u.relation.remote_tuple().second, + TRANSPORT_TCP)); + mgr.QueueEvent(Comm::outgoing_connection_broken, vl); + } + break; + + case broker::outgoing_connection_status::tag::incompatible: + if ( Comm::outgoing_connection_incompatible ) + { + val_list* vl = new val_list; + vl->append(new StringVal(u.relation.remote_tuple().first)); + vl->append(new PortVal(u.relation.remote_tuple().second, + TRANSPORT_TCP)); + mgr.QueueEvent(Comm::outgoing_connection_incompatible, vl); + } + break; + + default: + reporter->InternalWarning( + "unknown broker::outgoing_connection_status::tag : %d", + static_cast(u.status)); + break; + } + } + + for ( auto& u : incoming_connection_updates ) + { + idle = false; + + switch ( u.status ) { + case broker::incoming_connection_status::tag::established: + if ( Comm::incoming_connection_established ) + { + val_list* vl = new val_list; + vl->append(new StringVal(u.peer_name)); + mgr.QueueEvent(Comm::incoming_connection_established, vl); + } + break; + + case broker::incoming_connection_status::tag::disconnected: + if ( Comm::incoming_connection_broken ) + { + val_list* vl = new val_list; + vl->append(new StringVal(u.peer_name)); + mgr.QueueEvent(Comm::incoming_connection_broken, vl); + } + break; + + default: + reporter->InternalWarning( + "unknown broker::incoming_connection_status::tag : %d", + static_cast(u.status)); + break; + } + } + + for ( auto& ps : print_subscriptions ) + { + auto print_messages = ps.second.q.want_pop(); + + if ( print_messages.empty() ) + continue; + + ps.second.received += print_messages.size(); + idle = false; + + if ( ! Comm::print_handler ) + continue; + + for ( auto& pm : print_messages ) + { + if ( pm.size() != 1 ) + { + reporter->Warning("got print message of invalid size: %zd", + pm.size()); + continue; + } + + std::string* msg = broker::get(pm[0]); + + if ( ! msg ) + { + reporter->Warning("got print message of invalid type: %d", + static_cast(broker::which(pm[0]))); + continue; + } + + val_list* vl = new val_list; + vl->append(new StringVal(move(*msg))); + mgr.QueueEvent(Comm::print_handler, vl); + } + } + + for ( auto& es : event_subscriptions ) + { + auto event_messages = es.second.q.want_pop(); + + if ( event_messages.empty() ) + continue; + + es.second.received += event_messages.size(); + idle = false; + + for ( auto& em : event_messages ) + { + if ( em.empty() ) + { + reporter->Warning("got empty event message"); + continue; + } + + std::string* event_name = broker::get(em[0]); + + if ( ! event_name ) + { + reporter->Warning("got event message w/o event name: %d", + static_cast(broker::which(em[0]))); + continue; + } + + EventHandlerPtr ehp = event_registry->Lookup(event_name->data()); + + if ( ! ehp ) + continue; + + auto arg_types = ehp->FType()->ArgTypes()->Types(); + + if ( static_cast(arg_types->length()) != em.size() - 1 ) + { + reporter->Warning("got event message with invalid # of args," + " got %zd, expected %d", em.size() - 1, + arg_types->length()); + continue; + } + + val_list* vl = new val_list; + + for ( auto i = 1u; i < em.size(); ++i ) + { + auto val = data_to_val(move(em[i]), (*arg_types)[i - 1]); + + if ( val ) + vl->append(val); + else + { + reporter->Warning("failed to convert remote event arg # %d", + i - 1); + break; + } + } + + if ( static_cast(vl->length()) == em.size() - 1 ) + mgr.QueueEvent(ehp, vl); + else + delete_vals(vl); + } + } + + struct unref_guard { + unref_guard(Val* v) : val(v) {} + ~unref_guard() { Unref(val); } + Val* val; + }; + + for ( auto& ls : log_subscriptions ) + { + auto log_messages = ls.second.q.want_pop(); + + if ( log_messages.empty() ) + continue; + + ls.second.received += log_messages.size(); + idle = false; + + for ( auto& lm : log_messages ) + { + if ( lm.size() != 2 ) + { + reporter->Warning("got bad remote log size: %zd (expect 2)", + lm.size()); + continue; + } + + if ( ! broker::get(lm[0]) ) + { + reporter->Warning("got remote log w/o stream id: %d", + static_cast(broker::which(lm[0]))); + continue; + } + + if ( ! broker::get(lm[1]) ) + { + reporter->Warning("got remote log w/o columns: %d", + static_cast(broker::which(lm[1]))); + continue; + } + + auto stream_id = data_to_val(move(lm[0]), log_id_type); + + if ( ! stream_id ) + { + reporter->Warning("failed to unpack remote log stream id"); + continue; + } + + unref_guard stream_id_unreffer{stream_id}; + auto columns_type = log_mgr->StreamColumns(stream_id->AsEnumVal()); + + if ( ! columns_type ) + { + reporter->Warning("got remote log for unknown stream: %s", + stream_id->Type()->AsEnumType()->Lookup( + stream_id->AsEnum())); + continue; + } + + auto columns = data_to_val(move(lm[1]), columns_type); + + if ( ! columns ) + { + reporter->Warning("failed to unpack remote log stream columns" + " for stream: %s", + stream_id->Type()->AsEnumType()->Lookup( + stream_id->AsEnum())); + continue; + } + + log_mgr->Write(stream_id->AsEnumVal(), columns->AsRecordVal()); + Unref(columns); + } + } + + for ( const auto& s : data_stores ) + { + auto responses = s.second->store->responses().want_pop(); + + if ( responses.empty() ) + continue; + + statistics.report_count += responses.size(); + idle = false; + + for ( auto& response : responses ) + { + auto ck = static_cast(response.cookie); + auto it = pending_queries.find(ck); + + if ( it == pending_queries.end() ) + { + reporter->Warning("unmatched response to query on store %s", + s.second->store->id().data()); + continue; + } + + auto query = *it; + + if ( query->Disabled() ) + { + // Trigger timer must have timed the query out already. + delete query; + pending_queries.erase(it); + continue; + } + + switch ( response.reply.stat ) { + case broker::store::result::status::timeout: + // Fine, trigger's timeout takes care of things. + break; + case broker::store::result::status::failure: + query->Result(query_result()); + break; + case broker::store::result::status::success: + query->Result(query_result(response_to_val(move(response)))); + break; + default: + reporter->InternalWarning("unknown store response status: %d", + static_cast(response.reply.stat)); + break; + } + + delete query; + pending_queries.erase(it); + } + } + + auto reports = broker::report::default_queue->want_pop(); + statistics.report_count += reports.size(); + + for ( auto& report : reports ) + { + idle = false; + + if ( report.size() < 2 ) + { + reporter->Warning("got broker report msg of size %zu, expect 4", + report.size()); + continue; + } + + uint64_t* level = broker::get(report[1]); + + if ( ! level ) + { + reporter->Warning("got broker report msg w/ bad level type: %d", + static_cast(broker::which(report[1]))); + continue; + } + + auto lvl = static_cast(*level); + + switch ( lvl ) { + case broker::report::level::debug: + DBG_LOG(DBG_BROKER, broker::to_string(report).data()); + break; + case broker::report::level::info: + reporter->Info("broker info: %s", + broker::to_string(report).data()); + break; + case broker::report::level::warn: + reporter->Warning("broker warning: %s", + broker::to_string(report).data()); + break; + case broker::report::level::error: + reporter->Error("broker error: %s", + broker::to_string(report).data()); + break; + } + } + + SetIdle(idle); + } + +bool comm::Manager::AddStore(StoreHandleVal* handle) + { + if ( ! Enabled() ) + return false; + + if ( ! handle->store ) + return false; + + auto key = make_pair(handle->store->id(), handle->store_type); + + if ( data_stores.find(key) != data_stores.end() ) + return false; + + data_stores[key] = handle; + Ref(handle); + return true; + } + +comm::StoreHandleVal* +comm::Manager::LookupStore(const broker::store::identifier& id, + comm::StoreType type) + { + if ( ! Enabled() ) + return nullptr; + + auto key = make_pair(id, type); + auto it = data_stores.find(key); + + if ( it == data_stores.end() ) + return nullptr; + + return it->second; + } + +bool comm::Manager::CloseStore(const broker::store::identifier& id, + StoreType type) + { + if ( ! Enabled() ) + return false; + + auto key = make_pair(id, type); + auto it = data_stores.find(key); + + if ( it == data_stores.end() ) + return false; + + for ( auto it = pending_queries.begin(); it != pending_queries.end(); ) + { + auto query = *it; + + if ( query->GetStoreType() == type && query->StoreID() == id ) + { + it = pending_queries.erase(it); + query->Abort(); + delete query; + } + else + ++it; + } + + delete it->second->store; + it->second->store = nullptr; + Unref(it->second); + return true; + } + +bool comm::Manager::TrackStoreQuery(StoreQueryCallback* cb) + { + assert(Enabled()); + return pending_queries.insert(cb).second; + } + +comm::Stats comm::Manager::ConsumeStatistics() + { + statistics.outgoing_peer_count = peers.size(); + statistics.data_store_count = data_stores.size(); + statistics.pending_query_count = pending_queries.size(); + + for ( auto& s : print_subscriptions ) + { + statistics.print_count[s.first] = s.second.received; + s.second.received = 0; + } + + for ( auto& s : event_subscriptions ) + { + statistics.event_count[s.first] = s.second.received; + s.second.received = 0; + } + + for ( auto& s : log_subscriptions ) + { + statistics.log_count[s.first] = s.second.received; + s.second.received = 0; + } + + auto rval = move(statistics); + statistics = Stats{}; + return rval; + } diff --git a/src/comm/Manager.h b/src/comm/Manager.h new file mode 100644 index 0000000000..0093c0bd90 --- /dev/null +++ b/src/comm/Manager.h @@ -0,0 +1,364 @@ +#ifndef BRO_COMM_MANAGER_H +#define BRO_COMM_MANAGER_H + +#include +#include +#include +#include +#include +#include +#include "comm/Store.h" +#include "Reporter.h" +#include "iosource/IOSource.h" +#include "Val.h" + +namespace comm { + +/** + * Communication statistics. Some are tracked in relation to last + * sample (comm::Manager::ConsumeStatistics()). + */ +struct Stats { + // Number of outgoing peer connections (at time of sample). + size_t outgoing_peer_count = 0; + // Number of data stores (at time of sample). + size_t data_store_count = 0; + // Number of pending data store queries (at time of sample). + size_t pending_query_count = 0; + // Number of data store responses received (since last sample). + size_t response_count = 0; + // Number of outgoing connection updates received (since last sample). + size_t outgoing_conn_status_count = 0; + // Number of incoming connection updates received (since last sample). + size_t incoming_conn_status_count = 0; + // Number of broker report messages (e.g. debug, warning, errors) received + // (since last sample). + size_t report_count = 0; + // Number of print messages received per topic-prefix (since last sample). + std::map print_count; + // Number of event messages received per topic-prefix (since last sample). + std::map event_count; + // Number of log messages received per topic-prefix (since last sample). + std::map log_count; +}; + +/** + * Manages various forms of communication between peer Bro processes + * or other external applications via use of the Broker messaging library. + */ +class Manager : public iosource::IOSource { +friend class StoreHandleVal; +public: + + /** + * Destructor. Any still-pending data store queries are aborted. + */ + ~Manager(); + + /** + * Enable use of communication. + * @param flags used to tune the local Broker endpoint's behavior. + * See the Comm::EndpointFlags record type. + * @return true if communication is successfully initialized. + */ + bool Enable(Val* flags); + + /** + * Changes endpoint flags originally supplied to comm::Manager::Enable(). + * @param flags the new behavior flags to use. + * @return true if flags were changed. + */ + bool SetEndpointFlags(Val* flags); + + /** + * @return true if comm::Manager::Enable() has previously been called and + * it succeeded. + */ + bool Enabled() + { return endpoint != nullptr; } + + /** + * Listen for remote connections. + * @param port the TCP port to listen on. + * @param addr an address string on which to accept connections, e.g. + * "127.0.0.1". A nullptr refers to @p INADDR_ANY. + * @param reuse_addr equivalent to behavior of SO_REUSEADDR. + * @return true if the local endpoint is now listening for connections. + */ + bool Listen(uint16_t port, const char* addr = nullptr, + bool reuse_addr = true); + + /** + * Initiate a remote connection. + * @param addr an address to connect to, e.g. "localhost" or "127.0.0.1". + * @param port the TCP port on which the remote side is listening. + * @param retry_interval an interval at which to retry establishing the + * connection with the remote peer. + * @return true if it's possible to try connecting with the peer and + * it's a new peer. The actual connection may not be established until a + * later point in time. + */ + bool Connect(std::string addr, uint16_t port, + std::chrono::duration retry_interval); + + /** + * Remove a remote connection. + * @param addr the address used in comm::Manager::Connect(). + * @param port the port used in comm::Manager::Connect(). + * @return true if the arguments match a previously successful call to + * comm::Manager::Connect(). + */ + bool Disconnect(const std::string& addr, uint16_t port); + + /** + * Print a simple message to any interested peers. + * @param topic a topic string associated with the print message. + * Peers advertise interest by registering a subscription to some prefix + * of this topic name. + * @param msg the string to send to peers. + * @param flags tune the behavior of how the message is send. + * See the Comm::SendFlags record type. + * @return true if the message is sent successfully. + */ + bool Print(std::string topic, std::string msg, Val* flags); + + /** + * Send an event to any interested peers. + * @param topic a topic string associated with the print message. + * Peers advertise interest by registering a subscription to some prefix + * of this topic name. + * @param msg the event to send to peers, which is the name of the event + * as a string followed by all of its arguments. + * @param flags tune the behavior of how the message is send. + * See the Comm::SendFlags record type. + * @return true if the message is sent successfully. + */ + bool Event(std::string topic, broker::message msg, int flags); + + /** + * Send an event to any interested peers. + * @param topic a topic string associated with the print message. + * Peers advertise interest by registering a subscription to some prefix + * of this topic name. + * @param args the event and its arguments to send to peers. See the + * Comm::EventArgs record type. + * @param flags tune the behavior of how the message is send. + * See the Comm::SendFlags record type. + * @return true if the message is sent successfully. + */ + bool Event(std::string topic, RecordVal* args, Val* flags); + + /** + * Send a log entry to any interested peers. The topic name used is + * implicitly "bro/log/". + * @param stream_id the stream to which the log entry belongs. + * @param columns the data which comprises the log entry. + * @param flags tune the behavior of how the message is send. + * See the Comm::SendFlags record type. + * @return true if the message is sent successfully. + */ + bool Log(EnumVal* stream_id, RecordVal* columns, int flags); + + /** + * Automatically send an event to any interested peers whenever it is + * locally dispatched (e.g. using "event my_event(...);" in a script). + * @param topic a topic string associated with the event message. + * Peers advertise interest by registering a subscription to some prefix + * of this topic name. + * @param event a Bro event value. + * @param flags tune the behavior of how the message is send. + * See the Comm::SendFlags record type. + * @return true if automatic event sending is now enabled. + */ + bool AutoEvent(std::string topic, Val* event, Val* flags); + + /** + * Stop automatically sending an event to peers upon local dispatch. + * @param topic a topic originally given to comm::Manager::AutoEvent(). + * @param event an event originally given to comm::Manager::AutoEvent(). + * @return true if automatic events will no occur for the topic/event pair. + */ + bool AutoEventStop(const std::string& topic, Val* event); + + /** + * Create an EventArgs record value from an event and its arguments. + * @param args the event and its arguments. The event is always the first + * elements in the list. + * @return an EventArgs record value. If an invalid event or arguments + * were supplied the optional "name" field will not be set. + */ + RecordVal* MakeEventArgs(val_list* args); + + /** + * Register interest in peer print messages that use a certain topic prefix. + * @param topic_prefix a prefix to match against remote message topics. + * e.g. an empty prefix will match everything and "a" will match "alice" + * and "amy" but not "bob". + * @return true if it's a new print subscriptions and it is now registered. + */ + bool SubscribeToPrints(std::string topic_prefix); + + /** + * Unregister interest in peer print messages. + * @param topic_prefix a prefix previously supplied to a successful call + * to comm::Manager::SubscribeToPrints(). + * @return true if interest in topic prefix is no longer advertised. + */ + bool UnsubscribeToPrints(const std::string& topic_prefix); + + /** + * Register interest in peer event messages that use a certain topic prefix. + * @param topic_prefix a prefix to match against remote message topics. + * e.g. an empty prefix will match everything and "a" will match "alice" + * and "amy" but not "bob". + * @return true if it's a new event subscription and it is now registered. + */ + bool SubscribeToEvents(std::string topic_prefix); + + /** + * Unregister interest in peer event messages. + * @param topic_prefix a prefix previously supplied to a successful call + * to comm::Manager::SubscribeToEvents(). + * @return true if interest in topic prefix is no longer advertised. + */ + bool UnsubscribeToEvents(const std::string& topic_prefix); + + /** + * Register interest in peer log messages that use a certain topic prefix. + * @param topic_prefix a prefix to match against remote message topics. + * e.g. an empty prefix will match everything and "a" will match "alice" + * and "amy" but not "bob". + * @return true if it's a new log subscription and it is now registered. + */ + bool SubscribeToLogs(std::string topic_prefix); + + /** + * Unregister interest in peer log messages. + * @param topic_prefix a prefix previously supplied to a successful call + * to comm::Manager::SubscribeToLogs(). + * @return true if interest in topic prefix is no longer advertised. + */ + bool UnsubscribeToLogs(const std::string& topic_prefix); + + /** + * Allow sending messages to peers if associated with the given topic. + * This has no effect if auto publication behavior is enabled via the flags + * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags(). + * @param t a topic to allow messages to be published under. + * @return true if successful. + */ + bool PublishTopic(broker::topic t); + + /** + * Disallow sending messages to peers if associated with the given topic. + * This has no effect if auto publication behavior is enabled via the flags + * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags(). + * @param t a topic to disallow messages to be published under. + * @return true if successful. + */ + bool UnpublishTopic(broker::topic t); + + /** + * Allow advertising interest in the given topic to peers. + * This has no effect if auto advertise behavior is enabled via the flags + * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags(). + * @param t a topic to allow advertising interest/subscription to peers. + * @return true if successful. + */ + bool AdvertiseTopic(broker::topic t); + + /** + * Disallow advertising interest in the given topic to peers. + * This has no effect if auto advertise behavior is enabled via the flags + * supplied to comm::Manager::Enable() or comm::Manager::SetEndpointFlags(). + * @param t a topic to disallow advertising interest/subscription to peers. + * @return true if successful. + */ + bool UnadvertiseTopic(broker::topic t); + + /** + * Register the availability of a data store. + * @param handle the data store. + * @return true if the store was valid and not already away of it. + */ + bool AddStore(StoreHandleVal* handle); + + /** + * Lookup a data store by it's identifier name and type. + * @param id the store's name. + * @param type the type of data store. + * @return a pointer to the store handle if it exists else nullptr. + */ + StoreHandleVal* LookupStore(const broker::store::identifier& id, StoreType type); + + /** + * Close and unregister a data store. Any existing references to the + * store handle will not be able to be used for any data store operations. + * @param id the stores' name. + * @param type the type of the data store. + * @return true if such a store existed and is now closed. + */ + bool CloseStore(const broker::store::identifier& id, StoreType type); + + /** + * Register a data store query callback. + * @param cb the callback info to use when the query completes or times out. + * @return true if now tracking a data store query. + */ + bool TrackStoreQuery(StoreQueryCallback* cb); + + /** + * @return communication statistics. + */ + Stats ConsumeStatistics(); + + /** + * Convert Comm::SendFlags to int flags for use with broker::send(). + */ + static int send_flags_to_int(Val* flags); + +private: + + // IOSource interface overrides: + void GetFds(iosource::FD_Set* read, iosource::FD_Set* write, + iosource::FD_Set* except) override; + + double NextTimestamp(double* local_network_time) override; + + void Process() override; + + const char* Tag() override + { return "Comm::Manager"; } + + broker::endpoint& Endpoint() + { return *endpoint; } + + struct QueueWithStats { + broker::message_queue q; + size_t received = 0; + }; + + std::unique_ptr endpoint; + std::map, broker::peering> peers; + std::map print_subscriptions; + std::map event_subscriptions; + std::map log_subscriptions; + + std::map, + StoreHandleVal*> data_stores; + std::unordered_set pending_queries; + + Stats statistics; + + static VectorType* vector_of_data_type; + static EnumType* log_id_type; + static int send_flags_self_idx; + static int send_flags_peers_idx; + static int send_flags_unsolicited_idx; +}; + +} // namespace comm + +extern comm::Manager* comm_mgr; + +#endif // BRO_COMM_MANAGER_H diff --git a/src/comm/Store.cc b/src/comm/Store.cc new file mode 100644 index 0000000000..453fdee34a --- /dev/null +++ b/src/comm/Store.cc @@ -0,0 +1,204 @@ +#include "Store.h" +#include "comm/Manager.h" + +#include +#include +#include + +#ifdef HAVE_ROCKSDB +#include +#include +#endif + +OpaqueType* comm::opaque_of_store_handle; + +comm::StoreHandleVal::StoreHandleVal(broker::store::identifier id, + comm::StoreType arg_type, + broker::util::optional arg_back, + RecordVal* backend_options, std::chrono::duration resync) + : OpaqueVal(opaque_of_store_handle), + store(), store_type(arg_type), backend_type(arg_back) + { + using BifEnum::Store::BackendType; + std::unique_ptr backend; + + if ( backend_type ) + switch ( *backend_type ) { + case BackendType::MEMORY: + backend.reset(new broker::store::memory_backend); + break; + case BackendType::SQLITE: + { + auto sqlite = new broker::store::sqlite_backend; + std::string path = backend_options->Lookup(0)->AsRecordVal() + ->Lookup(0)->AsStringVal()->CheckString(); + + if ( sqlite->open(path) ) + backend.reset(sqlite); + else + { + reporter->Error("failed to open sqlite backend at path %s: %s", + path.data(), sqlite->last_error().data()); + delete sqlite; + } + } + break; + case BackendType::ROCKSDB: + { +#ifdef HAVE_ROCKSDB + std::string path = backend_options->Lookup(1)->AsRecordVal() + ->Lookup(0)->AsStringVal()->CheckString(); + rocksdb::Options rock_op; + rock_op.create_if_missing = true; + + auto rocksdb = new broker::store::rocksdb_backend; + + if ( rocksdb->open(path, options).ok() ) + backend.reset(rocksdb); + else + { + reporter->Error("failed to open rocksdb backend at path %s: %s", + path.data(), rocksdb->last_error().data()); + delete rocksdb; + } +#else + reporter->Error("rocksdb backend support is not enabled"); +#endif + } + break; + default: + reporter->FatalError("unknown data store backend: %d", + static_cast(*backend_type)); + } + + switch ( store_type ) { + case StoreType::FRONTEND: + store = new broker::store::frontend(comm_mgr->Endpoint(), move(id)); + break; + case StoreType::MASTER: + store = new broker::store::master(comm_mgr->Endpoint(), move(id), + move(backend)); + break; + case StoreType::CLONE: + store = new broker::store::clone(comm_mgr->Endpoint(), move(id), resync, + move(backend)); + break; + default: + reporter->FatalError("unknown data store type: %d", + static_cast(store_type)); + } + } + +void comm::StoreHandleVal::ValDescribe(ODesc* d) const + { + using BifEnum::Store::BackendType; + d->Add("broker::store::"); + + switch ( store_type ) { + case StoreType::FRONTEND: + d->Add("frontend"); + break; + case StoreType::MASTER: + d->Add("master"); + break; + case StoreType::CLONE: + d->Add("clone"); + break; + default: + d->Add("unknown"); + } + + d->Add("{"); + d->Add(store->id()); + + if ( backend_type ) + { + d->Add(", "); + + switch ( *backend_type ) { + case BackendType::MEMORY: + d->Add("memory"); + break; + case BackendType::SQLITE: + d->Add("sqlite"); + break; + case BackendType::ROCKSDB: + d->Add("rocksdb"); + break; + default: + d->Add("unknown"); + } + } + + d->Add("}"); + } + +IMPLEMENT_SERIAL(comm::StoreHandleVal, SER_COMM_STORE_HANDLE_VAL); + +bool comm::StoreHandleVal::DoSerialize(SerialInfo* info) const + { + DO_SERIALIZE(SER_COMM_STORE_HANDLE_VAL, OpaqueVal); + + bool have_store = store != nullptr; + + if ( ! SERIALIZE(have_store) ) + return false; + + if ( ! have_store ) + return true; + + if ( ! SERIALIZE(static_cast(store_type)) ) + return false; + + if ( ! SERIALIZE_STR(store->id().data(), store->id().size()) ) + return false; + + return true; + } + +bool comm::StoreHandleVal::DoUnserialize(UnserialInfo* info) + { + DO_UNSERIALIZE(OpaqueVal); + + bool have_store; + + if ( ! UNSERIALIZE(&have_store) ) + return false; + + if ( ! have_store ) + { + store = nullptr; + return true; + } + + int type; + + if ( ! UNSERIALIZE(&type) ) + return false; + + const char* id_str; + int len; + + if ( ! UNSERIALIZE_STR(&id_str, &len) ) + return false; + + broker::store::identifier id(id_str, len); + delete [] id_str; + + auto handle = comm_mgr->LookupStore(id, static_cast(type)); + + if ( ! handle ) + { + // Passing serialized version of store handles to other Bro processes + // doesn't make sense, only allow local clones of the handle val. + reporter->Error("failed to look up unserialized store handle %s, %d", + id.data(), type); + store = nullptr; + return false; + } + + store = handle->store; + store_type = handle->store_type; + backend_type = handle->backend_type; + return true; + } diff --git a/src/comm/Store.h b/src/comm/Store.h new file mode 100644 index 0000000000..9a17d617e9 --- /dev/null +++ b/src/comm/Store.h @@ -0,0 +1,153 @@ +#ifndef BRO_COMM_STORE_H +#define BRO_COMM_STORE_H + +#include "comm/store.bif.h" +#include "comm/data.bif.h" +#include "Reporter.h" +#include "Type.h" +#include "Val.h" +#include "Trigger.h" + +#include + +namespace comm { + +extern OpaqueType* opaque_of_store_handle; + +/** + * Enumerates the possible types of data stores. + */ +enum StoreType { + // Just a view in to a remote store, contains no data itself. + FRONTEND, + MASTER, + CLONE, +}; + +/** + * Create a Store::QueryStatus value. + * @param success whether the query status should be set to success or failure. + * @return a Store::QueryStatus value. + */ +inline EnumVal* query_status(bool success) + { + static EnumType* store_query_status = nullptr; + static int success_val; + static int failure_val; + + if ( ! store_query_status ) + { + store_query_status = internal_type("Store::QueryStatus")->AsEnumType(); + success_val = store_query_status->Lookup("Store", "SUCCESS"); + failure_val = store_query_status->Lookup("Store", "FAILURE"); + } + + return new EnumVal(success ? success_val : failure_val, store_query_status); + } + +/** + * @return a Store::QueryResult value that has a Store::QueryStatus indicating + * a failure. + */ +inline RecordVal* query_result() + { + auto rval = new RecordVal(BifType::Record::Store::QueryResult); + rval->Assign(0, query_status(false)); + rval->Assign(1, new RecordVal(BifType::Record::Comm::Data)); + return rval; + } + +/** + * @param data the result of the query. + * @return a Store::QueryResult value that has a Store::QueryStatus indicating + * a success. + */ +inline RecordVal* query_result(RecordVal* data) + { + auto rval = new RecordVal(BifType::Record::Store::QueryResult); + rval->Assign(0, query_status(true)); + rval->Assign(1, data); + return rval; + } + +/** + * Used for asynchronous data store queries which use "when" statements. + */ +class StoreQueryCallback { +public: + + StoreQueryCallback(Trigger* arg_trigger, const CallExpr* arg_call, + broker::store::identifier arg_store_id, + StoreType arg_store_type) + : trigger(arg_trigger), call(arg_call), store_id(move(arg_store_id)), + store_type(arg_store_type) + { + Ref(trigger); + } + + ~StoreQueryCallback() + { + Unref(trigger); + } + + void Result(RecordVal* result) + { + trigger->Cache(call, result); + trigger->Release(); + Unref(result); + } + + void Abort() + { + auto result = query_result(); + trigger->Cache(call, result); + trigger->Release(); + Unref(result); + } + + bool Disabled() const + { return trigger->Disabled(); } + + const broker::store::identifier& StoreID() const + { return store_id; } + + StoreType GetStoreType() const + { return store_type; } + +private: + + Trigger* trigger; + const CallExpr* call; + broker::store::identifier store_id; + StoreType store_type; +}; + +/** + * An opaque handle which wraps a Broker data store. + */ +class StoreHandleVal : public OpaqueVal { +public: + + StoreHandleVal(broker::store::identifier id, + comm::StoreType arg_type, + broker::util::optional arg_back, + RecordVal* backend_options, + std::chrono::duration resync = std::chrono::seconds(1)); + + void ValDescribe(ODesc* d) const override; + + DECLARE_SERIAL(StoreHandleVal); + + broker::store::frontend* store; + comm::StoreType store_type; + broker::util::optional backend_type; + +protected: + + StoreHandleVal() + {} +}; + +} // namespace comm + +#endif // BRO_COMM_STORE_H diff --git a/src/comm/comm.bif b/src/comm/comm.bif new file mode 100644 index 0000000000..23e163c748 --- /dev/null +++ b/src/comm/comm.bif @@ -0,0 +1,199 @@ + +##! General functions regarding Bro's broker communication mechanisms. + +%%{ +#include "comm/Manager.h" +%%} + +module Comm; + +type Comm::EndpointFlags: record; + +## Enable use of communication. +## +## flags: used to tune the local Broker endpoint behavior. +## +## Returns: true if communication is successfully initialized. +function Comm::enable%(flags: EndpointFlags &default = EndpointFlags()%): bool + %{ + return new Val(comm_mgr->Enable(flags), TYPE_BOOL); + %} + +## Changes endpoint flags originally supplied to :bro:see:`Comm::enable`. +## +## flags: the new endpoint behavior flags to use. +## +## Returns: true of flags were changed. +function Comm::set_endpoint_flags%(flags: EndpointFlags &default = EndpointFlags()%): bool + %{ + return new Val(comm_mgr->SetEndpointFlags(flags), TYPE_BOOL); + %} + +## Allow sending messages to peers if associated with the given topic. +## This has no effect if auto publication behavior is enabled via the flags +## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`. +## +## topic: a topic to allow messages to be published under. +## +## Returns: true if successful. +function Comm::publish_topic%(topic: string%): bool + %{ + return new Val(comm_mgr->PublishTopic(topic->CheckString()), TYPE_BOOL); + %} + +## Disallow sending messages to peers if associated with the given topic. +## This has no effect if auto publication behavior is enabled via the flags +## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`. +## +## topic: a topic to disallow messages to be published under. +## +## Returns: true if successful. +function Comm::unpublish_topic%(topic: string%): bool + %{ + return new Val(comm_mgr->UnpublishTopic(topic->CheckString()), TYPE_BOOL); + %} + +## Allow advertising interest in the given topic to peers. +## This has no effect if auto advertise behavior is enabled via the flags +## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`. +## +## topic: a topic to allow advertising interest/subscription to peers. +## +## Returns: true if successful. +function Comm::advertise_topic%(topic: string%): bool + %{ + return new Val(comm_mgr->AdvertiseTopic(topic->CheckString()), TYPE_BOOL); + %} + +## Disallow advertising interest in the given topic to peers. +## This has no effect if auto advertise behavior is enabled via the flags +## supplied to :bro:see:`Comm::enable` or :bro:see:`Comm::set_endpoint_flags`. +## +## topic: a topic to disallow advertising interest/subscription to peers. +## +## Returns: true if successful. +function Comm::unadvertise_topic%(topic: string%): bool + %{ + return new Val(comm_mgr->UnadvertiseTopic(topic->CheckString()), TYPE_BOOL); + %} + +## Generated when a connection has been established due to a previous call +## to :bro:see:`Comm::connect`. +## +## peer_address: the address used to connect to the peer. +## +## peer_port: the port used to connect to the peer. +## +## peer_name: the name by which the peer identified itself. +event Comm::outgoing_connection_established%(peer_address: string, + peer_port: port, + peer_name: string%); + +## Generated when a previously established connection becomes broken. +## Reconnection will automatically be attempted at a frequency given +## by the original call to :bro:see:`Comm::connect`. +## +## peer_address: the address used to connect to the peer. +## +## peer_port: the port used to connect to the peer. +## +## .. bro:see:: Comm::outgoing_connection_established +event Comm::outgoing_connection_broken%(peer_address: string, + peer_port: port%); + +## Generated when a connection via :bro:see:`Comm::connect` has failed +## because the remote side is incompatible. +## +## peer_address: the address used to connect to the peer. +## +## peer_port: the port used to connect to the peer. +event Comm::outgoing_connection_incompatible%(peer_address: string, + peer_port: port%); + +## Generated when a peer has established a connection with this process +## as a result of previously performing a :bro:see:`Comm::listen`. +## +## peer_name: the name by which the peer identified itself. +event Comm::incoming_connection_established%(peer_name: string%); + +## Generated when a peer that previously established a connection with this +## process becomes disconnected. +## +## peer_name: the name by which the peer identified itself. +## +## .. bro:see:: Comm::incoming_connection_established +event Comm::incoming_connection_broken%(peer_name: string%); + +## Listen for remote connections. +## +## p: the TCP port to listen on. +## +## a: an address string on which to accept connections, e.g. +## "127.0.0.1". An empty string refers to @p INADDR_ANY. +## +## reuse: equivalent to behavior of SO_REUSEADDR. +## +## Returns: true if the local endpoint is now listening for connections. +## +## .. bro:see:: Comm::incoming_connection_established +function Comm::listen%(p: port, a: string &default = "", + reuse: bool &default = T%): bool + %{ + if ( ! p->IsTCP() ) + { + reporter->Error("listen port must use tcp"); + return new Val(false, TYPE_BOOL); + } + + auto rval = comm_mgr->Listen(p->Port(), a->Len() ? a->CheckString() : 0, + reuse); + return new Val(rval, TYPE_BOOL); + %} + +## Initiate a remote connection. +## +## a: an address to connect to, e.g. "localhost" or "127.0.0.1". +## +## p: the TCP port on which the remote side is listening. +## +## retry: an interval at which to retry establishing the +## connection with the remote peer if it cannot be made initially, or +## if it ever becomes disconnected. +## +## Returns: true if it's possible to try connecting with the peer and +## it's a new peer. The actual connection may not be established +## a later point in time. +## +## .. bro:see:: Comm::outgoing_connection_established +function Comm::connect%(a: string, p: port, retry: interval%): bool + %{ + if ( ! p->IsTCP() ) + { + reporter->Error("remote connection port must use tcp"); + return new Val(false, TYPE_BOOL); + } + + auto rval = comm_mgr->Connect(a->CheckString(), p->Port(), + std::chrono::duration(retry)); + return new Val(rval, TYPE_BOOL); + %} + +## Remove a remote connection. +## +## a: the address used in previous successful call to :bro:see:`Comm::connect`. +## +## p: the port used in previous successful call to :bro:see:`Comm::connect`. +## +## Returns: true if the arguments match a previously successful call to +## :bro:see:`Comm::connect`. +function Comm::disconnect%(a: string, p: port%): bool + %{ + if ( ! p->IsTCP() ) + { + reporter->Error("remote connection port must use tcp"); + return new Val(false, TYPE_BOOL); + } + + auto rval = comm_mgr->Disconnect(a->CheckString(), p->Port()); + return new Val(rval, TYPE_BOOL); + %} diff --git a/src/comm/data.bif b/src/comm/data.bif new file mode 100644 index 0000000000..7120046920 --- /dev/null +++ b/src/comm/data.bif @@ -0,0 +1,834 @@ + +##! Functions for inspecting and manipulating broker data. + +%%{ +#include "comm/Data.h" +%%} + +module Comm; + +## Enumerates the possible types that :bro:see:`Comm::Data` may be in terms of +## Bro data types. +enum DataType %{ + BOOL, + INT, + COUNT, + DOUBLE, + STRING, + ADDR, + SUBNET, + PORT, + TIME, + INTERVAL, + ENUM, + SET, + TABLE, + VECTOR, + RECORD, +%} + +type Comm::Data: record; + +type Comm::TableItem: record; + +## Convert any Bro value in to communication data. +## +## d: any Bro value to attempt to convert (not all types are supported). +## +## Returns: the converted communication data which may not set its only +## opaque field of the the conversion was not possible (the Bro data +## type does not support being converted to communicaiton data). +function Comm::data%(d: any%): Comm::Data + %{ + return comm::make_data_val(d); + %} + +## Retrieve the type of data associated with communication data. +## +## d: the communication data. +## +## Returns: the data type associated with the communication data. +function Comm::data_type%(d: Comm::Data%): Comm::DataType + %{ + return comm::get_data_type(d->AsRecordVal(), frame); + %} + +## Convert communication data with a type of :bro:see:`Comm::BOOL` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_bool%(d: Comm::Data%): bool + %{ + return comm::refine(d->AsRecordVal(), TYPE_BOOL, frame); + %} + +## Convert communication data with a type of :bro:see:`Comm::INT` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_int%(d: Comm::Data%): int + %{ + return comm::refine(d->AsRecordVal(), TYPE_INT, frame); + %} + +## Convert communication data with a type of :bro:see:`Comm::COUNT` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_count%(d: Comm::Data%): count + %{ + return comm::refine(d->AsRecordVal(), TYPE_COUNT, frame); + %} + +## Convert communication data with a type of :bro:see:`Comm::DOUBLE` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_double%(d: Comm::Data%): double + %{ + return comm::refine(d->AsRecordVal(), TYPE_DOUBLE, frame); + %} + +## Convert communication data with a type of :bro:see:`Comm::STRING` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_string%(d: Comm::Data%): string + %{ + return new StringVal(comm::require_data_type(d->AsRecordVal(), + TYPE_STRING, + frame)); + %} + +## Convert communication data with a type of :bro:see:`Comm::ADDR` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_addr%(d: Comm::Data%): addr + %{ + auto& a = comm::require_data_type(d->AsRecordVal(), + TYPE_ADDR, frame); + auto bits = reinterpret_cast(&a.bytes()); + return new AddrVal(IPAddr(*bits)); + %} + +## Convert communication data with a type of :bro:see:`Comm::SUBNET` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_subnet%(d: Comm::Data%): subnet + %{ + auto& a = comm::require_data_type(d->AsRecordVal(), + TYPE_SUBNET, frame); + auto bits = reinterpret_cast(&a.network().bytes()); + return new SubNetVal(IPPrefix(IPAddr(*bits), a.length())); + %} + +## Convert communication data with a type of :bro:see:`Comm::PORT` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_port%(d: Comm::Data%): port + %{ + auto& a = comm::require_data_type(d->AsRecordVal(), + TYPE_SUBNET, frame); + return new PortVal(a.number(), comm::to_bro_port_proto(a.type())); + %} + +## Convert communication data with a type of :bro:see:`Comm::TIME` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_time%(d: Comm::Data%): time + %{ + auto v = comm::require_data_type(d->AsRecordVal(), + TYPE_TIME, frame).value; + return new Val(v, TYPE_TIME); + %} + +## Convert communication data with a type of :bro:see:`Comm::INTERVAL` to +## an actual Bro value. +## +## d: the communication data to convert. +## +## Returns: the value retrieved from the communication data. +function Comm::refine_to_interval%(d: Comm::Data%): interval + %{ + auto v = comm::require_data_type(d->AsRecordVal(), + TYPE_TIME, frame).value; + return new Val(v, TYPE_INTERVAL); + %} + +## Convert communication data with a type of :bro:see:`Comm::ENUM` to +## the name of the enum value. :bro:see:`lookup_ID` may be used to convert +## the name to the actual enum value. +## +## d: the communication data to convert. +## +## Returns: the enum name retrieved from the communication data. +function Comm::refine_to_enum_name%(d: Comm::Data%): string + %{ + auto& v = comm::require_data_type(d->AsRecordVal(), + TYPE_ENUM, frame).name; + return new StringVal(v); + %} + +## Create communication data of type "set". +function Comm::set_create%(%): Comm::Data + %{ + return comm::make_data_val(broker::set()); + %} + +## Remove all elements within a set. +## +## s: the set to clear. +## +## Returns: always true. +function Comm::set_clear%(s: Comm::Data%): bool + %{ + auto& v = comm::require_data_type(s->AsRecordVal(), TYPE_TABLE, + frame); + v.clear(); + return new Val(true, TYPE_BOOL); + %} + +## Get the number of elements within a set. +## +## s: the set to query. +## +## Returns: the number of elements in the set. +function Comm::set_size%(s: Comm::Data%): count + %{ + auto& v = comm::require_data_type(s->AsRecordVal(), TYPE_TABLE, + frame); + return new Val(static_cast(v.size()), TYPE_COUNT); + %} + +## Check if a set contains a particular element. +## +## s: the set to query. +## +## key: the element to check for existence. +## +## Returns: true if the key exists in the set. +function Comm::set_contains%(s: Comm::Data, key: Comm::Data%): bool + %{ + auto& v = comm::require_data_type(s->AsRecordVal(), TYPE_TABLE, + frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + return new Val(v.find(k) != v.end(), TYPE_BOOL); + %} + +### Insert an element into a set. +## +## s: the set to modify. +## +## key: the element to insert. +## +## Returns: true if the key was inserted, or false if it already existed. +function Comm::set_insert%(s: Comm::Data, key: Comm::Data%): bool + %{ + auto& v = comm::require_data_type(s->AsRecordVal(), TYPE_TABLE, + frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + return new Val(v.insert(k).second, TYPE_BOOL); + %} + +## Remove an element from a set. +## +## s: the set to modify. +## +## key: the element to remove. +## +## Returns: true if the element existed in the set and is now removed. +function Comm::set_remove%(s: Comm::Data, key: Comm::Data%): bool + %{ + auto& v = comm::require_data_type(s->AsRecordVal(), TYPE_TABLE, + frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + return new Val(v.erase(k) > 0, TYPE_BOOL); + %} + +## Create an iterator for a set. Note that this makes a copy of the set +## internally to ensure the iterator is always valid. +## +## s: the set to iterate over. +## +## Returns: an iterator. +function Comm::set_iterator%(s: Comm::Data%): opaque of Comm::SetIterator + %{ + return new comm::SetIterator(s->AsRecordVal(), TYPE_TABLE, frame); + %} + +## Check if there are no more elements to iterate over. +## +## it: an iterator. +## +## Returns: true if there are no more elements to iterator over, i.e. +## the iterator is one-past-the-final-element. +function Comm::set_iterator_last%(it: opaque of Comm::SetIterator%): bool + %{ + auto set_it = static_cast(it); + return new Val(set_it->it == set_it->dat.end(), TYPE_BOOL); + %} + +## Advance an iterator. +## +## it: an iterator. +## +## Returns: true if the iterator, after advancing, still references an element +## in the collection. False if the iterator, after advancing, is +## one-past-the-final-element. +function Comm::set_iterator_next%(it: opaque of Comm::SetIterator%): bool + %{ + auto set_it = static_cast(it); + + if ( set_it->it == set_it->dat.end() ) + return new Val(false, TYPE_BOOL); + + ++set_it->it; + return new Val(set_it->it != set_it->dat.end(), TYPE_BOOL); + %} + +## Retrieve the data at an iterator's current position. +## +## it: an iterator. +## +## Returns: element in the collection that the iterator currently references. +function Comm::set_iterator_value%(it: opaque of Comm::SetIterator%): Comm::Data + %{ + auto set_it = static_cast(it); + auto rval = new RecordVal(BifType::Record::Comm::Data); + + if ( set_it->it == set_it->dat.end() ) + { + reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->Warning("attempt to retrieve value of invalid set iterator"); + reporter->PopLocation(); + return rval; + } + + rval->Assign(0, new comm::DataVal(*set_it->it)); + return rval; + %} + +## Create communication data of type "table". +function Comm::table_create%(%): Comm::Data + %{ + return comm::make_data_val(broker::table()); + %} + +## Remove all elements within a table. +## +## t: the table to clear. +## +## Returns: always true. +function Comm::table_clear%(t: Comm::Data%): bool + %{ + auto& v = comm::require_data_type(t->AsRecordVal(), + TYPE_TABLE, frame); + v.clear(); + return new Val(true, TYPE_BOOL); + %} + +## Get the number of elements within a table. +## +## t: the table to query. +## +## Returns: the number of elements in the table. +function Comm::table_size%(t: Comm::Data%): count + %{ + auto& v = comm::require_data_type(t->AsRecordVal(), + TYPE_TABLE, frame); + return new Val(static_cast(v.size()), TYPE_COUNT); + %} + +## Check if a table contains a particular key. +## +## t: the table to query. +## +## key: the key to check for existence. +## +## Returns: true if the key exists in the set. +function Comm::table_contains%(t: Comm::Data, key: Comm::Data%): bool + %{ + auto& v = comm::require_data_type(t->AsRecordVal(), + TYPE_TABLE, frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + return new Val(v.find(k) != v.end(), TYPE_BOOL); + %} + +## Insert a key-value pair into a table. +## +## t: the table to modify. +## +## key: the key at which to insert the value. +## +## val: the value to insert. +## +## Returns: true if the key-value pair was inserted, or false if the key +## already existed in the table. +function Comm::table_insert%(t: Comm::Data, key: Comm::Data, val: Comm::Data%): Comm::Data + %{ + auto& table = comm::require_data_type(t->AsRecordVal(), + TYPE_TABLE, frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + auto& v = comm::opaque_field_to_data(val->AsRecordVal(), frame); + + try + { + auto& prev = table.at(k); + auto rval = comm::make_data_val(move(prev)); + prev = v; + return rval; + } + catch (const std::out_of_range&) + { + table[k] = v; + return new RecordVal(BifType::Record::Comm::Data); + } + %} + +## Remove a key-value pair from a table. +## +## t: the table to modify. +## +## key: the key to remove from the table. +## +## Returns: the value associated with the key. If the key did not exist, then +## the optional field of the returned record is not set. +function Comm::table_remove%(t: Comm::Data, key: Comm::Data%): Comm::Data + %{ + auto& table = comm::require_data_type(t->AsRecordVal(), + TYPE_TABLE, frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + auto it = table.find(k); + + if ( it == table.end() ) + return new RecordVal(BifType::Record::Comm::Data); + else + { + auto rval = comm::make_data_val(move(it->second)); + table.erase(it); + return rval; + } + %} + +## Retrieve a value from a table. +## +## t: the table to query. +## +## key: the key to lookup. +## +## Returns: the value associated with the key. If the key did not exist, then +## the optional field of the returned record is not set. +function Comm::table_lookup%(t: Comm::Data, key: Comm::Data%): Comm::Data + %{ + auto& table = comm::require_data_type(t->AsRecordVal(), + TYPE_TABLE, frame); + auto& k = comm::opaque_field_to_data(key->AsRecordVal(), frame); + auto it = table.find(k); + + if ( it == table.end() ) + return new RecordVal(BifType::Record::Comm::Data); + else + return comm::make_data_val(it->second); + %} + +## Create an iterator for a table. Note that this makes a copy of the table +## internally to ensure the iterator is always valid. +## +## t: the table to iterate over. +## +## Returns: an iterator. +function Comm::table_iterator%(t: Comm::Data%): opaque of Comm::TableIterator + %{ + return new comm::TableIterator(t->AsRecordVal(), TYPE_TABLE, frame); + %} + +## Check if there are no more elements to iterate over. +## +## it: an iterator. +## +## Returns: true if there are no more elements to iterator over, i.e. +## the iterator is one-past-the-final-element. +function Comm::table_iterator_last%(it: opaque of Comm::TableIterator%): bool + %{ + auto ti = static_cast(it); + return new Val(ti->it == ti->dat.end(), TYPE_BOOL); + %} + +## Advance an iterator. +## +## it: an iterator. +## +## Returns: true if the iterator, after advancing, still references an element +## in the collection. False if the iterator, after advancing, is +## one-past-the-final-element. +function Comm::table_iterator_next%(it: opaque of Comm::TableIterator%): bool + %{ + auto ti = static_cast(it); + + if ( ti->it == ti->dat.end() ) + return new Val(false, TYPE_BOOL); + + ++ti->it; + return new Val(ti->it != ti->dat.end(), TYPE_BOOL); + %} + +## Retrieve the data at an iterator's current position. +## +## it: an iterator. +## +## Returns: element in the collection that the iterator currently references. +function Comm::table_iterator_value%(it: opaque of Comm::TableIterator%): Comm::TableItem + %{ + auto ti = static_cast(it); + auto rval = new RecordVal(BifType::Record::Comm::TableItem); + auto key_val = new RecordVal(BifType::Record::Comm::Data); + auto val_val = new RecordVal(BifType::Record::Comm::Data); + rval->Assign(0, key_val); + rval->Assign(1, val_val); + + if ( ti->it == ti->dat.end() ) + { + reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->Warning("attempt to retrieve value of invalid table iterator"); + reporter->PopLocation(); + return rval; + } + + key_val->Assign(0, new comm::DataVal(ti->it->first)); + val_val->Assign(0, new comm::DataVal(ti->it->second)); + return rval; + %} + +## Create communication data of type "vector". +function Comm::vector_create%(%): Comm::Data + %{ + return comm::make_data_val(broker::vector()); + %} + +## Remove all elements within a vector. +## +## v: the vector to clear. +## +## Returns: always true. +function Comm::vector_clear%(v: Comm::Data%): bool + %{ + auto& vec = comm::require_data_type(v->AsRecordVal(), + TYPE_VECTOR, frame); + vec.clear(); + return new Val(true, TYPE_BOOL); + %} + +## Get the number of elements within a vector. +## +## v: the vector to query. +## +## Returns: the number of elements in the vector. +function Comm::vector_size%(v: Comm::Data%): count + %{ + auto& vec = comm::require_data_type(v->AsRecordVal(), + TYPE_VECTOR, frame); + return new Val(static_cast(vec.size()), TYPE_COUNT); + %} + +## Insert an element into a vector at a particular position, possibly displacing +## existing elements (insertion always grows the size of the vector by one). +## +## v: the vector to modify. +## +## d: the element to insert. +## +## idx: the index at which to insert the data. If it is greater than the +## current size of the vector, the element is inserted at the end. +## +## Returns: always true. +function Comm::vector_insert%(v: Comm::Data, d: Comm::Data, idx: count%): bool + %{ + auto& vec = comm::require_data_type(v->AsRecordVal(), + TYPE_VECTOR, frame); + auto& item = comm::opaque_field_to_data(d->AsRecordVal(), frame); + idx = min(idx, static_cast(vec.size())); + vec.insert(vec.begin() + idx, item); + return new Val(true, TYPE_BOOL); + %} + +## Replace an element in a vector at a particular position. +## +## v: the vector to modify. +## +## d: the element to insert. +## +## idx: the index to replace. +## +## Returns: the value that was just evicted. If the index was larger than any +## valid index, the optional field of the returned record is not set. +function Comm::vector_replace%(v: Comm::Data, d: Comm::Data, idx: count%): Comm::Data + %{ + auto& vec = comm::require_data_type(v->AsRecordVal(), + TYPE_VECTOR, frame); + auto& item = comm::opaque_field_to_data(d->AsRecordVal(), frame); + + if ( idx >= vec.size() ) + return new RecordVal(BifType::Record::Comm::Data); + + auto rval = comm::make_data_val(move(vec[idx])); + vec[idx] = item; + return rval; + %} + +## Remove an element from a vector at a particular position. +## +## v: the vector to modify. +## +## idx: the index to remove. +## +## Returns: the value that was just evicted. If the index was larger than any +## valid index, the optional field of the returned record is not set. +function Comm::vector_remove%(v: Comm::Data, idx: count%): Comm::Data + %{ + auto& vec = comm::require_data_type(v->AsRecordVal(), + TYPE_VECTOR, frame); + + if ( idx >= vec.size() ) + return new RecordVal(BifType::Record::Comm::Data); + + auto rval = comm::make_data_val(move(vec[idx])); + vec.erase(vec.begin() + idx); + return rval; + %} + +## Lookup an element in a vector at a particular position. +## +## v: the vector to query. +## +## idx: the index to lookup. +## +## Returns: the value at the index. If the index was larger than any +## valid index, the optional field of the returned record is not set. +function Comm::vector_lookup%(v: Comm::Data, idx: count%): Comm::Data + %{ + auto& vec = comm::require_data_type(v->AsRecordVal(), + TYPE_VECTOR, frame); + + if ( idx >= vec.size() ) + return new RecordVal(BifType::Record::Comm::Data); + + return comm::make_data_val(vec[idx]); + %} + +## Create an iterator for a vector. Note that this makes a copy of the vector +## internally to ensure the iterator is always valid. +## +## v: the vector to iterate over. +## +## Returns: an iterator. +function Comm::vector_iterator%(v: Comm::Data%): opaque of Comm::VectorIterator + %{ + return new comm::VectorIterator(v->AsRecordVal(), TYPE_VECTOR, frame); + %} + +## Check if there are no more elements to iterate over. +## +## it: an iterator. +## +## Returns: true if there are no more elements to iterator over, i.e. +## the iterator is one-past-the-final-element. +function Comm::vector_iterator_last%(it: opaque of Comm::VectorIterator%): bool + %{ + auto vi = static_cast(it); + return new Val(vi->it == vi->dat.end(), TYPE_BOOL); + %} + +## Advance an iterator. +## +## it: an iterator. +## +## Returns: true if the iterator, after advancing, still references an element +## in the collection. False if the iterator, after advancing, is +## one-past-the-final-element. +function Comm::vector_iterator_next%(it: opaque of Comm::VectorIterator%): bool + %{ + auto vi = static_cast(it); + + if ( vi->it == vi->dat.end() ) + return new Val(false, TYPE_BOOL); + + ++vi->it; + return new Val(vi->it != vi->dat.end(), TYPE_BOOL); + %} + +## Retrieve the data at an iterator's current position. +## +## it: an iterator. +## +## Returns: element in the collection that the iterator currently references. +function Comm::vector_iterator_value%(it: opaque of Comm::VectorIterator%): Comm::Data + %{ + auto vi = static_cast(it); + auto rval = new RecordVal(BifType::Record::Comm::Data); + + if ( vi->it == vi->dat.end() ) + { + reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->Warning("attempt to retrieve value of invalid vector iterator"); + reporter->PopLocation(); + return rval; + } + + rval->Assign(0, new comm::DataVal(*vi->it)); + return rval; + %} + +## Create communication data of type "record". +## +## sz: the number of fields in the record. +## +## Returns: record data, with all fields uninitialized. +function Comm::record_create%(sz: count%): Comm::Data + %{ + return comm::make_data_val(broker::record(std::vector(sz))); + %} + +## Get the number of fields within a record. +## +## r: the record to query. +## +## Returns: the number of fields in the record. +function Comm::record_size%(r: Comm::Data%): count + %{ + auto& v = comm::require_data_type(r->AsRecordVal(), + TYPE_RECORD, frame); + return new Val(static_cast(v.fields.size()), TYPE_COUNT); + %} + +## Replace a field in a record at a particular position. +## +## t: the table to modify. +## +## d: the new field value to assign. +## +## idx: the index to replace. +## +## Returns: false if the index was larger than any valid index, else true. +function Comm::record_assign%(r: Comm::Data, d: Comm::Data, idx: count%): bool + %{ + auto& v = comm::require_data_type(r->AsRecordVal(), + TYPE_RECORD, frame); + auto& item = comm::opaque_field_to_data(d->AsRecordVal(), frame); + + if ( idx >= v.fields.size() ) + return new Val(false, TYPE_BOOL); + + v.fields[idx] = item; + return new Val(true, TYPE_BOOL); + %} + +## Lookup a field in a record at a particular position. +## +## r: the record to query. +## +## idx: the index to lookup. +## +## Returns: the value at the index. The optional field of the returned record +## may not be set if the field of the record has no value or if the +## the index was not valid. +function Comm::record_lookup%(r: Comm::Data, idx: count%): Comm::Data + %{ + auto& v = comm::require_data_type(r->AsRecordVal(), + TYPE_RECORD, frame); + + if ( idx >= v.size() ) + return new RecordVal(BifType::Record::Comm::Data); + + if ( ! v.fields[idx] ) + return new RecordVal(BifType::Record::Comm::Data); + + return comm::make_data_val(*v.fields[idx]); + %} + +## Create an iterator for a record. Note that this makes a copy of the record +## internally to ensure the iterator is always valid. +## +## r: the record to iterate over. +## +## Returns: an iterator. +function Comm::record_iterator%(r: Comm::Data%): opaque of Comm::RecordIterator + %{ + return new comm::RecordIterator(r->AsRecordVal(), TYPE_RECORD, frame); + %} + +## Check if there are no more elements to iterate over. +## +## it: an iterator. +## +## Returns: true if there are no more elements to iterator over, i.e. +## the iterator is one-past-the-final-element. +function Comm::record_iterator_last%(it: opaque of Comm::RecordIterator%): bool + %{ + auto ri = static_cast(it); + return new Val(ri->it == ri->dat.fields.end(), TYPE_BOOL); + %} + +## Advance an iterator. +## +## it: an iterator. +## +## Returns: true if the iterator, after advancing, still references an element +## in the collection. False if the iterator, after advancing, is +## one-past-the-final-element. +function Comm::record_iterator_next%(it: opaque of Comm::RecordIterator%): bool + %{ + auto ri = static_cast(it); + + if ( ri->it == ri->dat.fields.end() ) + return new Val(false, TYPE_BOOL); + + ++ri->it; + return new Val(ri->it != ri->dat.fields.end(), TYPE_BOOL); + %} + +## Retrieve the data at an iterator's current position. +## +## it: an iterator. +## +## Returns: element in the collection that the iterator currently references. +function Comm::record_iterator_value%(it: opaque of Comm::RecordIterator%): Comm::Data + %{ + auto ri = static_cast(it); + auto rval = new RecordVal(BifType::Record::Comm::Data); + + if ( ri->it == ri->dat.fields.end() ) + { + reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->Warning("attempt to retrieve value of invalid record iterator"); + reporter->PopLocation(); + return rval; + } + + if ( ! *ri->it ) + return rval; // field isn't set + + rval->Assign(0, new comm::DataVal(**ri->it)); + return rval; + %} diff --git a/src/comm/messaging.bif b/src/comm/messaging.bif new file mode 100644 index 0000000000..faa22c8126 --- /dev/null +++ b/src/comm/messaging.bif @@ -0,0 +1,211 @@ + +##! Functions for peering and various messaging patterns (e.g. print/log/event). + +%%{ +#include "comm/Manager.h" +#include "logging/Manager.h" +%%} + +module Comm; + +type Comm::SendFlags: record; + +type Comm::EventArgs: record; + +## Used to handle remote print messages from peers that call +## :bro:see:`Comm::print`. +event Comm::print_handler%(msg: string%); + +## Print a simple message to any interested peers. The receiver can use +## :bro:see:`Comm::print_handler` to handle messages. +## +## topic: a topic associated with the printed message. +## +## msg: the print message to send to peers. +## +## flags: tune the behavior of how the message is sent. +## +## Returns: true if the message is sent. +function Comm::print%(topic: string, msg: string, + flags: SendFlags &default = SendFlags()%): bool + %{ + auto rval = comm_mgr->Print(topic->CheckString(), msg->CheckString(), + flags); + return new Val(rval, TYPE_BOOL); + %} + +## Register interest in all peer print messages that use a certain topic prefix. +## use :bro:see:`Comm::print_handler` to handle received messages. +## +## topic_prefix: a prefix to match against remote message topics. +## e.g. an empty prefix matches everything and "a" matches +## "alice" and "amy" but not "bob". +## +## Returns: true if it's a new print subscription and it is now registered. +function Comm::subscribe_to_prints%(topic_prefix: string%): bool + %{ + auto rval = comm_mgr->SubscribeToPrints(topic_prefix->CheckString()); + return new Val(rval, TYPE_BOOL); + %} + +## Unregister interest in all peer print messages that use a topic prefix. +## +## topic_prefix: a prefix previously supplied to a successful call to +## :bro:see:`Comm::subscribe_to_prints`. +## +## Returns: true if interest in the topic prefix is no longer advertised. +function Comm::unsubscribe_to_prints%(topic_prefix: string%): bool + %{ + auto rval = comm_mgr->UnsubscribeToPrints(topic_prefix->CheckString()); + return new Val(rval, TYPE_BOOL); + %} + +## Create a data structure that may be used to send a remote event via +## :bro:see:`Comm::event`. +## +## args: an event, followed by a list of argument values that may be used +## to call it. +## +## Returns: opaque communication data that may be used to send a remote event. +function Comm::event_args%(...%): Comm::EventArgs + %{ + auto rval = comm_mgr->MakeEventArgs(@ARGS@); + return rval; + %} + +## Send an event to any interested peers. +## +## topic: a topic associated with the event message. +## +## args: event arguments as made by :bro:see:`Comm::event_args`. +## +## flags: tune the behavior of how the message is sent. +## +## Returns: true if the message is sent. +function Comm::event%(topic: string, args: Comm::EventArgs, + flags: SendFlags &default = SendFlags()%): bool + %{ + auto rval = comm_mgr->Event(topic->CheckString(), args->AsRecordVal(), + flags); + return new Val(rval, TYPE_BOOL); + %} + +## Automatically send an event to any interested peers whenever it is +## locally dispatched (e.g. using "event my_event(...);" in a script). +## +## topic: a topic string associated with the event message. +## Peers advertise interest by registering a subscription to some prefix +## of this topic name. +## +## ev: a Bro event value. +## +## flags: tune the behavior of how the message is send. +## +## Returns: true if automatic event sending is now enabled. +function Comm::auto_event%(topic: string, ev: any, + flags: SendFlags &default = SendFlags()%): bool + %{ + auto rval = comm_mgr->AutoEvent(topic->CheckString(), ev, flags); + return new Val(rval, TYPE_BOOL); + %} + +## Stop automatically sending an event to peers upon local dispatch. +## +## topic: a topic originally given to :bro:see:`Comm::auto_event`. +## +## ev: an event originally given to :bro:see:`Comm::auto_event`. +## +## Returns: true if automatic events will no occur for the topic/event pair. +function Comm::auto_event_stop%(topic: string, ev: any%): bool + %{ + auto rval = comm_mgr->AutoEventStop(topic->CheckString(), ev); + return new Val(rval, TYPE_BOOL); + %} + +## Register interest in all peer event messages that use a certain topic prefix. +## +## topic_prefix: a prefix to match against remote message topics. +## e.g. an empty prefix matches everything and "a" matches +## "alice" and "amy" but not "bob". +## +## Returns: true if it's a new event subscription and it is now registered. +function Comm::subscribe_to_events%(topic_prefix: string%): bool + %{ + auto rval = comm_mgr->SubscribeToEvents(topic_prefix->CheckString()); + return new Val(rval, TYPE_BOOL); + %} + +## Unregister interest in all peer event messages that use a topic prefix. +## +## topic_prefix: a prefix previously supplied to a successful call to +## :bro:see:`Comm::subscribe_to_events`. +## +## Returns: true if interest in the topic prefix is no longer advertised. +function Comm::unsubscribe_to_events%(topic_prefix: string%): bool + %{ + auto rval = comm_mgr->UnsubscribeToEvents(topic_prefix->CheckString()); + return new Val(rval, TYPE_BOOL); + %} + +## Enable remote logs for a given log stream. +## +## id: the log stream to enable remote logs for. +## +## flags: tune the behavior of how log entry messages are sent. +## +## Returns: true if remote logs are enabled for the stream. +function +Comm::enable_remote_logs%(id: Log::ID, + flags: SendFlags &default = SendFlags()%): bool + %{ + auto rval = log_mgr->EnableRemoteLogs(id->AsEnumVal(), + comm::Manager::send_flags_to_int(flags)); + return new Val(rval, TYPE_BOOL); + %} + +## Disable remote logs for a given log stream. +## +## id: the log stream to disable remote logs for. +## +## Returns: true if remote logs are disabled for the stream. +function Comm::disable_remote_logs%(id: Log::ID%): bool + %{ + auto rval = log_mgr->DisableRemoteLogs(id->AsEnumVal()); + return new Val(rval, TYPE_BOOL); + %} + +## Returns: true if remote logs are enabled for the given stream. +function Comm::remote_logs_enabled%(id: Log::ID%): bool + %{ + auto rval = log_mgr->RemoteLogsAreEnabled(id->AsEnumVal()); + return new Val(rval, TYPE_BOOL); + %} + +## Register interest in all peer log messages that use a certain topic prefix. +## Logs are implicitly sent with topic "bro/log/" and the +## receiving side processes them through the logging framework as usual. +## +## topic_prefix: a prefix to match against remote message topics. +## e.g. an empty prefix matches everything and "a" matches +## "alice" and "amy" but not "bob". +## +## Returns: true if it's a new log subscription and it is now registered. +function Comm::subscribe_to_logs%(topic_prefix: string%): bool + %{ + auto rval = comm_mgr->SubscribeToLogs(topic_prefix->CheckString()); + return new Val(rval, TYPE_BOOL); + %} + +## Unregister interest in all peer log messages that use a topic prefix. +## Logs are implicitly sent with topic "bro/log/" and the +## receiving side processes them through the logging framework as usual. +## +## topic_prefix: a prefix previously supplied to a successful call to +## :bro:see:`Comm::subscribe_to_logs`. +## +## Returns: true if interest in the topic prefix is no longer advertised. +function Comm::unsubscribe_to_logs%(topic_prefix: string%): bool + %{ + auto rval = comm_mgr->UnsubscribeToLogs(topic_prefix->CheckString()); + return new Val(rval, TYPE_BOOL); + %} diff --git a/src/comm/store.bif b/src/comm/store.bif new file mode 100644 index 0000000000..02aa5b8564 --- /dev/null +++ b/src/comm/store.bif @@ -0,0 +1,590 @@ + +##! Functions to interface with broker's distributed data store. + +%%{ +#include "comm/Manager.h" +#include "comm/Store.h" +#include "comm/Data.h" +#include "Trigger.h" +%%} + +module Store; + +type Store::ExpiryTime: record; + +type Store::QueryResult: record; + +type Store::BackendOptions: record; + +## Enumerates the possible storage backends. +enum BackendType %{ + MEMORY, + SQLITE, + ROCKSDB, +%} + +## Create a master data store which contains key-value pairs. +## +## id: a unique name for the data store. +## +## b: the storage backend to use. +## +## options: tunes how some storage backends operate. +## +## Returns: a handle to the data store. +function Store::create_master%(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions()%): opaque of Store::Handle + %{ + auto id_str = id->CheckString(); + auto type = comm::StoreType::MASTER; + auto rval = comm_mgr->LookupStore(id_str, type); + + if ( rval ) + { + Ref(rval); + return rval; + } + + rval = new comm::StoreHandleVal(id_str, type, + static_cast(b->AsEnum()), + options->AsRecordVal()); + assert(comm_mgr->AddStore(rval)); + return rval; + %} + +## Create a clone of a master data store which may live with a remote peer. +## A clone automatically synchronizes to the master by automatically receiving +## modifications and applying them locally. Direct modifications are not +## possible, they must be sent through the master store, which then +## automatically broadcasts the changes out to clones. But queries may be made +## directly against the local cloned copy, which may be resolved quicker than +## reaching out to a remote master store. +## +## id: the unique name which identifies the master data store. +## +## b: the storage backend to use. +## +## options: tunes how some storage backends operate. +## +## resync: the interval at which to re-attempt synchronizing with the master +## store should the connection be lost. If the clone has not yet +## synchronized for the first time, updates and queries queue up until +## the synchronization completes. After, if the connection to the +## master store is lost, queries continue to use the clone's version, +## but updates will be lost until the master is once again available. +## +## Returns: a handle to the data store. +function Store::create_clone%(id: string, b: BackendType &default = MEMORY, + options: BackendOptions &default = BackendOptions(), + resync: interval &default = 1sec%): opaque of Store::Handle + %{ + auto id_str = id->CheckString(); + auto type = comm::StoreType::CLONE; + auto rval = comm_mgr->LookupStore(id_str, type); + + if ( rval ) + { + Ref(rval); + return rval; + } + + rval = new comm::StoreHandleVal(id_str, type, + static_cast(b->AsEnum()), + options->AsRecordVal(), + std::chrono::duration(resync)); + assert(comm_mgr->AddStore(rval)); + return rval; + %} + +## Create a frontend interface to an existing master data store that allows +## querying and updating its contents. +## +## id: the unique name which identifies the master data store. +## +## Returns: a handle to the data store. +function Store::create_frontend%(id: string%): opaque of Store::Handle + %{ + auto id_str = id->CheckString(); + auto type = comm::StoreType::FRONTEND; + auto rval = comm_mgr->LookupStore(id_str, type); + + if ( rval ) + { + Ref(rval); + return rval; + } + + rval = new comm::StoreHandleVal(id_str, type, {}, nullptr); + assert(comm_mgr->AddStore(rval)); + return rval; + %} + +## Close a data store. +## +## h: a data store handle. +## +## Returns: true if store was valid and is now closed. The handle can no +## longer be used for data store operations. +function Store::close_by_handle%(h: opaque of Store::Handle%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + return new Val(comm_mgr->CloseStore(handle->store->id(), + handle->store_type), TYPE_BOOL); + %} + +########################### +# non-blocking update API # +########################### + +## Insert a key-value pair in to the store. +## +## h: the handle of the store to modify. +## +## k: the key to insert. +## +## v: the value to insert. +## +## e: the expiration time of the key-value pair. +## +## Returns: false if the store handle was not valid. +function Store::insert%(h: opaque of Store::Handle, + k: Comm::Data, v: Comm::Data, + e: Store::ExpiryTime &default = Store::ExpiryTime()%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + auto& val = comm::opaque_field_to_data(v->AsRecordVal(), frame); + + using broker::store::expiration_time; + broker::util::optional expiry; + + auto abs_expiry_val = e->AsRecordVal()->Lookup(0); + + if ( abs_expiry_val ) + { + expiry = expiration_time(abs_expiry_val->AsTime()); + handle->store->insert(key, val, expiry); + return new Val(true, TYPE_BOOL); + } + + auto rel_expiry_val = e->AsRecordVal()->Lookup(1); + + if ( rel_expiry_val ) + { + auto ct = broker::time_point::now().value; + expiry = expiration_time(rel_expiry_val->AsInterval(), ct); + handle->store->insert(key, val, expiry); + return new Val(true, TYPE_BOOL); + } + + handle->store->insert(key, val, expiry); + return new Val(true, TYPE_BOOL); + %} + +## Remove a key-value pair from the store. +## +## h: the handle of the store to modify. +## +## k: the key to remove. +## +## Returns: false if the store handle was not valid. +function Store::erase%(h: opaque of Store::Handle, k: Comm::Data%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + handle->store->erase(key); + return new Val(true, TYPE_BOOL); + %} + +## Remove all key-value pairs from the store. +## +## h: the handle of the store to modify. +## +## Returns: false if the store handle was not valid. +function Store::clear%(h: opaque of Store::Handle%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + handle->store->clear(); + return new Val(true, TYPE_BOOL); + %} + +## Increment an integer value in a data store. +## +## h: the handle of the store to modify. +## +## k: the key whose associated value is to be modified. +## +## by: the amount to increment the value by. A non-existent key will first +## create it with an implicit value of zero before incrementing. +## +## Returns: false if the store handle was not valid. +function Store::increment%(h: opaque of Store::Handle, + k: Comm::Data, by: int &default = +1%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + handle->store->increment(key, by); + return new Val(true, TYPE_BOOL); + %} + +## Decrement an integer value in a data store. +## +## h: the handle of the store to modify. +## +## k: the key whose associated value is to be modified. +## +## by: the amount to decrement the value by. A non-existent key will first +## create it with an implicit value of zero before decrementing. +## +## Returns: false if the store handle was not valid. +function Store::decrement%(h: opaque of Store::Handle, + k: Comm::Data, by: int &default = +1%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + handle->store->decrement(key, by); + return new Val(true, TYPE_BOOL); + %} + +## Add an element to a set value in a data store. +## +## h: the handle of the store to modify. +## +## k: the key whose associated value is to be modified. +## +## element: the element to add to the set. A non-existent key will first +## create it with an implicit empty set value before modifying. +## +## Returns: false if the store handle was not valid. +function Store::add_to_set%(h: opaque of Store::Handle, + k: Comm::Data, element: Comm::Data%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + auto& ele = comm::opaque_field_to_data(element->AsRecordVal(), frame); + handle->store->add_to_set(key, ele); + return new Val(true, TYPE_BOOL); + %} + +## Remove an element from a set value in a data store. +## +## h: the handle of the store to modify. +## +## k: the key whose associated value is to be modified. +## +## element: the element to remove from the set. A non-existent key will +## implicitly create an empty set value associated with the key. +## +## Returns: false if the store handle was not valid. +function Store::remove_from_set%(h: opaque of Store::Handle, + k: Comm::Data, element: Comm::Data%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + auto& ele = comm::opaque_field_to_data(element->AsRecordVal(), frame); + handle->store->remove_from_set(key, ele); + return new Val(true, TYPE_BOOL); + %} + +## Add a new item to the head of a vector value in a data store. +## +## h: the handle of store to modify. +## +## k: the key whose associated value is to be modified. +## +## item: the element to insert in to the vector. A non-existent key will first +## create empty vector value before modifying. +## +## Returns: the handle of store to modify. +function Store::push_left%(h: opaque of Store::Handle, k: Comm::Data, + items: Comm::DataVector%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + broker::vector items_vector; + auto items_vv = items->AsVector(); + + for ( auto i = 0u; i < items_vv->size(); ++i ) + { + auto& item = comm::opaque_field_to_data((*items_vv)[i]->AsRecordVal(), + frame); + items_vector.emplace_back(item); + } + + handle->store->push_left(key, move(items_vector)); + return new Val(true, TYPE_BOOL); + %} + +## Add a new item to the tail of a vector value in a data store. +## +## h: the handle of store to modify. +## +## k: the key whose associated value is to be modified. +## +## item: the element to insert in to the vector. A non-existent key will first +## create empty vector value before modifying. +## +## Returns: the handle of store to modify. +function Store::push_right%(h: opaque of Store::Handle, k: Comm::Data, + items: Comm::DataVector%): bool + %{ + auto handle = static_cast(h); + + if ( ! handle->store ) + return new Val(false, TYPE_BOOL); + + auto& key = comm::opaque_field_to_data(k->AsRecordVal(), frame); + broker::vector items_vector; + auto items_vv = items->AsVector(); + + for ( auto i = 0u; i < items_vv->size(); ++i ) + { + auto& item = comm::opaque_field_to_data((*items_vv)[i]->AsRecordVal(), + frame); + items_vector.emplace_back(item); + } + + handle->store->push_right(key, move(items_vector)); + return new Val(true, TYPE_BOOL); + %} + +########################## +# non-blocking query API # +########################## + +%%{ +static bool prepare_for_query(Val* opaque, Frame* frame, + comm::StoreHandleVal** handle, + double* timeout, + comm::StoreQueryCallback** cb) + { + *handle = static_cast(opaque); + + if ( ! (*handle)->store ) + return false; + + Trigger* trigger = frame->GetTrigger(); + + if ( ! trigger ) + { + reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->Error("Store queries can only be called inside when-condition"); + reporter->PopLocation(); + return false; + } + + *timeout = trigger->TimeoutValue(); + + if ( *timeout < 0 ) + { + reporter->PushLocation(frame->GetCall()->GetLocationInfo()); + reporter->Error("Store queries must specify a timeout block"); + reporter->PopLocation(); + return false; + } + + frame->SetDelayed(); + trigger->Hold(); + *cb = new comm::StoreQueryCallback(trigger, frame->GetCall(), + (*handle)->store->id(), + (*handle)->store_type); + comm_mgr->TrackStoreQuery(*cb); + return true; + } + +%%} + +## Pop the head of a data store vector value. +## +## h: the handle of the store to query. +## +## k: the key associated with the vector to modify. +## +## Returns: the result of the query. +function Store::pop_left%(h: opaque of Store::Handle, + k: Comm::Data%): Store::QueryResult + %{ + if ( ! comm_mgr->Enabled() ) + return comm::query_result(); + + Val* key = k->AsRecordVal()->Lookup(0); + + if ( ! key ) + return comm::query_result(); + + double timeout; + comm::StoreQueryCallback* cb; + comm::StoreHandleVal* handle; + + if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) ) + return comm::query_result(); + + handle->store->pop_left(static_cast(key)->data, + std::chrono::duration(timeout), cb); + return 0; + %} + +## Pop the tail of a data store vector value. +## +## h: the handle of the store to query. +## +## k: the key associated with the vector to modify. +## +## Returns: the result of the query. +function Store::pop_right%(h: opaque of Store::Handle, + k: Comm::Data%): Store::QueryResult + %{ + if ( ! comm_mgr->Enabled() ) + return comm::query_result(); + + Val* key = k->AsRecordVal()->Lookup(0); + + if ( ! key ) + return comm::query_result(); + + double timeout; + comm::StoreQueryCallback* cb; + comm::StoreHandleVal* handle; + + if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) ) + return comm::query_result(); + + handle->store->pop_right(static_cast(key)->data, + std::chrono::duration(timeout), cb); + return 0; + %} + +## Lookup the value associated with a key in a data store. +## +## h: the handle of the store to query. +## +## k: the key to lookup. +## +## Returns: the result of the query. +function Store::lookup%(h: opaque of Store::Handle, + k: Comm::Data%): Store::QueryResult + %{ + if ( ! comm_mgr->Enabled() ) + return comm::query_result(); + + Val* key = k->AsRecordVal()->Lookup(0); + + if ( ! key ) + return comm::query_result(); + + double timeout; + comm::StoreQueryCallback* cb; + comm::StoreHandleVal* handle; + + if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) ) + return comm::query_result(); + + handle->store->lookup(static_cast(key)->data, + std::chrono::duration(timeout), cb); + return 0; + %} + +## Check if a data store contains a given key. +## +## h: the handle of the store to query. +## +## k: the key to check for existence. +## +## Returns: the result of the query (uses :bro:see:`Comm::BOOL`). +function Store::exists%(h: opaque of Store::Handle, + k: Comm::Data%): Store::QueryResult + %{ + if ( ! comm_mgr->Enabled() ) + return comm::query_result(); + + Val* key = k->AsRecordVal()->Lookup(0); + + if ( ! key ) + return comm::query_result(); + + double timeout; + comm::StoreQueryCallback* cb; + comm::StoreHandleVal* handle; + + if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) ) + return comm::query_result(); + + handle->store->exists(static_cast(key)->data, + std::chrono::duration(timeout), cb); + return 0; + %} + +## Retrieve all keys in a data store. +## +## h: the handle of the store to query. +## +## Returns: the result of the query (uses :bro:see:`Comm::VECTOR`). +function Store::keys%(h: opaque of Store::Handle%): Store::QueryResult + %{ + double timeout; + comm::StoreQueryCallback* cb; + comm::StoreHandleVal* handle; + + if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) ) + return comm::query_result(); + + handle->store->keys(std::chrono::duration(timeout), cb); + return 0; + %} + +## Get the number of key-value pairs in a data store. +## +## h: the handle of the store to query. +## +## Returns: the result of the query (uses :bro:see:`Comm::COUNT`). +function Store::size%(h: opaque of Store::Handle%): Store::QueryResult + %{ + if ( ! comm_mgr->Enabled() ) + return comm::query_result(); + + double timeout; + comm::StoreQueryCallback* cb; + comm::StoreHandleVal* handle; + + if ( ! prepare_for_query(h, frame, &handle, &timeout, &cb) ) + return comm::query_result(); + + handle->store->size(std::chrono::duration(timeout), cb); + return 0; + %} diff --git a/src/logging/Manager.cc b/src/logging/Manager.cc index 1fe5db3b26..d6d7fbb908 100644 --- a/src/logging/Manager.cc +++ b/src/logging/Manager.cc @@ -16,6 +16,10 @@ #include "WriterBackend.h" #include "logging.bif.h" +#ifdef ENABLE_BROKER +#include "comm/Manager.h" +#endif + using namespace logging; struct Manager::Filter { @@ -69,6 +73,11 @@ struct Manager::Stream { WriterMap writers; // Writers indexed by id/path pair. +#ifdef ENABLE_BROKER + bool enable_remote; + int remote_flags; +#endif + ~Stream(); }; @@ -287,6 +296,11 @@ bool Manager::CreateStream(EnumVal* id, RecordVal* sval) streams[idx]->event = event ? event_registry->Lookup(event->Name()) : 0; streams[idx]->columns = columns->Ref()->AsRecordType(); +#ifdef ENABLE_BROKER + streams[idx]->enable_remote = internal_val("Log::enable_remote_logging")->AsBool(); + streams[idx]->remote_flags = broker::PEERS; +#endif + DBG_LOG(DBG_LOGGING, "Created new logging stream '%s', raising event %s", streams[idx]->name.c_str(), event ? streams[idx]->event->Name() : ""); @@ -828,6 +842,11 @@ bool Manager::Write(EnumVal* id, RecordVal* columns) #endif } +#ifdef ENABLE_BROKER + if ( stream->enable_remote ) + comm_mgr->Log(id, columns, stream->remote_flags); +#endif + Unref(columns); if ( error ) @@ -1206,6 +1225,53 @@ void Manager::Terminate() } } +#ifdef ENABLE_BROKER + +bool Manager::EnableRemoteLogs(EnumVal* stream_id, int flags) + { + auto stream = FindStream(stream_id); + + if ( ! stream ) + return false; + + stream->enable_remote = true; + stream->remote_flags = flags; + return true; + } + +bool Manager::DisableRemoteLogs(EnumVal* stream_id) + { + auto stream = FindStream(stream_id); + + if ( ! stream ) + return false; + + stream->enable_remote = false; + return true; + } + +bool Manager::RemoteLogsAreEnabled(EnumVal* stream_id) + { + auto stream = FindStream(stream_id); + + if ( ! stream ) + return false; + + return stream->enable_remote; + } + +RecordType* Manager::StreamColumns(EnumVal* stream_id) + { + auto stream = FindStream(stream_id); + + if ( ! stream ) + return nullptr; + + return stream->columns; + } + +#endif + // Timer which on dispatching rotates the filter. class RotationTimer : public Timer { public: diff --git a/src/logging/Manager.h b/src/logging/Manager.h index b8264927a3..5d3372fb9b 100644 --- a/src/logging/Manager.h +++ b/src/logging/Manager.h @@ -157,6 +157,34 @@ public: */ void Terminate(); +#ifdef ENABLE_BROKER + /** + * Enable remote logs for a given stream. + * @param stream_id the stream to enable remote logs for. + * @param flags tune behavior of how log entries are sent to peer endpoints. + * @return true if remote logs are enabled. + */ + bool EnableRemoteLogs(EnumVal* stream_id, int flags); + + /** + * Disable remote logs for a given stream. + * @param stream_id the stream to disable remote logs for. + * @return true if remote logs are disabled. + */ + bool DisableRemoteLogs(EnumVal* stream_id); + + /** + * @return true if remote logs are enabled for a given stream. + */ + bool RemoteLogsAreEnabled(EnumVal* stream_id); + + /** + * @return the type which corresponds to the columns in a log entry for + * a given log stream. + */ + RecordType* StreamColumns(EnumVal* stream_id); +#endif + protected: friend class WriterFrontend; friend class RotationFinishedMessage; diff --git a/src/main.cc b/src/main.cc index 15aea3d3fe..13552bbbda 100644 --- a/src/main.cc +++ b/src/main.cc @@ -63,6 +63,10 @@ extern "C" void OPENSSL_add_all_algorithms_conf(void); #include "3rdparty/sqlite3.h" +#ifdef ENABLE_BROKER +#include +#endif + Brofiler brofiler; #ifndef HAVE_STRSEP @@ -81,9 +85,6 @@ int perftools_leaks = 0; int perftools_profile = 0; #endif -const char* prog; -char* writefile = 0; -name_list prefixes; DNS_Mgr* dns_mgr; TimerMgr* timer_mgr; logging::Manager* log_mgr = 0; @@ -94,6 +95,13 @@ analyzer::Manager* analyzer_mgr = 0; file_analysis::Manager* file_mgr = 0; broxygen::Manager* broxygen_mgr = 0; iosource::Manager* iosource_mgr = 0; +#ifdef ENABLE_BROKER +comm::Manager* comm_mgr = 0; +#endif + +const char* prog; +char* writefile = 0; +name_list prefixes; Stmt* stmts; EventHandlerPtr net_done = 0; RuleMatcher* rule_matcher = 0; @@ -851,6 +859,10 @@ int main(int argc, char** argv) input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); +#ifdef ENABLE_BROKER + comm_mgr = new comm::Manager(); +#endif + plugin_mgr->InitPreScript(); analyzer_mgr->InitPreScript(); file_mgr->InitPreScript(); diff --git a/testing/btest/Baseline/comm.clone_store/clone.clone.out b/testing/btest/Baseline/comm.clone_store/clone.clone.out new file mode 100644 index 0000000000..8a7c89a19b --- /dev/null +++ b/testing/btest/Baseline/comm.clone_store/clone.clone.out @@ -0,0 +1,5 @@ +clone keys, [status=Store::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] +lookup, one, [status=Store::SUCCESS, result=[d=broker::data{111}]] +lookup, two, [status=Store::SUCCESS, result=[d=broker::data{222}]] +lookup, myset, [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup, myvec, [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] diff --git a/testing/btest/Baseline/comm.clone_store/master.master.out b/testing/btest/Baseline/comm.clone_store/master.master.out new file mode 100644 index 0000000000..e69de29bb2 diff --git a/testing/btest/Baseline/comm.connection_updates/recv.recv.out b/testing/btest/Baseline/comm.connection_updates/recv.recv.out new file mode 100644 index 0000000000..3f2a1a9670 --- /dev/null +++ b/testing/btest/Baseline/comm.connection_updates/recv.recv.out @@ -0,0 +1,2 @@ +Comm::incoming_connection_established, connector +Comm::incoming_connection_broken, connector diff --git a/testing/btest/Baseline/comm.connection_updates/send.send.out b/testing/btest/Baseline/comm.connection_updates/send.send.out new file mode 100644 index 0000000000..e23422e320 --- /dev/null +++ b/testing/btest/Baseline/comm.connection_updates/send.send.out @@ -0,0 +1 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp, listener diff --git a/testing/btest/Baseline/comm.data/out b/testing/btest/Baseline/comm.data/out new file mode 100644 index 0000000000..eea78d39a2 --- /dev/null +++ b/testing/btest/Baseline/comm.data/out @@ -0,0 +1,99 @@ +Comm::BOOL +Comm::INT +Comm::COUNT +Comm::DOUBLE +Comm::STRING +Comm::ADDR +Comm::SUBNET +Comm::PORT +Comm::TIME +Comm::INTERVAL +Comm::ENUM +Comm::SET +Comm::TABLE +Comm::VECTOR +Comm::RECORD +*************************** +T +F +1 +0 +-1 +1 +0 +1.1 +-11.1 +hello +1.2.3.4 +192.168.0.0/16 +22/tcp +42.0 +180.0 +Comm::BOOL +*************************** +{ +two, +one, +three +} +0 +T +1 +T +F +T +2 +T +1 +F +{ +bye +} +0 +*************************** +{ +[two] = 2, +[one] = 1, +[three] = 3 +} +0 +[d=] +1 +T +42 +F +[d=] +2 +[d=broker::data{7}] +2 +37 +[d=broker::data{42}] +1 +*************************** +[zero, one, two] +0 +T +T +T +T +[hi, salutations, hello, greetings] +4 +[d=broker::data{hello}] +[d=broker::data{bah}] +[d=broker::data{hi}] +[hi, salutations, bah, greetings] +[d=broker::data{bah}] +[hi, salutations, greetings] +3 +*************************** +[a=, b=bee, c=1] +[a=test, b=bee, c=1] +[a=test, b=testagain, c=1] +3 +T +T +T +[d=broker::data{hi}] +[d=broker::data{hello}] +[d=broker::data{37}] +3 diff --git a/testing/btest/Baseline/comm.master_store/out b/testing/btest/Baseline/comm.master_store/out new file mode 100644 index 0000000000..defdc9a3e1 --- /dev/null +++ b/testing/btest/Baseline/comm.master_store/out @@ -0,0 +1,14 @@ +lookup(two): [status=Store::SUCCESS, result=[d=broker::data{222}]] +lookup(four): [status=Store::SUCCESS, result=[d=]] +lookup(myset): [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup(one): [status=Store::SUCCESS, result=[d=broker::data{111}]] +lookup(myvec): [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +exists(one): [status=Store::SUCCESS, result=[d=broker::data{1}]] +exists(two): [status=Store::SUCCESS, result=[d=broker::data{0}]] +exists(myset): [status=Store::SUCCESS, result=[d=broker::data{1}]] +exists(four): [status=Store::SUCCESS, result=[d=broker::data{0}]] +pop_right(myvec): [status=Store::SUCCESS, result=[d=broker::data{omega}]] +pop_left(myvec): [status=Store::SUCCESS, result=[d=broker::data{delta}]] +keys: [status=Store::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] +size: [status=Store::SUCCESS, result=[d=broker::data{3}]] +size (after clear): [status=Store::SUCCESS, result=[d=broker::data{0}]] diff --git a/testing/btest/Baseline/comm.remote_event/recv.recv.out b/testing/btest/Baseline/comm.remote_event/recv.recv.out new file mode 100644 index 0000000000..7dab0284ea --- /dev/null +++ b/testing/btest/Baseline/comm.remote_event/recv.recv.out @@ -0,0 +1,6 @@ +got event msg, ping, 0 +got event msg, ping, 1 +got event msg, ping, 2 +got event msg, ping, 3 +got event msg, ping, 4 +got event msg, ping, 5 diff --git a/testing/btest/Baseline/comm.remote_event/send.send.out b/testing/btest/Baseline/comm.remote_event/send.send.out new file mode 100644 index 0000000000..0e529e08fc --- /dev/null +++ b/testing/btest/Baseline/comm.remote_event/send.send.out @@ -0,0 +1,11 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp +got event msg, pong, 0 +got auto event msg, ping, 0 +got event msg, pong, 1 +got auto event msg, ping, 1 +got event msg, pong, 2 +got auto event msg, ping, 2 +got event msg, pong, 3 +got auto event msg, ping, 3 +got event msg, pong, 4 +got auto event msg, ping, 4 diff --git a/testing/btest/Baseline/comm.remote_log/recv.recv.out b/testing/btest/Baseline/comm.remote_log/recv.recv.out new file mode 100644 index 0000000000..3e0957442d --- /dev/null +++ b/testing/btest/Baseline/comm.remote_log/recv.recv.out @@ -0,0 +1,6 @@ +wrote log, [msg=ping, num=0] +wrote log, [msg=ping, num=1] +wrote log, [msg=ping, num=2] +wrote log, [msg=ping, num=3] +wrote log, [msg=ping, num=4] +wrote log, [msg=ping, num=5] diff --git a/testing/btest/Baseline/comm.remote_log/recv.test.log b/testing/btest/Baseline/comm.remote_log/recv.test.log new file mode 100644 index 0000000000..0d6dae756c --- /dev/null +++ b/testing/btest/Baseline/comm.remote_log/recv.test.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2015-01-26-22-47-11 +#fields msg num +#types string count +ping 0 +ping 1 +ping 2 +ping 3 +ping 4 +ping 5 +#close 2015-01-26-22-47-11 diff --git a/testing/btest/Baseline/comm.remote_log/send.send.out b/testing/btest/Baseline/comm.remote_log/send.send.out new file mode 100644 index 0000000000..e2415290d6 --- /dev/null +++ b/testing/btest/Baseline/comm.remote_log/send.send.out @@ -0,0 +1 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/Baseline/comm.remote_log/send.test.log b/testing/btest/Baseline/comm.remote_log/send.test.log new file mode 100644 index 0000000000..0d6dae756c --- /dev/null +++ b/testing/btest/Baseline/comm.remote_log/send.test.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2015-01-26-22-47-11 +#fields msg num +#types string count +ping 0 +ping 1 +ping 2 +ping 3 +ping 4 +ping 5 +#close 2015-01-26-22-47-11 diff --git a/testing/btest/Baseline/comm.remote_print/recv.recv.out b/testing/btest/Baseline/comm.remote_print/recv.recv.out new file mode 100644 index 0000000000..6e5a37abbf --- /dev/null +++ b/testing/btest/Baseline/comm.remote_print/recv.recv.out @@ -0,0 +1,6 @@ +got print msg, ping 0 +got print msg, ping 1 +got print msg, ping 2 +got print msg, ping 3 +got print msg, ping 4 +got print msg, ping 5 diff --git a/testing/btest/Baseline/comm.remote_print/send.send.out b/testing/btest/Baseline/comm.remote_print/send.send.out new file mode 100644 index 0000000000..777afdc0d2 --- /dev/null +++ b/testing/btest/Baseline/comm.remote_print/send.send.out @@ -0,0 +1,6 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp +got print msg, pong 0 +got print msg, pong 1 +got print msg, pong 2 +got print msg, pong 3 +got print msg, pong 4 diff --git a/testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out b/testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out new file mode 100644 index 0000000000..8a7c89a19b --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.clone_store/clone.clone.out @@ -0,0 +1,5 @@ +clone keys, [status=Store::SUCCESS, result=[d=broker::data{[one, two, myset, myvec]}]] +lookup, one, [status=Store::SUCCESS, result=[d=broker::data{111}]] +lookup, two, [status=Store::SUCCESS, result=[d=broker::data{222}]] +lookup, myset, [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup, myvec, [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] diff --git a/testing/btest/Baseline/core.leaks.comm.data/bro..stdout b/testing/btest/Baseline/core.leaks.comm.data/bro..stdout new file mode 100644 index 0000000000..eea78d39a2 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.data/bro..stdout @@ -0,0 +1,99 @@ +Comm::BOOL +Comm::INT +Comm::COUNT +Comm::DOUBLE +Comm::STRING +Comm::ADDR +Comm::SUBNET +Comm::PORT +Comm::TIME +Comm::INTERVAL +Comm::ENUM +Comm::SET +Comm::TABLE +Comm::VECTOR +Comm::RECORD +*************************** +T +F +1 +0 +-1 +1 +0 +1.1 +-11.1 +hello +1.2.3.4 +192.168.0.0/16 +22/tcp +42.0 +180.0 +Comm::BOOL +*************************** +{ +two, +one, +three +} +0 +T +1 +T +F +T +2 +T +1 +F +{ +bye +} +0 +*************************** +{ +[two] = 2, +[one] = 1, +[three] = 3 +} +0 +[d=] +1 +T +42 +F +[d=] +2 +[d=broker::data{7}] +2 +37 +[d=broker::data{42}] +1 +*************************** +[zero, one, two] +0 +T +T +T +T +[hi, salutations, hello, greetings] +4 +[d=broker::data{hello}] +[d=broker::data{bah}] +[d=broker::data{hi}] +[hi, salutations, bah, greetings] +[d=broker::data{bah}] +[hi, salutations, greetings] +3 +*************************** +[a=, b=bee, c=1] +[a=test, b=bee, c=1] +[a=test, b=testagain, c=1] +3 +T +T +T +[d=broker::data{hi}] +[d=broker::data{hello}] +[d=broker::data{37}] +3 diff --git a/testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout b/testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout new file mode 100644 index 0000000000..defdc9a3e1 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.master_store/bro..stdout @@ -0,0 +1,14 @@ +lookup(two): [status=Store::SUCCESS, result=[d=broker::data{222}]] +lookup(four): [status=Store::SUCCESS, result=[d=]] +lookup(myset): [status=Store::SUCCESS, result=[d=broker::data{{a, c, d}}]] +lookup(one): [status=Store::SUCCESS, result=[d=broker::data{111}]] +lookup(myvec): [status=Store::SUCCESS, result=[d=broker::data{[delta, alpha, beta, gamma, omega]}]] +exists(one): [status=Store::SUCCESS, result=[d=broker::data{1}]] +exists(two): [status=Store::SUCCESS, result=[d=broker::data{0}]] +exists(myset): [status=Store::SUCCESS, result=[d=broker::data{1}]] +exists(four): [status=Store::SUCCESS, result=[d=broker::data{0}]] +pop_right(myvec): [status=Store::SUCCESS, result=[d=broker::data{omega}]] +pop_left(myvec): [status=Store::SUCCESS, result=[d=broker::data{delta}]] +keys: [status=Store::SUCCESS, result=[d=broker::data{[myvec, myset, one]}]] +size: [status=Store::SUCCESS, result=[d=broker::data{3}]] +size (after clear): [status=Store::SUCCESS, result=[d=broker::data{0}]] diff --git a/testing/btest/Baseline/core.leaks.comm.remote_event/recv.recv.out b/testing/btest/Baseline/core.leaks.comm.remote_event/recv.recv.out new file mode 100644 index 0000000000..7dab0284ea --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_event/recv.recv.out @@ -0,0 +1,6 @@ +got event msg, ping, 0 +got event msg, ping, 1 +got event msg, ping, 2 +got event msg, ping, 3 +got event msg, ping, 4 +got event msg, ping, 5 diff --git a/testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out b/testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out new file mode 100644 index 0000000000..0e529e08fc --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_event/send.send.out @@ -0,0 +1,11 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp +got event msg, pong, 0 +got auto event msg, ping, 0 +got event msg, pong, 1 +got auto event msg, ping, 1 +got event msg, pong, 2 +got auto event msg, ping, 2 +got event msg, pong, 3 +got auto event msg, ping, 3 +got event msg, pong, 4 +got auto event msg, ping, 4 diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/recv.recv.out b/testing/btest/Baseline/core.leaks.comm.remote_log/recv.recv.out new file mode 100644 index 0000000000..3e0957442d --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_log/recv.recv.out @@ -0,0 +1,6 @@ +wrote log, [msg=ping, num=0] +wrote log, [msg=ping, num=1] +wrote log, [msg=ping, num=2] +wrote log, [msg=ping, num=3] +wrote log, [msg=ping, num=4] +wrote log, [msg=ping, num=5] diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/recv.test.log b/testing/btest/Baseline/core.leaks.comm.remote_log/recv.test.log new file mode 100644 index 0000000000..4fe7790779 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_log/recv.test.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2015-02-12-17-33-13 +#fields msg num +#types string count +ping 0 +ping 1 +ping 2 +ping 3 +ping 4 +ping 5 +#close 2015-02-12-17-33-14 diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out b/testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out new file mode 100644 index 0000000000..e2415290d6 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_log/send.send.out @@ -0,0 +1 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp diff --git a/testing/btest/Baseline/core.leaks.comm.remote_log/send.test.log b/testing/btest/Baseline/core.leaks.comm.remote_log/send.test.log new file mode 100644 index 0000000000..884517b252 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_log/send.test.log @@ -0,0 +1,15 @@ +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path test +#open 2015-02-12-17-33-13 +#fields msg num +#types string count +ping 0 +ping 1 +ping 2 +ping 3 +ping 4 +ping 5 +#close 2015-02-12-17-33-15 diff --git a/testing/btest/Baseline/core.leaks.comm.remote_print/recv.recv.out b/testing/btest/Baseline/core.leaks.comm.remote_print/recv.recv.out new file mode 100644 index 0000000000..6e5a37abbf --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_print/recv.recv.out @@ -0,0 +1,6 @@ +got print msg, ping 0 +got print msg, ping 1 +got print msg, ping 2 +got print msg, ping 3 +got print msg, ping 4 +got print msg, ping 5 diff --git a/testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out b/testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out new file mode 100644 index 0000000000..777afdc0d2 --- /dev/null +++ b/testing/btest/Baseline/core.leaks.comm.remote_print/send.send.out @@ -0,0 +1,6 @@ +Comm::outgoing_connection_established, 127.0.0.1, 9999/tcp +got print msg, pong 0 +got print msg, pong 1 +got print msg, pong 2 +got print msg, pong 3 +got print msg, pong 4 diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index b94df659b4..7b144198ee 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-10-31-20-38-14 +#open 2015-02-03-22-47-13 #fields name #types string scripts/base/init-bare.bro @@ -14,6 +14,8 @@ scripts/base/init-bare.bro build/scripts/base/bif/reporter.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/event.bif.bro + scripts/base/frameworks/comm/__load__.bro + scripts/base/frameworks/comm/main.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -47,6 +49,10 @@ scripts/base/init-bare.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro + build/scripts/base/bif/comm.bif.bro + build/scripts/base/bif/data.bif.bro + build/scripts/base/bif/messaging.bif.bro + build/scripts/base/bif/store.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -115,4 +121,4 @@ scripts/base/init-bare.bro build/scripts/base/bif/plugins/Bro_SQLiteWriter.sqlite.bif.bro scripts/policy/misc/loaded-scripts.bro scripts/base/utils/paths.bro -#close 2014-10-31-20-38-14 +#close 2015-02-03-22-47-13 diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 67de0fc1dc..b102ad26a5 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -3,7 +3,7 @@ #empty_field (empty) #unset_field - #path loaded_scripts -#open 2014-10-31-20-38-48 +#open 2015-02-03-22-47-15 #fields name #types string scripts/base/init-bare.bro @@ -14,6 +14,8 @@ scripts/base/init-bare.bro build/scripts/base/bif/reporter.bif.bro build/scripts/base/bif/plugins/Bro_SNMP.types.bif.bro build/scripts/base/bif/event.bif.bro + scripts/base/frameworks/comm/__load__.bro + scripts/base/frameworks/comm/main.bro scripts/base/frameworks/logging/__load__.bro scripts/base/frameworks/logging/main.bro build/scripts/base/bif/logging.bif.bro @@ -47,6 +49,10 @@ scripts/base/init-bare.bro build/scripts/base/bif/bloom-filter.bif.bro build/scripts/base/bif/cardinality-counter.bif.bro build/scripts/base/bif/top-k.bif.bro + build/scripts/base/bif/comm.bif.bro + build/scripts/base/bif/data.bif.bro + build/scripts/base/bif/messaging.bif.bro + build/scripts/base/bif/store.bif.bro build/scripts/base/bif/plugins/__load__.bro build/scripts/base/bif/plugins/Bro_ARP.events.bif.bro build/scripts/base/bif/plugins/Bro_AYIYA.events.bif.bro @@ -247,4 +253,4 @@ scripts/base/init-default.bro scripts/base/misc/find-checksum-offloading.bro scripts/base/misc/find-filtered-trace.bro scripts/policy/misc/loaded-scripts.bro -#close 2014-10-31-20-38-48 +#close 2015-02-03-22-47-15 diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-connector_bro/output new file mode 100644 index 0000000000..1921e6596a --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-connector_bro/output @@ -0,0 +1,23 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +connecting-connector.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-listener_bro/output new file mode 100644 index 0000000000..7516680533 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_connecting-listener_bro/output @@ -0,0 +1,25 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +connecting-listener.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Comm::incoming_connection_broken(peer_name: string) + { + print "Comm::incoming_connection_broken", peer_name; + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-connector_bro/output new file mode 100644 index 0000000000..434e94d977 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-connector_bro/output @@ -0,0 +1,35 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +events-connector.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + Comm::auto_event("bro/event/my_auto_event", my_auto_event); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + Comm::event("bro/event/my_event", Comm::event_args(my_event, "hi", 0)); + event my_auto_event("stuff", 88); + Comm::event("bro/event/my_event", Comm::event_args(my_event, "...", 1)); + event my_auto_event("more stuff", 51); + Comm::event("bro/event/my_event", Comm::event_args(my_event, "bye", 2)); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-listener_bro/output new file mode 100644 index 0000000000..a8b7c133ff --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_events-listener_bro/output @@ -0,0 +1,41 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +events-listener.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; +global msg_count = 0; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event my_event(msg: string, c: count) + { + ++msg_count; + print "got my_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } + +event my_auto_event(msg: string, c: count) + { + ++msg_count; + print "got my_auto_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-connector_bro/output new file mode 100644 index 0000000000..ae8c3b4ec5 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-connector_bro/output @@ -0,0 +1,44 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +logs-connector.bro + +@load ./testlog + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; +redef Log::enable_local_logging = F; +redef Log::enable_remote_logging = F; +global n = 0; + +event bro_init() + { + Comm::enable(); + Comm::enable_remote_logs(Test::LOG); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event do_write() + { + if ( n == 6 ) + return; + + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + event do_write(); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-listener_bro/output new file mode 100644 index 0000000000..472229ea04 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_logs-listener_bro/output @@ -0,0 +1,29 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +logs-listener.bro + +@load ./testlog + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_logs("bro/log/Test::LOG"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + + if ( rec$num == 5 ) + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-connector_bro/output new file mode 100644 index 0000000000..b796155c59 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-connector_bro/output @@ -0,0 +1,30 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +printing-connector.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + Comm::print("bro/print/hi", "hello"); + Comm::print("bro/print/stuff", "..."); + Comm::print("bro/print/bye", "goodbye"); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-listener_bro/output new file mode 100644 index 0000000000..de6741d3c4 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_printing-listener_bro/output @@ -0,0 +1,30 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +printing-listener.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; +global msg_count = 0; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_prints("bro/print/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Comm::print_handler(msg: string) + { + ++msg_count; + print "got print message", msg; + + if ( msg_count == 3 ) + terminate(); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-connector_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-connector_bro/output new file mode 100644 index 0000000000..c5268417a6 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-connector_bro/output @@ -0,0 +1,57 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +stores-connector.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + h = Store::create_master("mystore"); + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + + when ( local res = Store::size(h) ) + { + print "master size", res; + event ready(); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1secs); + Comm::auto_event("bro/event/ready", ready); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-listener_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-listener_bro/output new file mode 100644 index 0000000000..38dc7ef34f --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_stores-listener_bro/output @@ -0,0 +1,47 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +stores-listener.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout", key; } + } + +event ready() + { + h = Store::create_clone("mystore"); + + when ( local res = Store::keys(h) ) + { + print "clone keys", res; + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/ready"); + Comm::listen(broker_port, "127.0.0.1"); + } diff --git a/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_testlog_bro/output b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_testlog_bro/output new file mode 100644 index 0000000000..e37b34c518 --- /dev/null +++ b/testing/btest/Baseline/doc.sphinx.include-doc_frameworks_comm_testlog_bro/output @@ -0,0 +1,23 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +testlog.bro + + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event bro_init() &priority=5 + { + Comm::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); + } diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 9519ffec5e..aaac3dec68 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -191,7 +191,7 @@ 0.000000 MetaHookPost CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::__create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::__write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Cluster::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Communication::LOG)) -> 0.000000 MetaHookPost CallFunction(Log::add_default_filter, (Conn::LOG)) -> @@ -285,8 +285,8 @@ 0.000000 MetaHookPost CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) -> 0.000000 MetaHookPost CallFunction(Log::create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -> -0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -> -0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) -> +0.000000 MetaHookPost CallFunction(Log::write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) -> 0.000000 MetaHookPost CallFunction(Notice::want_pp, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::build, ()) -> 0.000000 MetaHookPost CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) -> @@ -401,10 +401,12 @@ 0.000000 MetaHookPost LoadFile(./bro.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./broxygen.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./cardinality-counter.bif.bro) -> -1 +0.000000 MetaHookPost LoadFile(./comm.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./const.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./consts) -> -1 0.000000 MetaHookPost LoadFile(./consts.bro) -> -1 0.000000 MetaHookPost LoadFile(./contents) -> -1 +0.000000 MetaHookPost LoadFile(./data.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./dcc-send) -> -1 0.000000 MetaHookPost LoadFile(./entities) -> -1 0.000000 MetaHookPost LoadFile(./event.bif.bro) -> -1 @@ -425,6 +427,7 @@ 0.000000 MetaHookPost LoadFile(./main) -> -1 0.000000 MetaHookPost LoadFile(./main.bro) -> -1 0.000000 MetaHookPost LoadFile(./max) -> -1 +0.000000 MetaHookPost LoadFile(./messaging.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./min) -> -1 0.000000 MetaHookPost LoadFile(./mozilla-ca-list) -> -1 0.000000 MetaHookPost LoadFile(./netstats) -> -1 @@ -440,6 +443,7 @@ 0.000000 MetaHookPost LoadFile(./sftp) -> -1 0.000000 MetaHookPost LoadFile(./site) -> -1 0.000000 MetaHookPost LoadFile(./std-dev) -> -1 +0.000000 MetaHookPost LoadFile(./store.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./strings.bif.bro) -> -1 0.000000 MetaHookPost LoadFile(./sum) -> -1 0.000000 MetaHookPost LoadFile(./top-k.bif.bro) -> -1 @@ -474,6 +478,7 @@ 0.000000 MetaHookPost LoadFile(base<...>/analyzer.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/bro.bif) -> -1 0.000000 MetaHookPost LoadFile(base<...>/cluster) -> -1 +0.000000 MetaHookPost LoadFile(base<...>/comm) -> -1 0.000000 MetaHookPost LoadFile(base<...>/communication) -> -1 0.000000 MetaHookPost LoadFile(base<...>/conn) -> -1 0.000000 MetaHookPost LoadFile(base<...>/conn-ids) -> -1 @@ -730,7 +735,7 @@ 0.000000 MetaHookPre CallFunction(Log::__create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::__create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::__write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Cluster::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Communication::LOG)) 0.000000 MetaHookPre CallFunction(Log::add_default_filter, (Conn::LOG)) @@ -824,8 +829,8 @@ 0.000000 MetaHookPre CallFunction(Log::create_stream, (Weird::LOG, [columns=, ev=Weird::log_weird])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (X509::LOG, [columns=, ev=X509::log_x509])) 0.000000 MetaHookPre CallFunction(Log::create_stream, (mysql::LOG, [columns=, ev=MySQL::log_mysql])) -0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) -0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::default_path_func, (PacketFilter::LOG, , [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) +0.000000 MetaHookPre CallFunction(Log::write, (PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T])) 0.000000 MetaHookPre CallFunction(Notice::want_pp, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::build, ()) 0.000000 MetaHookPre CallFunction(PacketFilter::combine_filters, (ip or not ip, and, )) @@ -940,10 +945,12 @@ 0.000000 MetaHookPre LoadFile(./bro.bif.bro) 0.000000 MetaHookPre LoadFile(./broxygen.bif.bro) 0.000000 MetaHookPre LoadFile(./cardinality-counter.bif.bro) +0.000000 MetaHookPre LoadFile(./comm.bif.bro) 0.000000 MetaHookPre LoadFile(./const.bif.bro) 0.000000 MetaHookPre LoadFile(./consts) 0.000000 MetaHookPre LoadFile(./consts.bro) 0.000000 MetaHookPre LoadFile(./contents) +0.000000 MetaHookPre LoadFile(./data.bif.bro) 0.000000 MetaHookPre LoadFile(./dcc-send) 0.000000 MetaHookPre LoadFile(./entities) 0.000000 MetaHookPre LoadFile(./event.bif.bro) @@ -964,6 +971,7 @@ 0.000000 MetaHookPre LoadFile(./main) 0.000000 MetaHookPre LoadFile(./main.bro) 0.000000 MetaHookPre LoadFile(./max) +0.000000 MetaHookPre LoadFile(./messaging.bif.bro) 0.000000 MetaHookPre LoadFile(./min) 0.000000 MetaHookPre LoadFile(./mozilla-ca-list) 0.000000 MetaHookPre LoadFile(./netstats) @@ -979,6 +987,7 @@ 0.000000 MetaHookPre LoadFile(./sftp) 0.000000 MetaHookPre LoadFile(./site) 0.000000 MetaHookPre LoadFile(./std-dev) +0.000000 MetaHookPre LoadFile(./store.bif.bro) 0.000000 MetaHookPre LoadFile(./strings.bif.bro) 0.000000 MetaHookPre LoadFile(./sum) 0.000000 MetaHookPre LoadFile(./top-k.bif.bro) @@ -1013,6 +1022,7 @@ 0.000000 MetaHookPre LoadFile(base<...>/analyzer.bif) 0.000000 MetaHookPre LoadFile(base<...>/bro.bif) 0.000000 MetaHookPre LoadFile(base<...>/cluster) +0.000000 MetaHookPre LoadFile(base<...>/comm) 0.000000 MetaHookPre LoadFile(base<...>/communication) 0.000000 MetaHookPre LoadFile(base<...>/conn) 0.000000 MetaHookPre LoadFile(base<...>/conn-ids) @@ -1269,7 +1279,7 @@ 0.000000 | HookCallFunction Log::__create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::__create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::__create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::__write(PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Log::add_default_filter(Cluster::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Communication::LOG) 0.000000 | HookCallFunction Log::add_default_filter(Conn::LOG) @@ -1363,8 +1373,8 @@ 0.000000 | HookCallFunction Log::create_stream(Weird::LOG, [columns=, ev=Weird::log_weird]) 0.000000 | HookCallFunction Log::create_stream(X509::LOG, [columns=, ev=X509::log_x509]) 0.000000 | HookCallFunction Log::create_stream(mysql::LOG, [columns=, ev=MySQL::log_mysql]) -0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T]) -0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1424736245.843493, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::default_path_func(PacketFilter::LOG, , [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T]) +0.000000 | HookCallFunction Log::write(PacketFilter::LOG, [ts=1425338938.385247, node=bro, filter=ip or not ip, init=T, success=T]) 0.000000 | HookCallFunction Notice::want_pp() 0.000000 | HookCallFunction PacketFilter::build() 0.000000 | HookCallFunction PacketFilter::combine_filters(ip or not ip, and, ) diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 43f29d40a1..3c91872f5a 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,5 +1,5 @@ [btest] -TestDirs = doc bifs language core scripts istate coverage signatures plugins +TestDirs = doc bifs language core scripts istate coverage signatures plugins comm TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp @@ -25,3 +25,4 @@ TMPDIR=%(testbase)s/.tmp BRO_PROFILER_FILE=%(testbase)s/.tmp/script-coverage.XXXXXX BTEST_RST_FILTER=$SCRIPTS/rst-filter BRO_DNS_FAKE=1 +BROKER_PORT=9999/tcp diff --git a/testing/btest/comm/clone_store.bro b/testing/btest/comm/clone_store.bro new file mode 100644 index 0000000000..b4ddabc1e4 --- /dev/null +++ b/testing/btest/comm/clone_store.bro @@ -0,0 +1,112 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: btest-bg-run clone "bro -b ../clone.bro broker_port=$BROKER_PORT >clone.out" +# @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out" + +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone/clone.out +# @TEST-EXEC: btest-diff master/master.out + +@TEST-START-FILE clone.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout"; } + } + +event ready() + { + h = Store::create_clone("mystore"); + + when ( local res = Store::keys(h) ) + { + print "clone keys", res; + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/ready"); + Comm::listen(broker_port, "127.0.0.1"); + } + +@TEST-END-FILE + +@TEST-START-FILE master.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + h = Store::create_master("mystore"); + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + + when ( local res = Store::size(h) ) + { event ready(); } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1secs); + Comm::auto_event("bro/event/ready", ready); + } + +@TEST-END-FILE diff --git a/testing/btest/comm/connection_updates.bro b/testing/btest/comm/connection_updates.bro new file mode 100644 index 0000000000..67f66646c9 --- /dev/null +++ b/testing/btest/comm/connection_updates.bro @@ -0,0 +1,57 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff send/send.out + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name;; + } + +event Comm::incoming_connection_broken(peer_name: string) + { + print "Comm::incoming_connection_broken", peer_name;; + terminate(); + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name;; + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/comm/data.bro b/testing/btest/comm/data.bro new file mode 100644 index 0000000000..a7de41be7a --- /dev/null +++ b/testing/btest/comm/data.bro @@ -0,0 +1,222 @@ +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: btest-diff out + +type bro_set: set[string]; +type bro_table: table[string] of count; +type bro_vector: vector of string; + +type bro_record : record { + a: string &optional; + b: string &default = "bee"; + c: count; +}; + +function comm_record_to_bro_record_recurse(it: opaque of Comm::RecordIterator, + rval: bro_record, + idx: count): bro_record + { + if ( Comm::record_iterator_last(it) ) + return rval; + + local field_value = Comm::record_iterator_value(it); + + if ( field_value?$d ) + switch ( idx ) { + case 0: + rval$a = Comm::refine_to_string(field_value); + break; + case 1: + rval$b = Comm::refine_to_string(field_value); + break; + case 2: + rval$c = Comm::refine_to_count(field_value); + break; + }; + + ++idx; + Comm::record_iterator_next(it); + return comm_record_to_bro_record_recurse(it, rval, idx); + } + +function comm_record_to_bro_record(d: Comm::Data): bro_record + { + return comm_record_to_bro_record_recurse(Comm::record_iterator(d), + bro_record($c = 0), 0); + } + +function +comm_set_to_bro_set_recurse(it: opaque of Comm::SetIterator, + rval: bro_set): bro_set + { + if ( Comm::set_iterator_last(it) ) + return rval; + + add rval[Comm::refine_to_string(Comm::set_iterator_value(it))]; + Comm::set_iterator_next(it); + return comm_set_to_bro_set_recurse(it, rval); + } + + +function comm_set_to_bro_set(d: Comm::Data): bro_set + { + return comm_set_to_bro_set_recurse(Comm::set_iterator(d), bro_set()); + } + +function +comm_table_to_bro_table_recurse(it: opaque of Comm::TableIterator, + rval: bro_table): bro_table + { + if ( Comm::table_iterator_last(it) ) + return rval; + + local item = Comm::table_iterator_value(it); + rval[Comm::refine_to_string(item$key)] = Comm::refine_to_count(item$val); + Comm::table_iterator_next(it); + return comm_table_to_bro_table_recurse(it, rval); + } + +function comm_table_to_bro_table(d: Comm::Data): bro_table + { + return comm_table_to_bro_table_recurse(Comm::table_iterator(d), + bro_table()); + } + +function comm_vector_to_bro_vector_recurse(it: opaque of Comm::VectorIterator, + rval: bro_vector): bro_vector + { + if ( Comm::vector_iterator_last(it) ) + return rval; + + rval[|rval|] = Comm::refine_to_string(Comm::vector_iterator_value(it)); + Comm::vector_iterator_next(it); + return comm_vector_to_bro_vector_recurse(it, rval); + } + +function comm_vector_to_bro_vector(d: Comm::Data): bro_vector + { + return comm_vector_to_bro_vector_recurse(Comm::vector_iterator(d), + bro_vector()); + } + +event bro_init() +{ +Comm::enable(); +print Comm::data_type(Comm::data(T)); +print Comm::data_type(Comm::data(+1)); +print Comm::data_type(Comm::data(1)); +print Comm::data_type(Comm::data(1.1)); +print Comm::data_type(Comm::data("1 (how creative)")); +print Comm::data_type(Comm::data(1.1.1.1)); +print Comm::data_type(Comm::data(1.1.1.1/1)); +print Comm::data_type(Comm::data(1/udp)); +print Comm::data_type(Comm::data(double_to_time(1))); +print Comm::data_type(Comm::data(1sec)); +print Comm::data_type(Comm::data(Comm::BOOL)); +local s: bro_set = bro_set("one", "two", "three"); +local t: bro_table = bro_table(["one"] = 1, ["two"] = 2, ["three"] = 3); +local v: bro_vector = bro_vector("zero", "one", "two"); +local r: bro_record = bro_record($c = 1); +print Comm::data_type(Comm::data(s)); +print Comm::data_type(Comm::data(t)); +print Comm::data_type(Comm::data(v)); +print Comm::data_type(Comm::data(r)); + +print "***************************"; + +print Comm::refine_to_bool(Comm::data(T)); +print Comm::refine_to_bool(Comm::data(F)); +print Comm::refine_to_int(Comm::data(+1)); +print Comm::refine_to_int(Comm::data(+0)); +print Comm::refine_to_int(Comm::data(-1)); +print Comm::refine_to_count(Comm::data(1)); +print Comm::refine_to_count(Comm::data(0)); +print Comm::refine_to_double(Comm::data(1.1)); +print Comm::refine_to_double(Comm::data(-11.1)); +print Comm::refine_to_string(Comm::data("hello")); +print Comm::refine_to_addr(Comm::data(1.2.3.4)); +print Comm::refine_to_subnet(Comm::data(192.168.1.1/16)); +print Comm::refine_to_port(Comm::data(22/tcp)); +print Comm::refine_to_time(Comm::data(double_to_time(42))); +print Comm::refine_to_interval(Comm::data(3min)); +print Comm::refine_to_enum_name(Comm::data(Comm::BOOL)); + +print "***************************"; + +local cs = Comm::data(s); +print comm_set_to_bro_set(cs); +cs = Comm::set_create(); +print Comm::set_size(cs); +print Comm::set_insert(cs, Comm::data("hi")); +print Comm::set_size(cs); +print Comm::set_contains(cs, Comm::data("hi")); +print Comm::set_contains(cs, Comm::data("bye")); +print Comm::set_insert(cs, Comm::data("bye")); +print Comm::set_size(cs); +print Comm::set_remove(cs, Comm::data("hi")); +print Comm::set_size(cs); +print Comm::set_remove(cs, Comm::data("hi")); +print comm_set_to_bro_set(cs); +Comm::set_clear(cs); +print Comm::set_size(cs); + +print "***************************"; + +local ct = Comm::data(t); +print comm_table_to_bro_table(ct); +ct = Comm::table_create(); +print Comm::table_size(ct); +print Comm::table_insert(ct, Comm::data("hi"), Comm::data(42)); +print Comm::table_size(ct); +print Comm::table_contains(ct, Comm::data("hi")); +print Comm::refine_to_count(Comm::table_lookup(ct, Comm::data("hi"))); +print Comm::table_contains(ct, Comm::data("bye")); +print Comm::table_insert(ct, Comm::data("bye"), Comm::data(7)); +print Comm::table_size(ct); +print Comm::table_insert(ct, Comm::data("bye"), Comm::data(37)); +print Comm::table_size(ct); +print Comm::refine_to_count(Comm::table_lookup(ct, Comm::data("bye"))); +print Comm::table_remove(ct, Comm::data("hi")); +print Comm::table_size(ct); + +print "***************************"; + +local cv = Comm::data(v); +print comm_vector_to_bro_vector(cv); +cv = Comm::vector_create(); +print Comm::vector_size(cv); +print Comm::vector_insert(cv, Comm::data("hi"), 0); +print Comm::vector_insert(cv, Comm::data("hello"), 1); +print Comm::vector_insert(cv, Comm::data("greetings"), 2); +print Comm::vector_insert(cv, Comm::data("salutations"), 1); +print comm_vector_to_bro_vector(cv); +print Comm::vector_size(cv); +print Comm::vector_replace(cv, Comm::data("bah"), 2); +print Comm::vector_lookup(cv, 2); +print Comm::vector_lookup(cv, 0); +print comm_vector_to_bro_vector(cv); +print Comm::vector_remove(cv, 2); +print comm_vector_to_bro_vector(cv); +print Comm::vector_size(cv); + +print "***************************"; + +local cr = Comm::data(r); +print comm_record_to_bro_record(cr); +r$a = "test"; +cr = Comm::data(r); +print comm_record_to_bro_record(cr); +r$b = "testagain"; +cr = Comm::data(r); +print comm_record_to_bro_record(cr); +cr = Comm::record_create(3); +print Comm::record_size(cr); +print Comm::record_assign(cr, Comm::data("hi"), 0); +print Comm::record_assign(cr, Comm::data("hello"), 1); +print Comm::record_assign(cr, Comm::data(37), 2); +print Comm::record_lookup(cr, 0); +print Comm::record_lookup(cr, 1); +print Comm::record_lookup(cr, 2); +print Comm::record_size(cr); +} diff --git a/testing/btest/comm/master_store.bro b/testing/btest/comm/master_store.bro new file mode 100644 index 0000000000..61331bd170 --- /dev/null +++ b/testing/btest/comm/master_store.bro @@ -0,0 +1,144 @@ +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: bro -b %INPUT >out +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff out + +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global lookup_count = 0; +const lookup_expect_count = 5; +global exists_count = 0; +const exists_expect_count = 4; +global pop_count = 0; +const pop_expect_count = 2; + +global test_size: event(where: string &default = ""); + +event test_clear() + { + Store::clear(h); + event test_size("after clear"); + } + +event test_size(where: string) + { + when ( local res = Store::size(h) ) + { + if ( where == "" ) + { + print fmt("size: %s", res); + event test_clear(); + } + else + { + print fmt("size (%s): %s", where, res); + terminate(); + } + } + timeout 10sec + { print "timeout"; } + } + +event test_keys() + { + when ( local res = Store::keys(h) ) + { + print fmt("keys: %s", res); + event test_size(); + } + timeout 10sec + { print "timeout"; } + } + +event test_pop(key: string) + { + when ( local lres = Store::pop_left(h, Comm::data(key)) ) + { + print fmt("pop_left(%s): %s", key, lres); + ++pop_count; + + if ( pop_count == pop_expect_count ) + event test_keys(); + } + timeout 10sec + { print "timeout"; } + + when ( local rres = Store::pop_right(h, Comm::data(key)) ) + { + print fmt("pop_right(%s): %s", key, rres); + ++pop_count; + + if ( pop_count == pop_expect_count ) + event test_keys(); + } + timeout 10sec + { print "timeout"; } + } + +function do_exists(key: string) + { + when ( local res = Store::exists(h, Comm::data(key)) ) + { + print fmt("exists(%s): %s", key, res); + ++exists_count; + + if ( exists_count == exists_expect_count ) + event test_pop("myvec"); + } + timeout 10sec + { print "timeout"; } + } + +event test_erase() + { + Store::erase(h, Comm::data("two")); + do_exists("one"); + do_exists("two"); + do_exists("myset"); + do_exists("four"); + } + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + print fmt("lookup(%s): %s", key, res); + ++lookup_count; + + if ( lookup_count == lookup_expect_count ) + event test_erase(); + } + timeout 10sec + { print "timeout"; } + } + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +event bro_init() + { + Comm::enable(); + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + h = Store::create_master("master"); + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + do_lookup("one"); + do_lookup("two"); + do_lookup("myset"); + do_lookup("four"); + do_lookup("myvec"); + } diff --git a/testing/btest/comm/remote_event.test b/testing/btest/comm/remote_event.test new file mode 100644 index 0000000000..8950cb9e1e --- /dev/null +++ b/testing/btest/comm/remote_event.test @@ -0,0 +1,94 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff send/send.out + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global event_handler: event(msg: string, c: count); +global auto_event_handler: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/"); + Comm::auto_event("bro/event/my_topic", auto_event_handler); + Comm::listen(broker_port, "127.0.0.1"); + } + +global event_count = 0; +global events_to_recv = 6; + +event event_handler(msg: string, n: count) + { + ++event_count; + print "got event msg", msg, n; + + if ( event_count == events_to_recv ) + { + terminate(); + return; + } + + event auto_event_handler(msg, n); + local args = Comm::event_args(event_handler, "pong", n); + Comm::event("bro/event/my_topic", args); + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global event_handler: event(msg: string, c: count); +global auto_event_handler: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/my_topic"); + Comm::connect("127.0.0.1", broker_port, 1secs); + } + +global event_count = 0; + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", peer_address, peer_port; + local args = Comm::event_args(event_handler, "ping", event_count); + Comm::event("bro/event/hi", args); + ++event_count; + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event event_handler(msg: string, n: count) + { + print "got event msg", msg, n; + local args = Comm::event_args(event_handler, "ping", event_count); + Comm::event("bro/event/hi", args); + ++event_count; + } + +event auto_event_handler(msg: string, n: count) + { + print "got auto event msg", msg, n; + } + +@TEST-END-FILE diff --git a/testing/btest/comm/remote_log.test b/testing/btest/comm/remote_log.test new file mode 100644 index 0000000000..060a822c5a --- /dev/null +++ b/testing/btest/comm/remote_log.test @@ -0,0 +1,96 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: btest-bg-run recv "bro -b ../common.bro ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: btest-bg-run send "bro -b ../common.bro ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff recv/test.log +# @TEST-EXEC: btest-diff send/send.out +# @TEST-EXEC: btest-diff send/test.log + +@TEST-START-FILE common.bro + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event bro_init() &priority=5 + { + Comm::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); + } + +@TEST-END-FILE + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::subscribe_to_logs("bro/log/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + + if ( rec$num == 5 ) + terminate(); + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::enable_remote_logs(Test::LOG); + Comm::connect("127.0.0.1", broker_port, 1secs); + } + +global n = 0; + +event do_write() + { + if ( n == 6 ) + return; + else + { + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", peer_address, peer_port; + event do_write(); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/comm/remote_print.test b/testing/btest/comm/remote_print.test new file mode 100644 index 0000000000..d77bc92e9c --- /dev/null +++ b/testing/btest/comm/remote_print.test @@ -0,0 +1,83 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt + +# @TEST-EXEC: btest-bg-run recv "bro -b ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: btest-bg-run send "bro -b ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 20 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff send/send.out + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + Comm::subscribe_to_prints("bro/print/"); + } + +global messages_to_recv = 6; +global messages_sent = 0; +global messages_recv = 0; + +event Comm::print_handler(msg: string) + { + ++messages_recv; + print "got print msg", msg; + + if ( messages_to_recv == messages_recv ) + { + terminate(); + return; + } + + Comm::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + ++messages_sent; + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_prints("bro/print/my_topic"); + Comm::connect("127.0.0.1", broker_port, 1secs); + } + +global messages_sent = 0; +global messages_recv = 0; +global peer_disconnected = F; + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", peer_address, peer_port; + Comm::print("bro/print/hi", fmt("ping %d", messages_sent)); + ++messages_sent; + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::print_handler(msg: string) + { + ++messages_recv; + print "got print msg", msg; + Comm::print("bro/print/hi", fmt("ping %d", messages_sent)); + ++messages_sent; + } + +@TEST-END-FILE diff --git a/testing/btest/core/leaks/comm/clone_store.bro b/testing/btest/core/leaks/comm/clone_store.bro new file mode 100644 index 0000000000..2a75bfa62f --- /dev/null +++ b/testing/btest/core/leaks/comm/clone_store.bro @@ -0,0 +1,113 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leak + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run clone "bro -m -b ../clone.bro broker_port=$BROKER_PORT >clone.out" +# @TEST-EXEC: btest-bg-run master "bro -b ../master.bro broker_port=$BROKER_PORT >master.out" + +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff clone/clone.out + +@TEST-START-FILE clone.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout"; } + } + +event ready() + { + h = Store::create_clone("mystore"); + + when ( local res = Store::keys(h) ) + { + print "clone keys", res; + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + Comm::subscribe_to_events("bro/event/ready"); + } + +@TEST-END-FILE + +@TEST-START-FILE master.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + + when ( local res = Store::size(h) ) + { event ready(); } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + h = Store::create_master("mystore"); + Comm::connect("127.0.0.1", broker_port, 1secs); + Comm::auto_event("bro/event/ready", ready); + } + +@TEST-END-FILE diff --git a/testing/btest/core/leaks/comm/data.bro b/testing/btest/core/leaks/comm/data.bro new file mode 100644 index 0000000000..bf614a2092 --- /dev/null +++ b/testing/btest/core/leaks/comm/data.bro @@ -0,0 +1,233 @@ +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leaks + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: btest-diff bro/.stdout + +type bro_set: set[string]; +type bro_table: table[string] of count; +type bro_vector: vector of string; + +type bro_record : record { + a: string &optional; + b: string &default = "bee"; + c: count; +}; + +function comm_record_to_bro_record_recurse(it: opaque of Comm::RecordIterator, + rval: bro_record, + idx: count): bro_record + { + if ( Comm::record_iterator_last(it) ) + return rval; + + local field_value = Comm::record_iterator_value(it); + + if ( field_value?$d ) + switch ( idx ) { + case 0: + rval$a = Comm::refine_to_string(field_value); + break; + case 1: + rval$b = Comm::refine_to_string(field_value); + break; + case 2: + rval$c = Comm::refine_to_count(field_value); + break; + }; + + ++idx; + Comm::record_iterator_next(it); + return comm_record_to_bro_record_recurse(it, rval, idx); + } + +function comm_record_to_bro_record(d: Comm::Data): bro_record + { + return comm_record_to_bro_record_recurse(Comm::record_iterator(d), + bro_record($c = 0), 0); + } + +function +comm_set_to_bro_set_recurse(it: opaque of Comm::SetIterator, + rval: bro_set): bro_set + { + if ( Comm::set_iterator_last(it) ) + return rval; + + add rval[Comm::refine_to_string(Comm::set_iterator_value(it))]; + Comm::set_iterator_next(it); + return comm_set_to_bro_set_recurse(it, rval); + } + + +function comm_set_to_bro_set(d: Comm::Data): bro_set + { + return comm_set_to_bro_set_recurse(Comm::set_iterator(d), bro_set()); + } + +function +comm_table_to_bro_table_recurse(it: opaque of Comm::TableIterator, + rval: bro_table): bro_table + { + if ( Comm::table_iterator_last(it) ) + return rval; + + local item = Comm::table_iterator_value(it); + rval[Comm::refine_to_string(item$key)] = Comm::refine_to_count(item$val); + Comm::table_iterator_next(it); + return comm_table_to_bro_table_recurse(it, rval); + } + +function comm_table_to_bro_table(d: Comm::Data): bro_table + { + return comm_table_to_bro_table_recurse(Comm::table_iterator(d), + bro_table()); + } + +function comm_vector_to_bro_vector_recurse(it: opaque of Comm::VectorIterator, + rval: bro_vector): bro_vector + { + if ( Comm::vector_iterator_last(it) ) + return rval; + + rval[|rval|] = Comm::refine_to_string(Comm::vector_iterator_value(it)); + Comm::vector_iterator_next(it); + return comm_vector_to_bro_vector_recurse(it, rval); + } + +function comm_vector_to_bro_vector(d: Comm::Data): bro_vector + { + return comm_vector_to_bro_vector_recurse(Comm::vector_iterator(d), + bro_vector()); + } + +event bro_init() + { +Comm::enable(); + } + +global did_it = F; + +event new_connection(c: connection) + { +if ( did_it ) return; +did_it = T; +print Comm::data_type(Comm::data(T)); +print Comm::data_type(Comm::data(+1)); +print Comm::data_type(Comm::data(1)); +print Comm::data_type(Comm::data(1.1)); +print Comm::data_type(Comm::data("1 (how creative)")); +print Comm::data_type(Comm::data(1.1.1.1)); +print Comm::data_type(Comm::data(1.1.1.1/1)); +print Comm::data_type(Comm::data(1/udp)); +print Comm::data_type(Comm::data(double_to_time(1))); +print Comm::data_type(Comm::data(1sec)); +print Comm::data_type(Comm::data(Comm::BOOL)); +local s: bro_set = bro_set("one", "two", "three"); +local t: bro_table = bro_table(["one"] = 1, ["two"] = 2, ["three"] = 3); +local v: bro_vector = bro_vector("zero", "one", "two"); +local r: bro_record = bro_record($c = 1); +print Comm::data_type(Comm::data(s)); +print Comm::data_type(Comm::data(t)); +print Comm::data_type(Comm::data(v)); +print Comm::data_type(Comm::data(r)); + +print "***************************"; + +print Comm::refine_to_bool(Comm::data(T)); +print Comm::refine_to_bool(Comm::data(F)); +print Comm::refine_to_int(Comm::data(+1)); +print Comm::refine_to_int(Comm::data(+0)); +print Comm::refine_to_int(Comm::data(-1)); +print Comm::refine_to_count(Comm::data(1)); +print Comm::refine_to_count(Comm::data(0)); +print Comm::refine_to_double(Comm::data(1.1)); +print Comm::refine_to_double(Comm::data(-11.1)); +print Comm::refine_to_string(Comm::data("hello")); +print Comm::refine_to_addr(Comm::data(1.2.3.4)); +print Comm::refine_to_subnet(Comm::data(192.168.1.1/16)); +print Comm::refine_to_port(Comm::data(22/tcp)); +print Comm::refine_to_time(Comm::data(double_to_time(42))); +print Comm::refine_to_interval(Comm::data(3min)); +print Comm::refine_to_enum_name(Comm::data(Comm::BOOL)); + +print "***************************"; + +local cs = Comm::data(s); +print comm_set_to_bro_set(cs); +cs = Comm::set_create(); +print Comm::set_size(cs); +print Comm::set_insert(cs, Comm::data("hi")); +print Comm::set_size(cs); +print Comm::set_contains(cs, Comm::data("hi")); +print Comm::set_contains(cs, Comm::data("bye")); +print Comm::set_insert(cs, Comm::data("bye")); +print Comm::set_size(cs); +print Comm::set_remove(cs, Comm::data("hi")); +print Comm::set_size(cs); +print Comm::set_remove(cs, Comm::data("hi")); +print comm_set_to_bro_set(cs); +Comm::set_clear(cs); +print Comm::set_size(cs); + +print "***************************"; + +local ct = Comm::data(t); +print comm_table_to_bro_table(ct); +ct = Comm::table_create(); +print Comm::table_size(ct); +print Comm::table_insert(ct, Comm::data("hi"), Comm::data(42)); +print Comm::table_size(ct); +print Comm::table_contains(ct, Comm::data("hi")); +print Comm::refine_to_count(Comm::table_lookup(ct, Comm::data("hi"))); +print Comm::table_contains(ct, Comm::data("bye")); +print Comm::table_insert(ct, Comm::data("bye"), Comm::data(7)); +print Comm::table_size(ct); +print Comm::table_insert(ct, Comm::data("bye"), Comm::data(37)); +print Comm::table_size(ct); +print Comm::refine_to_count(Comm::table_lookup(ct, Comm::data("bye"))); +print Comm::table_remove(ct, Comm::data("hi")); +print Comm::table_size(ct); + +print "***************************"; + +local cv = Comm::data(v); +print comm_vector_to_bro_vector(cv); +cv = Comm::vector_create(); +print Comm::vector_size(cv); +print Comm::vector_insert(cv, Comm::data("hi"), 0); +print Comm::vector_insert(cv, Comm::data("hello"), 1); +print Comm::vector_insert(cv, Comm::data("greetings"), 2); +print Comm::vector_insert(cv, Comm::data("salutations"), 1); +print comm_vector_to_bro_vector(cv); +print Comm::vector_size(cv); +print Comm::vector_replace(cv, Comm::data("bah"), 2); +print Comm::vector_lookup(cv, 2); +print Comm::vector_lookup(cv, 0); +print comm_vector_to_bro_vector(cv); +print Comm::vector_remove(cv, 2); +print comm_vector_to_bro_vector(cv); +print Comm::vector_size(cv); + +print "***************************"; + +local cr = Comm::data(r); +print comm_record_to_bro_record(cr); +r$a = "test"; +cr = Comm::data(r); +print comm_record_to_bro_record(cr); +r$b = "testagain"; +cr = Comm::data(r); +print comm_record_to_bro_record(cr); +cr = Comm::record_create(3); +print Comm::record_size(cr); +print Comm::record_assign(cr, Comm::data("hi"), 0); +print Comm::record_assign(cr, Comm::data("hello"), 1); +print Comm::record_assign(cr, Comm::data(37), 2); +print Comm::record_lookup(cr, 0); +print Comm::record_lookup(cr, 1); +print Comm::record_lookup(cr, 2); +print Comm::record_size(cr); +} diff --git a/testing/btest/core/leaks/comm/master_store.bro b/testing/btest/core/leaks/comm/master_store.bro new file mode 100644 index 0000000000..a5c1063e6f --- /dev/null +++ b/testing/btest/core/leaks/comm/master_store.bro @@ -0,0 +1,155 @@ +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leaks + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run bro bro -m -b -r $TRACES/http/get.trace %INPUT +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff bro/.stdout + +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global lookup_count = 0; +const lookup_expect_count = 5; +global exists_count = 0; +const exists_expect_count = 4; +global pop_count = 0; +const pop_expect_count = 2; + +global test_size: event(where: string &default = ""); + +event test_clear() + { + Store::clear(h); + event test_size("after clear"); + } + +event test_size(where: string) + { + when ( local res = Store::size(h) ) + { + if ( where == "" ) + { + print fmt("size: %s", res); + event test_clear(); + } + else + { + print fmt("size (%s): %s", where, res); + terminate(); + } + } + timeout 10sec + { print "timeout"; } + } + +event test_keys() + { + when ( local res = Store::keys(h) ) + { + print fmt("keys: %s", res); + event test_size(); + } + timeout 10sec + { print "timeout"; } + } + +event test_pop(key: string) + { + when ( local lres = Store::pop_left(h, Comm::data(key)) ) + { + print fmt("pop_left(%s): %s", key, lres); + ++pop_count; + + if ( pop_count == pop_expect_count ) + event test_keys(); + } + timeout 10sec + { print "timeout"; } + + when ( local rres = Store::pop_right(h, Comm::data(key)) ) + { + print fmt("pop_right(%s): %s", key, rres); + ++pop_count; + + if ( pop_count == pop_expect_count ) + event test_keys(); + } + timeout 10sec + { print "timeout"; } + } + +function do_exists(key: string) + { + when ( local res = Store::exists(h, Comm::data(key)) ) + { + print fmt("exists(%s): %s", key, res); + ++exists_count; + + if ( exists_count == exists_expect_count ) + event test_pop("myvec"); + } + timeout 10sec + { print "timeout"; } + } + +event test_erase() + { + Store::erase(h, Comm::data("two")); + do_exists("one"); + do_exists("two"); + do_exists("myset"); + do_exists("four"); + } + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + print fmt("lookup(%s): %s", key, res); + ++lookup_count; + + if ( lookup_count == lookup_expect_count ) + event test_erase(); + } + timeout 10sec + { print "timeout"; } + } + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +global did_it = F; + +event bro_init() + { + Comm::enable(); + h = Store::create_master("master"); + } + +event new_connection(c: connection) + { + if ( did_it ) return; + did_it = T; + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + do_lookup("one"); + do_lookup("two"); + do_lookup("myset"); + do_lookup("four"); + do_lookup("myvec"); + } diff --git a/testing/btest/core/leaks/comm/remote_event.test b/testing/btest/core/leaks/comm/remote_event.test new file mode 100644 index 0000000000..a329b527db --- /dev/null +++ b/testing/btest/core/leaks/comm/remote_event.test @@ -0,0 +1,96 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leak + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run recv "bro -m -b ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run send "bro -m -b ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff send/send.out + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global event_handler: event(msg: string, c: count); +global auto_event_handler: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + Comm::subscribe_to_events("bro/event/"); + Comm::auto_event("bro/event/my_topic", auto_event_handler); + } + +global event_count = 0; +global events_to_recv = 6; + +event event_handler(msg: string, n: count) + { + ++event_count; + print "got event msg", msg, n; + + if ( event_count == events_to_recv ) + { + terminate(); + return; + } + + event auto_event_handler(msg, n); + local args = Comm::event_args(event_handler, "pong", n); + Comm::event("bro/event/my_topic", args); + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global event_handler: event(msg: string, c: count); +global auto_event_handler: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/my_topic"); + Comm::connect("127.0.0.1", broker_port, 1secs); + } + +global event_count = 0; + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", peer_address, peer_port; + local args = Comm::event_args(event_handler, "ping", event_count); + Comm::event("bro/event/hi", args); + ++event_count; + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event event_handler(msg: string, n: count) + { + print "got event msg", msg, n; + local args = Comm::event_args(event_handler, "ping", event_count); + Comm::event("bro/event/hi", args); + ++event_count; + } + +event auto_event_handler(msg: string, n: count) + { + print "got auto event msg", msg, n; + } + +@TEST-END-FILE diff --git a/testing/btest/core/leaks/comm/remote_log.test b/testing/btest/core/leaks/comm/remote_log.test new file mode 100644 index 0000000000..6f20bf8cd4 --- /dev/null +++ b/testing/btest/core/leaks/comm/remote_log.test @@ -0,0 +1,98 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leak + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run recv "bro -m -b ../common.bro ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run send "bro -m -b ../common.bro ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff recv/test.log +# @TEST-EXEC: btest-diff send/send.out +# @TEST-EXEC: btest-diff send/test.log + +@TEST-START-FILE common.bro + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event bro_init() &priority=5 + { + Comm::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); + } + +@TEST-END-FILE + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::listen(broker_port, "127.0.0.1"); + Comm::subscribe_to_logs("bro/log/"); + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + + if ( rec$num == 5 ) + terminate(); + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::enable_remote_logs(Test::LOG); + Comm::connect("127.0.0.1", broker_port, 1secs); + } + +global n = 0; + +event do_write() + { + if ( n == 6 ) + return; + else + { + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", peer_address, peer_port; + event do_write(); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +@TEST-END-FILE diff --git a/testing/btest/core/leaks/comm/remote_print.test b/testing/btest/core/leaks/comm/remote_print.test new file mode 100644 index 0000000000..43fe50b632 --- /dev/null +++ b/testing/btest/core/leaks/comm/remote_print.test @@ -0,0 +1,85 @@ +# @TEST-SERIALIZE: brokercomm +# @TEST-REQUIRES: grep -q ENABLE_BROKER $BUILD/CMakeCache.txt +# @TEST-REQUIRES: bro --help 2>&1 | grep -q mem-leaks +# @TEST-GROUP: leak + +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run recv "bro -m -b ../recv.bro broker_port=$BROKER_PORT >recv.out" +# @TEST-EXEC: HEAP_CHECK_DUMP_DIRECTORY=. HEAPCHECK=local btest-bg-run send "bro -m -b ../send.bro broker_port=$BROKER_PORT >send.out" + +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: btest-diff recv/recv.out +# @TEST-EXEC: btest-diff send/send.out + +@TEST-START-FILE recv.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + Comm::subscribe_to_prints("bro/print/"); + } + +global messages_to_recv = 6; +global messages_sent = 0; +global messages_recv = 0; + +event Comm::print_handler(msg: string) + { + ++messages_recv; + print "got print msg", msg; + + if ( messages_to_recv == messages_recv ) + { + terminate(); + return; + } + + Comm::print("bro/print/my_topic", fmt("pong %d", messages_sent)); + ++messages_sent; + } + +@TEST-END-FILE + +@TEST-START-FILE send.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_prints("bro/print/my_topic"); + Comm::connect("127.0.0.1", broker_port, 1secs); + } + +global messages_sent = 0; +global messages_recv = 0; +global peer_disconnected = F; + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", peer_address, peer_port; + Comm::print("bro/print/hi", fmt("ping %d", messages_sent)); + ++messages_sent; + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::print_handler(msg: string) + { + ++messages_recv; + print "got print msg", msg; + Comm::print("bro/print/hi", fmt("ping %d", messages_sent)); + ++messages_sent; + } + +@TEST-END-FILE diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_connecting-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_connecting-connector_bro.btest new file mode 100644 index 0000000000..1921e6596a --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_connecting-connector_bro.btest @@ -0,0 +1,23 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +connecting-connector.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_connecting-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_connecting-listener_bro.btest new file mode 100644 index 0000000000..7516680533 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_connecting-listener_bro.btest @@ -0,0 +1,25 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +connecting-listener.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Comm::incoming_connection_broken(peer_name: string) + { + print "Comm::incoming_connection_broken", peer_name; + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_events-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_events-connector_bro.btest new file mode 100644 index 0000000000..434e94d977 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_events-connector_bro.btest @@ -0,0 +1,35 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +events-connector.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + Comm::auto_event("bro/event/my_auto_event", my_auto_event); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + Comm::event("bro/event/my_event", Comm::event_args(my_event, "hi", 0)); + event my_auto_event("stuff", 88); + Comm::event("bro/event/my_event", Comm::event_args(my_event, "...", 1)); + event my_auto_event("more stuff", 51); + Comm::event("bro/event/my_event", Comm::event_args(my_event, "bye", 2)); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_events-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_events-listener_bro.btest new file mode 100644 index 0000000000..a8b7c133ff --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_events-listener_bro.btest @@ -0,0 +1,41 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +events-listener.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; +global msg_count = 0; +global my_event: event(msg: string, c: count); +global my_auto_event: event(msg: string, c: count); + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event my_event(msg: string, c: count) + { + ++msg_count; + print "got my_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } + +event my_auto_event(msg: string, c: count) + { + ++msg_count; + print "got my_auto_event", msg, c; + + if ( msg_count == 5 ) + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_logs-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_logs-connector_bro.btest new file mode 100644 index 0000000000..ae8c3b4ec5 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_logs-connector_bro.btest @@ -0,0 +1,44 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +logs-connector.bro + +@load ./testlog + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; +redef Log::enable_local_logging = F; +redef Log::enable_remote_logging = F; +global n = 0; + +event bro_init() + { + Comm::enable(); + Comm::enable_remote_logs(Test::LOG); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event do_write() + { + if ( n == 6 ) + return; + + Log::write(Test::LOG, [$msg = "ping", $num = n]); + ++n; + event do_write(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + event do_write(); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_logs-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_logs-listener_bro.btest new file mode 100644 index 0000000000..472229ea04 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_logs-listener_bro.btest @@ -0,0 +1,29 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +logs-listener.bro + +@load ./testlog + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_logs("bro/log/Test::LOG"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Test::log_test(rec: Test::Info) + { + print "wrote log", rec; + + if ( rec$num == 5 ) + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_printing-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_printing-connector_bro.btest new file mode 100644 index 0000000000..b796155c59 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_printing-connector_bro.btest @@ -0,0 +1,30 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +printing-connector.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "connector"; + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1sec); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + print "Comm::outgoing_connection_established", + peer_address, peer_port, peer_name; + Comm::print("bro/print/hi", "hello"); + Comm::print("bro/print/stuff", "..."); + Comm::print("bro/print/bye", "goodbye"); + } + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_printing-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_printing-listener_bro.btest new file mode 100644 index 0000000000..de6741d3c4 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_printing-listener_bro.btest @@ -0,0 +1,30 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +printing-listener.bro + + +const broker_port: port &redef; +redef exit_only_after_terminate = T; +redef Comm::endpoint_name = "listener"; +global msg_count = 0; + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_prints("bro/print/"); + Comm::listen(broker_port, "127.0.0.1"); + } + +event Comm::incoming_connection_established(peer_name: string) + { + print "Comm::incoming_connection_established", peer_name; + } + +event Comm::print_handler(msg: string) + { + ++msg_count; + print "got print message", msg; + + if ( msg_count == 3 ) + terminate(); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_stores-connector_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_stores-connector_bro.btest new file mode 100644 index 0000000000..c5268417a6 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_stores-connector_bro.btest @@ -0,0 +1,57 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +stores-connector.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; + +function dv(d: Comm::Data): Comm::DataVector + { + local rval: Comm::DataVector; + rval[0] = d; + return rval; + } + +global ready: event(); + +event Comm::outgoing_connection_broken(peer_address: string, + peer_port: port) + { + terminate(); + } + +event Comm::outgoing_connection_established(peer_address: string, + peer_port: port, + peer_name: string) + { + local myset: set[string] = {"a", "b", "c"}; + local myvec: vector of string = {"alpha", "beta", "gamma"}; + h = Store::create_master("mystore"); + Store::insert(h, Comm::data("one"), Comm::data(110)); + Store::insert(h, Comm::data("two"), Comm::data(223)); + Store::insert(h, Comm::data("myset"), Comm::data(myset)); + Store::insert(h, Comm::data("myvec"), Comm::data(myvec)); + Store::increment(h, Comm::data("one")); + Store::decrement(h, Comm::data("two")); + Store::add_to_set(h, Comm::data("myset"), Comm::data("d")); + Store::remove_from_set(h, Comm::data("myset"), Comm::data("b")); + Store::push_left(h, Comm::data("myvec"), dv(Comm::data("delta"))); + Store::push_right(h, Comm::data("myvec"), dv(Comm::data("omega"))); + + when ( local res = Store::size(h) ) + { + print "master size", res; + event ready(); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::connect("127.0.0.1", broker_port, 1secs); + Comm::auto_event("bro/event/ready", ready); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_stores-listener_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_stores-listener_bro.btest new file mode 100644 index 0000000000..38dc7ef34f --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_stores-listener_bro.btest @@ -0,0 +1,47 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +stores-listener.bro + +const broker_port: port &redef; +redef exit_only_after_terminate = T; + +global h: opaque of Store::Handle; +global expected_key_count = 4; +global key_count = 0; + +function do_lookup(key: string) + { + when ( local res = Store::lookup(h, Comm::data(key)) ) + { + ++key_count; + print "lookup", key, res; + + if ( key_count == expected_key_count ) + terminate(); + } + timeout 10sec + { print "timeout", key; } + } + +event ready() + { + h = Store::create_clone("mystore"); + + when ( local res = Store::keys(h) ) + { + print "clone keys", res; + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 0))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 1))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 2))); + do_lookup(Comm::refine_to_string(Comm::vector_lookup(res$result, 3))); + } + timeout 10sec + { print "timeout"; } + } + +event bro_init() + { + Comm::enable(); + Comm::subscribe_to_events("bro/event/ready"); + Comm::listen(broker_port, "127.0.0.1"); + } diff --git a/testing/btest/doc/sphinx/include-doc_frameworks_comm_testlog_bro.btest b/testing/btest/doc/sphinx/include-doc_frameworks_comm_testlog_bro.btest new file mode 100644 index 0000000000..e37b34c518 --- /dev/null +++ b/testing/btest/doc/sphinx/include-doc_frameworks_comm_testlog_bro.btest @@ -0,0 +1,23 @@ +# @TEST-EXEC: cat %INPUT >output && btest-diff output + +testlog.bro + + +module Test; + +export { + redef enum Log::ID += { LOG }; + + type Info: record { + msg: string &log; + num: count &log; + }; + + global log_test: event(rec: Test::Info); +} + +event bro_init() &priority=5 + { + Comm::enable(); + Log::create_stream(Test::LOG, [$columns=Test::Info, $ev=log_test]); + } diff --git a/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest b/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest index 9966341119..4b15400080 100644 --- a/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest +++ b/testing/btest/doc/sphinx/include-scripts_base_protocols_conn_main_bro.btest @@ -17,12 +17,12 @@ export { resp_bytes: count &log &optional; conn_state: string &log &optional; local_orig: bool &log &optional; - missed_bytes: count &log &default=0; - history: string &log &optional; - orig_pkts: count &log &optional; - orig_ip_bytes: count &log &optional; - resp_pkts: count &log &optional; - resp_ip_bytes: count &log &optional; + local_resp: bool &log &optional; + ## + ## lower-case. Multiple packets of the same type will only be + ## Number of packets that the originator sent. + ## Number of IP level bytes that the originator sent (as seen on + ## Number of packets that the responder sent. + ## the wire, taken from the IP total_length header field). + ## Only set if :bro:id:`use_conn_size_analyzer` = T. tunnel_parents: set[string] &log; - }; -}