diff --git a/CHANGES b/CHANGES index ab9f60760e..557a72def8 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,15 @@ +5.0.0-dev.545 | 2022-06-02 12:00:53 +0200 + + * Add WebSocket support for exchanging events with external clients. + (Robin Sommer, Corelight) + + This exposes Broker's new WebSocket support in Zeek. To enable it, + call `Broker::listen_websocket()`. Zeek will then start listening on + port 9997 for incoming WebSocket connections. + + See the Broker documentation for a description of the message format + expected over these WebSocket connections. + 5.0.0-dev.540 | 2022-06-01 11:08:42 -0700 * GH-2136: Clean up DNS_Mgr before shutting down (Tim Wojtulewicz, Corelight) diff --git a/NEWS b/NEWS index 4b14c8d423..ff35a074e7 100644 --- a/NEWS +++ b/NEWS @@ -52,6 +52,12 @@ New Functionality use this functionality, see the TLS Decryption documentation at https://docs.zeek.org/en/master/frameworks/tls-decryption.html +- Zeek now provides WebSocket support for exchanging events with + external clients. To enable it, call `Broker::listen_websocket()`. + Zeek will then start listening on port 9997 for incoming WebSocket + connections. See the Broker documentation for a description of the + message format expected over these WebSocket connections. + - The new --with-gen-zam configure flag and its corresponding GEN_ZAM_EXE_PATH cmake variable allow reuse of a previously built Gen-ZAM code generator. This aids cross-compilation: the Zeek build process normally compiles Gen-ZAM on diff --git a/VERSION b/VERSION index e7a58896ce..dcc732fcd7 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.0-dev.543 +5.0.0-dev.545 diff --git a/auxil/broker b/auxil/broker index 5515c3b033..0353de8e4e 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit 5515c3b033b191a957b3ac45dbe7d0625936143a +Subproject commit 0353de8e4e26d90e609c1a09df3462ec68af2553 diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile index afd72835f2..83c3b44e9f 100644 --- a/ci/alpine/Dockerfile +++ b/ci/alpine/Dockerfile @@ -26,4 +26,4 @@ RUN apk add --no-cache \ openssh-client \ py3-pip -RUN pip3 install junit2html +RUN pip3 install websockets junit2html diff --git a/ci/centos-7/Dockerfile b/ci/centos-7/Dockerfile index 73135cc521..bb1ea3ca87 100644 --- a/ci/centos-7/Dockerfile +++ b/ci/centos-7/Dockerfile @@ -15,7 +15,7 @@ RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf RUN yum -y install \ https://repo.ius.io/ius-release-el7.rpm \ https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm \ - && yum -y install git224 ccache \ + && yum -y install git236 ccache \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ @@ -55,7 +55,7 @@ RUN curl -sSL "https://github.com/westes/flex/releases/download/v${FLEX_VERSION} && make -j`nproc` install) \ && rm -rf /tmp/flex-${FLEX_VERSION} -RUN pip3 install junit2html +RUN pip3 install websockets junit2html RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \ echo 'source /opt/rh/devtoolset-8/enable' >> /usr/bin/zeek-ci-env && \ diff --git a/ci/centos-stream-8/Dockerfile b/ci/centos-stream-8/Dockerfile index db2e8adeb7..32a0cc56fa 100644 --- a/ci/centos-stream-8/Dockerfile +++ b/ci/centos-stream-8/Dockerfile @@ -26,4 +26,4 @@ RUN dnf -y install \ zlib-devel \ && dnf clean all && rm -rf /var/cache/dnf -RUN pip3 install junit2html +RUN pip3 install websockets junit2html diff --git a/ci/centos-stream-9/Dockerfile b/ci/centos-stream-9/Dockerfile index 0973443e44..b2ac5783fb 100644 --- a/ci/centos-stream-9/Dockerfile +++ b/ci/centos-stream-9/Dockerfile @@ -42,4 +42,4 @@ RUN dnf -y --nobest install \ # Set the crypto policy to allow SHA-1 certificates - which we have in our tests RUN dnf -y --nobest install crypto-policies-scripts && update-crypto-policies --set LEGACY -RUN pip3 install junit2html +RUN pip3 install websockets junit2html diff --git a/ci/debian-10/Dockerfile b/ci/debian-10/Dockerfile index f384263c3e..d03f86af80 100644 --- a/ci/debian-10/Dockerfile +++ b/ci/debian-10/Dockerfile @@ -35,6 +35,6 @@ RUN apt-get update && apt-get -y install \ && mkdir -p "${CMAKE_DIR}" \ && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 \ - && pip3 install junit2html + && pip3 install websockets junit2html ENV PATH "${CMAKE_DIR}/bin:${PATH}" diff --git a/ci/debian-11/Dockerfile b/ci/debian-11/Dockerfile index 905f014521..a62cc35865 100644 --- a/ci/debian-11/Dockerfile +++ b/ci/debian-11/Dockerfile @@ -30,4 +30,4 @@ RUN apt-get update && apt-get -y install \ xz-utils \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install junit2html +RUN pip3 install websockets junit2html diff --git a/ci/debian-9-32bit/Dockerfile b/ci/debian-9-32bit/Dockerfile index 197fe28e27..9ee1d93f58 100644 --- a/ci/debian-9-32bit/Dockerfile +++ b/ci/debian-9-32bit/Dockerfile @@ -43,6 +43,6 @@ RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-11 100 # junit2html >= 31.0.0 requires jinj2 >= 3.0 which requires python >= 3.7 which is # a higher version of python3 than debian 9 provides. Fix the version of junit2html # to the last version before they required the newer jinja2. -RUN pip3 install junit2html==30.0.6 +RUN pip3 install websockets junit2html==30.0.6 ENV CXXFLAGS=-stdlib=libc++ diff --git a/ci/debian-9/Dockerfile b/ci/debian-9/Dockerfile index f59a333be4..731a9ffed2 100644 --- a/ci/debian-9/Dockerfile +++ b/ci/debian-9/Dockerfile @@ -40,7 +40,7 @@ RUN apt-get update && apt-get -y install \ # junit2html >= 31.0.0 requires jinj2 >= 3.0 which requires python >= 3.7 which is # a higher version of python3 than debian 9 provides. Fix the version of junit2html # to the last version before they required the newer jinja2. -RUN pip3 install junit2html==30.0.6 +RUN pip3 install websockets junit2html==30.0.6 ENV CC=/usr/bin/clang-11 ENV CXX=/usr/bin/clang++-11 diff --git a/ci/fedora-34/Dockerfile b/ci/fedora-34/Dockerfile index 1b75243206..1431fcf723 100644 --- a/ci/fedora-34/Dockerfile +++ b/ci/fedora-34/Dockerfile @@ -25,4 +25,4 @@ RUN dnf -y install \ zlib-devel \ && dnf clean all && rm -rf /var/cache/dnf -RUN pip3 install junit2html +RUN pip3 install websockets junit2html diff --git a/ci/fedora-35/Dockerfile b/ci/fedora-35/Dockerfile index 7a3c6fe3a5..708ed8321a 100644 --- a/ci/fedora-35/Dockerfile +++ b/ci/fedora-35/Dockerfile @@ -25,4 +25,4 @@ RUN dnf -y install \ zlib-devel \ && dnf clean all && rm -rf /var/cache/dnf -RUN pip3 install junit2html +RUN pip3 install websockets junit2html diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index 104be1d13b..17b95c5141 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -11,7 +11,7 @@ pkg upgrade -y curl pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")') pkg install -y $pyver-sqlite3 $pyver-pip -python -m pip install junit2html +python -m pip install websockets junit2html # Spicy detects whether it is run from build directory via `/proc`. echo "proc /proc procfs rw,noauto 0 0" >>/etc/fstab diff --git a/ci/macos/prepare.sh b/ci/macos/prepare.sh index 7c09587593..f76d948944 100755 --- a/ci/macos/prepare.sh +++ b/ci/macos/prepare.sh @@ -8,3 +8,4 @@ set -x brew update brew upgrade cmake openssl@1.1 brew install swig bison flex ccache +python3 -m pip install --user websockets diff --git a/ci/openssl-3.0/Dockerfile b/ci/openssl-3.0/Dockerfile index 98f9598b9b..69ad47a8bf 100644 --- a/ci/openssl-3.0/Dockerfile +++ b/ci/openssl-3.0/Dockerfile @@ -37,5 +37,5 @@ RUN apt-get update && apt-get -y install \ # Note - the symlink is important, otherwise cmake uses the wrong .so files. RUN wget https://www.openssl.org/source/openssl-3.0.0.tar.gz && tar xvf ./openssl-3.0.0.tar.gz && cd ./openssl-3.0.0 && ./Configure --prefix=/opt/openssl && make install && cd .. && rm -rf openssl-3.0.0 && ln -sf /opt/openssl/lib64 /opt/openssl/lib -RUN pip3 install junit2html +RUN pip3 install websockets junit2html RUN gem install coveralls-lcov diff --git a/ci/opensuse-leap-15.3/Dockerfile b/ci/opensuse-leap-15.3/Dockerfile index 36eed15c1e..b8fd970e88 100644 --- a/ci/opensuse-leap-15.3/Dockerfile +++ b/ci/opensuse-leap-15.3/Dockerfile @@ -28,7 +28,7 @@ RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.2 tar \ && rm -rf /var/cache/zypp -RUN pip3 install junit2html +RUN pip3 install websockets junit2html ENV CXX g++-9 ENV CC gcc-9 diff --git a/ci/ubuntu-18.04/Dockerfile b/ci/ubuntu-18.04/Dockerfile index 56c50eb6ae..8fc3f1326c 100644 --- a/ci/ubuntu-18.04/Dockerfile +++ b/ci/ubuntu-18.04/Dockerfile @@ -42,5 +42,5 @@ RUN apt-get update && apt-get -y install \ ENV CC=/usr/bin/clang-10 ENV CXX=/usr/bin/clang++-10 -RUN pip3 install junit2html +RUN pip3 install websockets junit2html RUN gem install coveralls-lcov diff --git a/ci/ubuntu-20.04/Dockerfile b/ci/ubuntu-20.04/Dockerfile index aaad8182ba..cc9c0fcddf 100644 --- a/ci/ubuntu-20.04/Dockerfile +++ b/ci/ubuntu-20.04/Dockerfile @@ -34,5 +34,5 @@ RUN apt-get update && apt-get -y install \ lcov \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install junit2html +RUN pip3 install websockets junit2html RUN gem install coveralls-lcov diff --git a/ci/ubuntu-21.10/Dockerfile b/ci/ubuntu-21.10/Dockerfile index 27005e95d9..47aea5c4aa 100644 --- a/ci/ubuntu-21.10/Dockerfile +++ b/ci/ubuntu-21.10/Dockerfile @@ -34,5 +34,5 @@ RUN apt-get update && apt-get -y install \ lcov \ && rm -rf /var/lib/apt/lists/* -RUN pip3 install junit2html +RUN pip3 install websockets junit2html RUN gem install coveralls-lcov diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index 891acbd601..7cf8d6991c 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -3,10 +3,18 @@ module Broker; export { - ## Default port for Broker communication. Where not specified + ## Default port for native Broker communication. Where not specified ## otherwise, this is the port to connect to and listen on. const default_port = 9999/tcp &redef; + ## Default port for Broker WebSocket communication. Where not specified + ## otherwise, this is the port to connect to and listen on for + ## WebSocket connections. + ## + ## See the Broker documentation for a specification of the message + ## format over WebSocket connections. + const default_port_websocket = 9997/tcp &redef; + ## Default interval to retry listening on a port if it's currently in ## use already. Use of the ZEEK_DEFAULT_LISTEN_RETRY environment variable ## (set as a number of seconds) will override this option and also @@ -18,6 +26,11 @@ export { ## .. zeek:see:: Broker::listen const default_listen_address = getenv("ZEEK_DEFAULT_LISTEN_ADDRESS") &redef; + ## Default address on which to listen for WebSocket connections. + ## + ## .. zeek:see:: Broker::listen_websocket + const default_listen_address_websocket = getenv("ZEEK_DEFAULT_LISTEN_ADDRESS") &redef; + ## Default interval to retry connecting to a peer if it cannot be made to ## work initially, or if it ever becomes disconnected. Use of the ## ZEEK_DEFAULT_CONNECT_RETRY environment variable (set as number of @@ -267,7 +280,7 @@ export { val: Broker::Data; }; - ## Listen for remote connections. + ## Listen for remote connections using the native Broker protocol. ## ## a: an address string on which to accept connections, e.g. ## "127.0.0.1". An empty string refers to INADDR_ANY. @@ -287,6 +300,26 @@ export { p: port &default = default_port, retry: interval &default = default_listen_retry): port; + ## Listen for remote connections using WebSocket. + ## + ## a: an address string on which to accept connections, e.g. + ## "127.0.0.1". An empty string refers to INADDR_ANY. + ## + ## p: the TCP port to listen on. The value 0 means that the OS should choose + ## the next available free port. + ## + ## retry: If non-zero, retries listening in regular intervals if the port cannot be + ## acquired immediately. 0 disables retries. If the + ## ZEEK_DEFAULT_LISTEN_RETRY environment variable is set (as number + ## of seconds), it overrides any value given here. + ## + ## Returns: the bound port or 0/? on failure. + ## + ## .. zeek:see:: Broker::status + global listen_websocket: function(a: string &default = default_listen_address_websocket, + p: port &default = default_port_websocket, + retry: interval &default = default_listen_retry): port; + ## Initiate a remote connection. ## ## a: an address to connect to, e.g. "localhost" or "127.0.0.1". @@ -473,7 +506,7 @@ event retry_listen(a: string, p: port, retry: interval) function listen(a: string, p: port, retry: interval): port { - local bound = __listen(a, p); + local bound = __listen(a, p, Broker::NATIVE); if ( bound == 0/tcp ) { @@ -489,6 +522,29 @@ function listen(a: string, p: port, retry: interval): port return bound; } +event retry_listen_websocket(a: string, p: port, retry: interval) + { + listen_websocket(a, p, retry); + } + +function listen_websocket(a: string, p: port, retry: interval): port + { + local bound = __listen(a, p, Broker::WEBSOCKET); + + if ( bound == 0/tcp ) + { + local e = getenv("ZEEK_DEFAULT_LISTEN_RETRY"); + + if ( e != "" ) + retry = double_to_interval(to_double(e)); + + if ( retry != 0secs ) + schedule retry { retry_listen_websocket(a, p, retry) }; + } + + return bound; + } + function peer(a: string, p: port, retry: interval): bool { return __peer(a, p, retry); diff --git a/src/broker/Manager.cc b/src/broker/Manager.cc index f3c6a64ee3..485dd41f57 100644 --- a/src/broker/Manager.cc +++ b/src/broker/Manager.cc @@ -527,12 +527,21 @@ void Manager::ClearStores() handle->store.clear(); } -uint16_t Manager::Listen(const string& addr, uint16_t port) +uint16_t Manager::Listen(const string& addr, uint16_t port, BrokerProtocol type) { if ( bstate->endpoint.is_shutdown() ) return 0; - bound_port = bstate->endpoint.listen(addr, port); + switch ( type ) + { + case BrokerProtocol::Native: + bound_port = bstate->endpoint.listen(addr, port); + break; + + case BrokerProtocol::WebSocket: + bound_port = bstate->endpoint.web_socket_listen(addr, port); + break; + } if ( bound_port == 0 ) Error("Failed to listen on %s:%" PRIu16, addr.empty() ? "INADDR_ANY" : addr.c_str(), port); diff --git a/src/broker/Manager.h b/src/broker/Manager.h index 8e03474c41..71e199eda8 100644 --- a/src/broker/Manager.h +++ b/src/broker/Manager.h @@ -79,6 +79,13 @@ struct Stats class Manager : public iosource::IOSource { public: + /** Broker protocol to expect on a listening port. */ + enum class BrokerProtocol + { + Native, /**< Broker's native binary protocol */ + WebSocket /** Broker's WebSocket protocol for external clients. */ + }; + static const broker::endpoint_info NoPeer; /** @@ -118,11 +125,13 @@ public: * @param port the TCP port to listen on. * @param addr an address string on which to accept connections, e.g. * "127.0.0.1". The empty string refers to @p INADDR_ANY. + * @param protocol protocol to speak on accepted connections * @return 0 on failure or the bound port otherwise. If *port* != 0, then the * return value equals *port* on success. If *port* equals 0, then the * return values represents the bound port as chosen by the OS. */ - uint16_t Listen(const std::string& addr, uint16_t port); + uint16_t Listen(const std::string& addr, uint16_t port, + BrokerProtocol protocol = BrokerProtocol::Native); /** * Initiate a peering with a remote endpoint. diff --git a/src/broker/comm.bif b/src/broker/comm.bif index e4552d84f5..c48a00620b 100644 --- a/src/broker/comm.bif +++ b/src/broker/comm.bif @@ -63,7 +63,12 @@ enum PeerStatus %{ RECONNECTING, %} -function Broker::__listen%(a: string, p: port%): port +enum BrokerProtocol %{ + NATIVE, + WEBSOCKET, +%} + +function Broker::__listen%(a: string, p: port, proto: BrokerProtocol%): port %{ zeek::Broker::Manager::ScriptScopeGuard ssg; @@ -73,7 +78,15 @@ function Broker::__listen%(a: string, p: port%): port return zeek::val_mgr->Port(0, TRANSPORT_UNKNOWN); } - auto rval = broker_mgr->Listen(a->Len() ? a->CheckString() : "", p->Port()); + zeek::Broker::Manager::BrokerProtocol proto_; + switch ( proto->AsEnum() ) + { + case BifEnum::Broker::NATIVE: proto_ = zeek::Broker::Manager::BrokerProtocol::Native; break; + case BifEnum::Broker::WEBSOCKET: proto_ = zeek::Broker::Manager::BrokerProtocol::WebSocket; break; + default: reporter->InternalError("unknown Broker protocol"); + } + + auto rval = broker_mgr->Listen(a->Len() ? a->CheckString() : "", p->Port(), proto_); return zeek::val_mgr->Port(rval, TRANSPORT_TCP); %} diff --git a/testing/btest/Baseline/broker.web-socket-events/client.output b/testing/btest/Baseline/broker.web-socket-events/client.output new file mode 100644 index 0000000000..6ecffabd1f --- /dev/null +++ b/testing/btest/Baseline/broker.web-socket-events/client.output @@ -0,0 +1,4 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +ping ['my-message', 1] +ping ['my-message', 2] +ping ['my-message', 3] diff --git a/testing/btest/Baseline/broker.web-socket-events/server.output b/testing/btest/Baseline/broker.web-socket-events/server.output new file mode 100644 index 0000000000..c1b7e176e2 --- /dev/null +++ b/testing/btest/Baseline/broker.web-socket-events/server.output @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +sender added peer: endpoint=127.0.0.1 msg=handshake successful +sender got pong: my-message, 1 +sender got pong: my-message, 2 +sender got pong: my-message, 3 +sender lost peer: endpoint=127.0.0.1 msg=lost connection to client diff --git a/testing/btest/broker/web-socket-events.zeek b/testing/btest/broker/web-socket-events.zeek new file mode 100644 index 0000000000..61cdddd864 --- /dev/null +++ b/testing/btest/broker/web-socket-events.zeek @@ -0,0 +1,124 @@ +# @TEST-GROUP: broker +# +# @TEST-PORT: BROKER_PORT +# +# @TEST-EXEC: btest-bg-run server "zeek -b %INPUT >output" +# @TEST-EXEC: btest-bg-run client "python3 ../client.py >output" +# +# @TEST-EXEC: btest-bg-wait 45 +# @TEST-EXEC: btest-diff client/output +# @TEST-EXEC: btest-diff server/output + +redef exit_only_after_terminate = T; +redef Broker::disable_ssl = T; + +global event_count = 0; + +global ping: event(msg: string, c: count); + +event zeek_init() + { + Broker::subscribe("/zeek/event/my_topic"); + Broker::listen_websocket("127.0.0.1", to_port(getenv("BROKER_PORT"))); + } + +function send_event() + { + ++event_count; + local e = Broker::make_event(ping, "my-message", event_count); + Broker::publish("/zeek/event/my_topic", e); + } + +event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("sender added peer: endpoint=%s msg=%s", endpoint$network$address, msg); + send_event(); + } + +event Broker::peer_lost(endpoint: Broker::EndpointInfo, msg: string) + { + print fmt("sender lost peer: endpoint=%s msg=%s", endpoint$network$address, msg); + terminate(); + } + +event pong(msg: string, n: count) + { + print fmt("sender got pong: %s, %s", msg, n); + send_event(); + } + + +@TEST-START-FILE client.py +import asyncio, websockets, os, time, json, sys + +ws_port = os.environ['BROKER_PORT'].split('/')[0] +ws_url = 'ws://localhost:%s/v1/messages/json' % ws_port +topic = '"/zeek/event/my_topic"' + +def broker_value(type, val): + return { + '@data-type': type, + 'data': val + } + +async def do_run(): + # Try up to 30 times. + connected = False + for i in range(30): + try: + ws = await websockets.connect(ws_url) + connected = True + + # send filter and wait for ack + await ws.send('[%s]' % topic) + ack_json = await ws.recv() + ack = json.loads(ack_json) + if not 'type' in ack or ack['type'] != 'ack': + print('*** unexpected ACK from server:') + print(ack_json) + sys.exit() + except Exception as e: + if not connected: + print('failed to connect to %s, try again (%s)' % (ws_url, e), file=sys.stderr) + time.sleep(1) + continue + else: + print('exception: %s' % e, file=sys.stderr) + sys.exit() + + for round in range(3): + # wait for ping + msg = await ws.recv() + msg = json.loads(msg) + if not 'type' in msg or msg['type'] != 'data-message': + continue + + ping = msg['data'][2]['data'] + name = ping[0]['data'] + args = [x['data'] for x in ping[1]['data']] + print(name, args) + + # send pong + pong = [broker_value('string', 'pong'), + broker_value('vector', [ + broker_value('string', args[0]), + broker_value('count', args[1]) + ])] + + ev = [broker_value('count', 1), broker_value('count', 1), broker_value('vector', pong)] + msg = { + 'type': 'data-message', + 'topic': '/zeek/event/my_topic', + '@data-type': 'vector', 'data': ev + } + + msg = json.dumps(msg) + await ws.send(msg) + + await ws.close() + sys.exit() + +loop = asyncio.get_event_loop() +loop.run_until_complete(do_run()) + +@TEST-END-FILE