Add experimental JavaScript support when libnode is available

zeek.on('zeek_init', () => {
        console.log('Hello, Zeek!');
    });

For interaction with external systems and HTTP APIs, JavaScript and the
Node.js ecosystem beat Zeek script. Make it more easily accessible by
including ZeekJS with Zeek directly.

When a recent enough libnode version is found on the build system, ZeekJS is
added as a builtin plugin. This behavior can be disabled via
``--disable-javascript``. Linux distributions providing such a package are
Ubuntu (22.10) and Debian (testing/bookworm) as libnode-dev.
Fedora provides it as nodejs-devel.

This plugin takes over loading of .js or .cjs files. When no such files
are provided to Zeek, Node and the V8 engine are not initialized and
should not get into the way.

This should be considered experimental.

.cirrus.yml

@@ -262,7 +262,6 @@ ubuntu2210_task:
     dockerfile: ci/ubuntu-22.10/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
 
 ubuntu22_task:
   container:

.gitmodules

@@ -73,3 +73,6 @@
 [submodule "auxil/libunistd"]
 	path = auxil/libunistd
 	url = https://github.com/zeek/libunistd
+[submodule "auxil/zeekjs"]
+	path = auxil/zeekjs
+	url = https://github.com/corelight/zeekjs.git

CMakeLists.txt

@@ -1046,6 +1046,26 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux )
     endif ()
 endif ()
 
+if ( NOT DISABLE_JAVASCRIPT )
+    set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/auxil/zeekjs/cmake)
+    find_package(Nodejs)
+
+    if ( NODEJS_FOUND )
+        if ( ${NODEJS_VERSION} VERSION_LESS "16.13.0" )
+            message(STATUS "Node.js version ${NODEJS_VERSION} is too old, need 16.13 or later. Not enabling JavaScript support.")
+            set(ZEEK_HAVE_JAVASCRIPT no)
+        else ()
+            set(ZEEKJS_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/zeekjs)
+            string(APPEND ZEEK_INCLUDE_PLUGINS ";${ZEEKJS_PLUGIN_PATH}")
+            set(ZEEK_HAVE_JAVASCRIPT yes)
+        endif ()
+    else ()
+        set(ZEEK_HAVE_JAVASCRIPT no)
+    endif ()
+endif ()
+
+set(ZEEK_HAVE_JAVASCRIPT ${ZEEK_HAVE_JAVASCRIPT} CACHE INTERNAL "Zeek has JavaScript support")
+
 set(DEFAULT_ZEEKPATH_PATHS . ${ZEEK_SCRIPT_INSTALL_PATH} ${ZEEK_SCRIPT_INSTALL_PATH}/policy ${ZEEK_SCRIPT_INSTALL_PATH}/site ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins)
 if ( MSVC )
     list(JOIN DEFAULT_ZEEKPATH_PATHS ";" DEFAULT_ZEEKPATH)
@@ -1378,6 +1398,7 @@ message(
     "\nSpicy:             ${_spicy}"
     "\nSpicy plugin:      ${_spicy_plugin}"
     "\nSpicy analyzers:   ${USE_SPICY_ANALYZERS}"
+    "\nJavaScript:        ${ZEEK_HAVE_JAVASCRIPT}"
     "\n"
     "\nlibmaxminddb:      ${USE_GEOIP}"
     "\nKerberos:          ${USE_KRB5}"

NEWS

@@ -70,6 +70,29 @@ Breaking Changes
 New Functionality
 -----------------
 
+- Experimental JavaScript support added:
+
+	/* hello.js */
+	zeek.on('zeek_init', () => {
+	  console.log('Hello, Zeek!');
+	});
+
+	$ zeek ./hello.js
+	Hello, Zeek!
+
+  When a recent version of the libnode package is installed, the externally
+  maintained ZeekJS plugin (https://github.com/corelight/zeekjs) is automatically
+  included as a builtin plugin. This allows Zeek to load and execute execute
+  JavaScript code located in ``.js`` or ``.cjs`` files. When no such files are
+  passed to Zeek, the JavaScript engine and Node.js environment aren't initialized
+  and there is no runtime impact.
+
+  The Linux distributions Fedora 37, Ubuntu 22.10 and the upcoming Debian 12
+  release provide suitable packages. On other platforms, Node.js can be built
+  from source with the ``--shared`` option.
+
+  To disable this functionality, pass ``--disable-javascript`` to configure.
+
 - Introduce a new command-line option ``-V`` / ``--build-info``. It produces
   verbose output in JSON format about the repository state and any included
   plugins.

auxil/zeekjs

@@ -0,0 +1 @@
+Subproject commit e4ae24051f31620e8bd7a93e8516797d6734b6d9

ci/debian-12/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20230405
+ENV DOCKERFILE_VERSION 20230413
 
 RUN apt-get update && apt-get -y install \
     bison \
@@ -17,8 +17,10 @@ RUN apt-get update && apt-get -y install \
     gcc \
     git \
     libkrb5-dev \
+    libnode-dev \
     libpcap-dev \
     libssl-dev \
+    libuv1-dev \
     make \
     python3 \
     python3-dev \

ci/fedora-37/Dockerfile

@@ -2,7 +2,7 @@ FROM fedora:37
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20221127
+ENV DOCKERFILE_VERSION 20230413
 
 RUN dnf -y install \
     bison \
@@ -16,6 +16,7 @@ RUN dnf -y install \
     git \
     libpcap-devel \
     make \
+    nodejs-devel \
     openssl \
     openssl-devel \
     procps-ng \

ci/ubuntu-22.10/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalide Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20220614
+ENV DOCKERFILE_VERSION 20230413
 
 RUN apt-get update && apt-get -y install \
     bc \
@@ -20,8 +20,10 @@ RUN apt-get update && apt-get -y install \
     lcov \
     libkrb5-dev \
     libmaxminddb-dev \
+    libnode-dev \
     libpcap-dev \
     libssl-dev \
+    libuv1-dev \
     make \
     python3 \
     python3-dev \

configure

@@ -325,6 +325,9 @@ while [ $# -ne 0 ]; do
         --disable-cpp-tests)
             append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL false
             ;;
+        --disable-javascript)
+            append_cache_entry DISABLE_JAVASCRIPT BOOL true
+            ;;
         --disable-port-prealloc)
             append_cache_entry PREALLOCATE_PORT_ARRAY BOOL false
             ;;

testing/btest/Baseline/javascript.hello/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Hello Zeek!

testing/btest/Baseline/javascript.http-request/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+http_request CHhAvVGS1DHFjwGM9 GET /download/CHANGES.bro-aux.txt 1.1

testing/btest/Baseline/javascript.http-uri-sha256/http.log

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+{"ts":XXXXXXXXXX.XXXXXX,"uid":"CHhAvVGS1DHFjwGM9","id.orig_h":"141.142.228.5","id.orig_p":59856,"id.resp_h":"192.150.187.43","id.resp_p":80,"trans_depth":1,"method":"GET","host":"bro.org","uri":"/download/CHANGES.bro-aux.txt","version":"1.1","user_agent":"Wget/1.14 (darwin12.2.0)","request_body_len":0,"response_body_len":4705,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FMnxxt3xjVcWNS2141"],"resp_mime_types":["text/plain"],"uri_sha256":"317d15b2212888791098eeff6c021ce949d830d16f3a4b6a38c6b267c2d56317"}

testing/btest/Baseline/javascript.intel/intel.log.noheader

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+1362692526.939084	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	141.142.228.5	Intel::ADDR	Conn::IN_ORIG	zeek	Intel::ADDR	json1	-	-	-
+1362692526.939527	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	bro.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	zeek	Intel::DOMAIN	json2	-	-	-

testing/btest/Baseline/javascript.suspend-continue/out

@@ -0,0 +1,6 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+0 suspend_processing
+0 continue_processing (delayed_enough=true)
+1362692526.939527 http_request CHhAvVGS1DHFjwGM9 GET <...>/CHANGES.bro-aux.txt 1.1
+1362692527.080972 Pcap::file_done <...>/get.trace
+1362692527.080972 zeek_done

testing/btest/btest.cfg

@@ -4,7 +4,7 @@
 build_dir = build
 
 [btest]
-TestDirs    = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry
+TestDirs    = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry javascript
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp

testing/btest/coverage/bare-load-baseline.test

@@ -14,5 +14,5 @@
 # @TEST-EXEC: cat loaded_scripts.log | grep -E -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log
-# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
+# @TEST-EXEC: grep -E -v 'Zeek_(AF_Packet|JavaScript)' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/coverage/default-load-baseline.test

@@ -13,5 +13,5 @@
 # @TEST-EXEC: cat loaded_scripts.log | grep -E -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log
-# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
+# @TEST-EXEC: grep -E -v 'Zeek_(AF_Packet|JavaScript)' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/javascript/hello.js

@@ -0,0 +1,9 @@
+/*
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b %INPUT > out
+ * @TEST-EXEC: btest-diff out
+ */
+
+zeek.on('zeek_init', () => {
+  console.log('Hello Zeek!');
+});

testing/btest/javascript/http-request.js

@@ -0,0 +1,9 @@
+/*
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace base/protocols/http %INPUT > out
+ * @TEST-EXEC: btest-diff out
+ */
+
+zeek.on('http_request', (c, method, orig_URI, escaped_URI, version) => {
+  console.log(`http_request ${c.uid} ${method} ${orig_URI} ${version}`);
+});

testing/btest/javascript/http-uri-sha256.js

@@ -0,0 +1,30 @@
+/*
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace main.zeek LogAscii::use_json=T
+ * @TEST-EXEC: btest-diff http.log
+ */
+@TEST-START-FILE main.zeek
+@load base/protocols/http
+
+# Extending log records only works in Zeek script.
+redef record HTTP::Info += {
+  ## The sha256 value of the orig_URI.
+  uri_sha256: string &optional &log;
+};
+
+# Load the JavaScript pieces
+@load ./main.js
+@TEST-END-FILE
+
+@TEST-START-FILE main.js
+const crypto = require('crypto');
+
+/*
+ * We can set fields directly on c.http from JavaScript and they'll appear
+ * in the http.log record. In this case, we compute the sha256 hash of
+ * the orig_URI and log it.
+ */
+zeek.on('http_request', { priority: -10 }, (c, method, orig_URI, escaped_URI, version) => {
+  c.http.uri_sha256 = crypto.createHash('sha256').update(orig_URI).digest().toString('hex');
+});
+@TEST-END-FILE

testing/btest/javascript/intel.js

@@ -0,0 +1,33 @@
+/*
+ * @TEST-DOC: Load intel data from a JSON file and populate via Intel::insert().
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace frameworks/intel/seen base/frameworks/intel base/protocols/http %INPUT
+ * @TEST-EXEC: zeek-cut < intel.log > intel.log.noheader
+ * @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff intel.log.noheader
+ *
+ * Following the intel file that we load via Intel::insert().
+@TEST-START-FILE intel.json_lines
+{"indicator": "141.142.228.5", "indicator_type": "Intel::ADDR", "meta": {"source": "json1"}}
+{"indicator": "bro.org", "indicator_type": "Intel::DOMAIN", "meta": {"source": "json2"}}
+@TEST-END-FILE
+*/
+const fs = require('fs');
+
+zeek.on('zeek_init', () => {
+  // Hold the packet processing until we've read the intel file.
+  zeek.invoke('suspend_processing');
+
+  // This reads the full file into memory, but is still async.
+  // There's fs.createReadStream() for the piecewise consumption.
+  fs.readFile('./intel.json_lines', 'utf8', (err, data) => {
+    for (const l of data.split('\n')) {
+      if (l.length == 0)
+        continue;
+
+      zeek.invoke('Intel::insert', [JSON.parse(l)]);
+    }
+
+    /* Once all intel data is loaded, continue processing. */
+    zeek.invoke('continue_processing');
+  });
+});

testing/btest/javascript/suspend-continue.js

@@ -0,0 +1,39 @@
+/*
+ * @TEST-DOC: Demo suspend and continue processing from JavaScript
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace base/protocols/http %INPUT > out
+ * @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+ */
+zeek.on('zeek_init', () => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} suspend_processing`);
+  zeek.invoke('suspend_processing');
+  const suspended_at = Date.now();
+
+  // Schedule a JavaScript timer (running based on wallclock)
+  // to continue execution in 333 msec.
+  setTimeout(() => {
+    const nt = zeek.invoke('network_time');
+    const continued_at = Date.now();
+    const delayed_ms = continued_at - suspended_at;
+    const delayed_enough = delayed_ms > 300;
+
+    console.log(`${nt} continue_processing (delayed_enough=${delayed_enough})`);
+    zeek.invoke('continue_processing');
+  }, 333);
+});
+
+zeek.on('http_request', (c, method, orig_URI, escaped_URI, version) => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} http_request ${c.uid} ${method} ${orig_URI} ${version}`);
+});
+
+zeek.on('Pcap::file_done', (path) => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} Pcap::file_done ${path}`);
+});
+
+zeek.on('zeek_done', () => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} zeek_done`);
+});

testing/btest/plugins/hooks-plugin/src/Plugin.cc

@@ -36,6 +36,7 @@ static std::set<std::string> sanitized_functions = {
 // contains any of these keywords, no log message is generated.
 static std::set<std::string> load_file_filter = {
 	"Zeek_AF_Packet",
+	"Zeek_JavaScript",
 };
 
 static bool skip_load_file_logging_for(const std::string& s)

testing/scripts/have-javascript

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if grep -q "ZEEK_HAVE_JAVASCRIPT:INTERNAL=yes" "${BUILD}"/CMakeCache.txt; then
+    exit 0
+fi
+
+exit 1