Merge remote-tracking branch 'origin/topic/awelzel/javascript'

* origin/topic/awelzel/javascript:
  ci/debian-11: Install libnode-dev, too
  CMakeLists: Convert string append to list append
  Add experimental JavaScript support when libnode is available

.cirrus.yml

@@ -262,7 +262,6 @@ ubuntu2210_task:
     dockerfile: ci/ubuntu-22.10/Dockerfile
     << : *RESOURCES_TEMPLATE
   << : *CI_TEMPLATE
-  << : *SKIP_TASK_ON_PR
 
 ubuntu22_task:
   container:

.gitmodules

@@ -73,3 +73,6 @@
 [submodule "auxil/libunistd"]
 	path = auxil/libunistd
 	url = https://github.com/zeek/libunistd
+[submodule "auxil/zeekjs"]
+	path = auxil/zeekjs
+	url = https://github.com/corelight/zeekjs.git

CHANGES

@@ -1,3 +1,37 @@
+6.0.0-dev.371 | 2023-04-14 13:02:35 +0200
+
+  * ci/debian-11: Install libnode-dev, too (Arne Welzel, Corelight)
+
+    Debian 11 doesn't have a new enough libnode version, so JavaScript
+    should not be attempted to be built.
+
+  * CMakeLists: Convert string append to list append (Arne Welzel, Corelight)
+
+    Seems the builtin plugins started with string(APPEND ...) and that
+    was copied over. Make it list(APPEND ...) instead.
+
+  * Add experimental JavaScript support when libnode is available (Arne Welzel, Corelight)
+
+        zeek.on('zeek_init', () => {
+            console.log('Hello, Zeek!');
+        });
+
+    For interaction with external systems and HTTP APIs, JavaScript and the
+    Node.js ecosystem beat Zeek script. Make it more easily accessible by
+    including ZeekJS with Zeek directly.
+
+    When a recent enough libnode version is found on the build system, ZeekJS is
+    added as a builtin plugin. This behavior can be disabled via
+    ``--disable-javascript``. Linux distributions providing such a package are
+    Ubuntu (22.10) and Debian (testing/bookworm) as libnode-dev.
+    Fedora provides it as nodejs-devel.
+
+    This plugin takes over loading of .js or .cjs files. When no such files
+    are provided to Zeek, Node and the V8 engine are not initialized and
+    should not get into the way.
+
+    This should be considered experimental.
+
 6.0.0-dev.367 | 2023-04-14 10:32:17 +0200
 
   * Revert "Type: Add TypeManager->TypeList() and use for ListVal()" (Arne Welzel, Corelight)

CMakeLists.txt

@@ -897,7 +897,7 @@ if ( NOT DISABLE_SPICY )
     endif ()
 
     set(SPICY_PLUGIN_BINARY_PATH ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin)
-    string(APPEND ZEEK_INCLUDE_PLUGINS ";${SPICY_PLUGIN_PATH}")
+    list(APPEND ZEEK_INCLUDE_PLUGINS ${SPICY_PLUGIN_PATH})
 else ()
     set(HAVE_SPICY no) # evaluated by Spicy plugin build
     set(USE_SPICY_ANALYZERS no)
@@ -1042,10 +1042,30 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux )
             set(AF_PACKET_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/zeek-af_packet-plugin)
         endif ()
 
-        string(APPEND ZEEK_INCLUDE_PLUGINS ";${AF_PACKET_PLUGIN_PATH}")
+        list(APPEND ZEEK_INCLUDE_PLUGINS ${AF_PACKET_PLUGIN_PATH})
     endif ()
 endif ()
 
+if ( NOT DISABLE_JAVASCRIPT )
+    set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/auxil/zeekjs/cmake)
+    find_package(Nodejs)
+
+    if ( NODEJS_FOUND )
+        if ( ${NODEJS_VERSION} VERSION_LESS "16.13.0" )
+            message(STATUS "Node.js version ${NODEJS_VERSION} is too old, need 16.13 or later. Not enabling JavaScript support.")
+            set(ZEEK_HAVE_JAVASCRIPT no)
+        else ()
+            set(ZEEKJS_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/zeekjs)
+            list(APPEND ZEEK_INCLUDE_PLUGINS ${ZEEKJS_PLUGIN_PATH})
+            set(ZEEK_HAVE_JAVASCRIPT yes)
+        endif ()
+    else ()
+        set(ZEEK_HAVE_JAVASCRIPT no)
+    endif ()
+endif ()
+
+set(ZEEK_HAVE_JAVASCRIPT ${ZEEK_HAVE_JAVASCRIPT} CACHE INTERNAL "Zeek has JavaScript support")
+
 set(DEFAULT_ZEEKPATH_PATHS . ${ZEEK_SCRIPT_INSTALL_PATH} ${ZEEK_SCRIPT_INSTALL_PATH}/policy ${ZEEK_SCRIPT_INSTALL_PATH}/site ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins)
 if ( MSVC )
     list(JOIN DEFAULT_ZEEKPATH_PATHS ";" DEFAULT_ZEEKPATH)
@@ -1378,6 +1398,7 @@ message(
     "\nSpicy:             ${_spicy}"
     "\nSpicy plugin:      ${_spicy_plugin}"
     "\nSpicy analyzers:   ${USE_SPICY_ANALYZERS}"
+    "\nJavaScript:        ${ZEEK_HAVE_JAVASCRIPT}"
     "\n"
     "\nlibmaxminddb:      ${USE_GEOIP}"
     "\nKerberos:          ${USE_KRB5}"

NEWS

@@ -70,6 +70,29 @@ Breaking Changes
 New Functionality
 -----------------
 
+- Experimental JavaScript support added:
+
+	/* hello.js */
+	zeek.on('zeek_init', () => {
+	  console.log('Hello, Zeek!');
+	});
+
+	$ zeek ./hello.js
+	Hello, Zeek!
+
+  When a recent version of the libnode package is installed, the externally
+  maintained ZeekJS plugin (https://github.com/corelight/zeekjs) is automatically
+  included as a builtin plugin. This allows Zeek to load and execute execute
+  JavaScript code located in ``.js`` or ``.cjs`` files. When no such files are
+  passed to Zeek, the JavaScript engine and Node.js environment aren't initialized
+  and there is no runtime impact.
+
+  The Linux distributions Fedora 37, Ubuntu 22.10 and the upcoming Debian 12
+  release provide suitable packages. On other platforms, Node.js can be built
+  from source with the ``--shared`` option.
+
+  To disable this functionality, pass ``--disable-javascript`` to configure.
+
 - Introduce a new command-line option ``-V`` / ``--build-info``. It produces
   verbose output in JSON format about the repository state and any included
   plugins.

VERSION

@@ -1 +1 @@
-6.0.0-dev.367
+6.0.0-dev.371

auxil/zeekjs

@@ -0,0 +1 @@
+Subproject commit e4ae24051f31620e8bd7a93e8516797d6734b6d9

ci/debian-11/Dockerfile

@@ -17,8 +17,10 @@ RUN apt-get update && apt-get -y install \
     gcc \
     git \
     libkrb5-dev \
+    libnode-dev \
     libpcap-dev \
     libssl-dev \
+    libuv1-dev \
     make \
     python3 \
     python3-dev \

ci/debian-12/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20230405
+ENV DOCKERFILE_VERSION 20230413
 
 RUN apt-get update && apt-get -y install \
     bison \
@@ -17,8 +17,10 @@ RUN apt-get update && apt-get -y install \
     gcc \
     git \
     libkrb5-dev \
+    libnode-dev \
     libpcap-dev \
     libssl-dev \
+    libuv1-dev \
     make \
     python3 \
     python3-dev \

ci/fedora-37/Dockerfile

@@ -2,7 +2,7 @@ FROM fedora:37
 
 # A version field to invalidate Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20221127
+ENV DOCKERFILE_VERSION 20230413
 
 RUN dnf -y install \
     bison \
@@ -16,6 +16,7 @@ RUN dnf -y install \
     git \
     libpcap-devel \
     make \
+    nodejs-devel \
     openssl \
     openssl-devel \
     procps-ng \

ci/ubuntu-22.10/Dockerfile

@@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
 # A version field to invalide Cirrus's build cache when needed, as suggested in
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
-ENV DOCKERFILE_VERSION 20220614
+ENV DOCKERFILE_VERSION 20230413
 
 RUN apt-get update && apt-get -y install \
     bc \
@@ -20,8 +20,10 @@ RUN apt-get update && apt-get -y install \
     lcov \
     libkrb5-dev \
     libmaxminddb-dev \
+    libnode-dev \
     libpcap-dev \
     libssl-dev \
+    libuv1-dev \
     make \
     python3 \
     python3-dev \

configure

@@ -325,6 +325,9 @@ while [ $# -ne 0 ]; do
         --disable-cpp-tests)
             append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL false
             ;;
+        --disable-javascript)
+            append_cache_entry DISABLE_JAVASCRIPT BOOL true
+            ;;
         --disable-port-prealloc)
             append_cache_entry PREALLOCATE_PORT_ARRAY BOOL false
             ;;

testing/btest/Baseline/javascript.hello/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+Hello Zeek!

testing/btest/Baseline/javascript.http-request/out

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+http_request CHhAvVGS1DHFjwGM9 GET /download/CHANGES.bro-aux.txt 1.1

testing/btest/Baseline/javascript.http-uri-sha256/http.log

@@ -0,0 +1,2 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+{"ts":XXXXXXXXXX.XXXXXX,"uid":"CHhAvVGS1DHFjwGM9","id.orig_h":"141.142.228.5","id.orig_p":59856,"id.resp_h":"192.150.187.43","id.resp_p":80,"trans_depth":1,"method":"GET","host":"bro.org","uri":"/download/CHANGES.bro-aux.txt","version":"1.1","user_agent":"Wget/1.14 (darwin12.2.0)","request_body_len":0,"response_body_len":4705,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FMnxxt3xjVcWNS2141"],"resp_mime_types":["text/plain"],"uri_sha256":"317d15b2212888791098eeff6c021ce949d830d16f3a4b6a38c6b267c2d56317"}

testing/btest/Baseline/javascript.intel/intel.log.noheader

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+1362692526.939084	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	141.142.228.5	Intel::ADDR	Conn::IN_ORIG	zeek	Intel::ADDR	json1	-	-	-
+1362692526.939527	CHhAvVGS1DHFjwGM9	141.142.228.5	59856	192.150.187.43	80	bro.org	Intel::DOMAIN	HTTP::IN_HOST_HEADER	zeek	Intel::DOMAIN	json2	-	-	-

testing/btest/Baseline/javascript.suspend-continue/out

@@ -0,0 +1,6 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+0 suspend_processing
+0 continue_processing (delayed_enough=true)
+1362692526.939527 http_request CHhAvVGS1DHFjwGM9 GET <...>/CHANGES.bro-aux.txt 1.1
+1362692527.080972 Pcap::file_done <...>/get.trace
+1362692527.080972 zeek_done

testing/btest/btest.cfg

@@ -4,7 +4,7 @@
 build_dir = build
 
 [btest]
-TestDirs    = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry
+TestDirs    = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry javascript
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp

testing/btest/coverage/bare-load-baseline.test

@@ -14,5 +14,5 @@
 # @TEST-EXEC: cat loaded_scripts.log | grep -E -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log
-# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
+# @TEST-EXEC: grep -E -v 'Zeek_(AF_Packet|JavaScript)' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/coverage/default-load-baseline.test

@@ -13,5 +13,5 @@
 # @TEST-EXEC: cat loaded_scripts.log | grep -E -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix
 # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix
 # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log
-# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
+# @TEST-EXEC: grep -E -v 'Zeek_(AF_Packet|JavaScript)' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log
 # @TEST-EXEC: btest-diff canonified_loaded_scripts.log

testing/btest/javascript/hello.js

@@ -0,0 +1,9 @@
+/*
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b %INPUT > out
+ * @TEST-EXEC: btest-diff out
+ */
+
+zeek.on('zeek_init', () => {
+  console.log('Hello Zeek!');
+});

testing/btest/javascript/http-request.js

@@ -0,0 +1,9 @@
+/*
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace base/protocols/http %INPUT > out
+ * @TEST-EXEC: btest-diff out
+ */
+
+zeek.on('http_request', (c, method, orig_URI, escaped_URI, version) => {
+  console.log(`http_request ${c.uid} ${method} ${orig_URI} ${version}`);
+});

testing/btest/javascript/http-uri-sha256.js

@@ -0,0 +1,30 @@
+/*
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace main.zeek LogAscii::use_json=T
+ * @TEST-EXEC: btest-diff http.log
+ */
+@TEST-START-FILE main.zeek
+@load base/protocols/http
+
+# Extending log records only works in Zeek script.
+redef record HTTP::Info += {
+  ## The sha256 value of the orig_URI.
+  uri_sha256: string &optional &log;
+};
+
+# Load the JavaScript pieces
+@load ./main.js
+@TEST-END-FILE
+
+@TEST-START-FILE main.js
+const crypto = require('crypto');
+
+/*
+ * We can set fields directly on c.http from JavaScript and they'll appear
+ * in the http.log record. In this case, we compute the sha256 hash of
+ * the orig_URI and log it.
+ */
+zeek.on('http_request', { priority: -10 }, (c, method, orig_URI, escaped_URI, version) => {
+  c.http.uri_sha256 = crypto.createHash('sha256').update(orig_URI).digest().toString('hex');
+});
+@TEST-END-FILE

testing/btest/javascript/intel.js

@@ -0,0 +1,33 @@
+/*
+ * @TEST-DOC: Load intel data from a JSON file and populate via Intel::insert().
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace frameworks/intel/seen base/frameworks/intel base/protocols/http %INPUT
+ * @TEST-EXEC: zeek-cut < intel.log > intel.log.noheader
+ * @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff intel.log.noheader
+ *
+ * Following the intel file that we load via Intel::insert().
+@TEST-START-FILE intel.json_lines
+{"indicator": "141.142.228.5", "indicator_type": "Intel::ADDR", "meta": {"source": "json1"}}
+{"indicator": "bro.org", "indicator_type": "Intel::DOMAIN", "meta": {"source": "json2"}}
+@TEST-END-FILE
+*/
+const fs = require('fs');
+
+zeek.on('zeek_init', () => {
+  // Hold the packet processing until we've read the intel file.
+  zeek.invoke('suspend_processing');
+
+  // This reads the full file into memory, but is still async.
+  // There's fs.createReadStream() for the piecewise consumption.
+  fs.readFile('./intel.json_lines', 'utf8', (err, data) => {
+    for (const l of data.split('\n')) {
+      if (l.length == 0)
+        continue;
+
+      zeek.invoke('Intel::insert', [JSON.parse(l)]);
+    }
+
+    /* Once all intel data is loaded, continue processing. */
+    zeek.invoke('continue_processing');
+  });
+});

testing/btest/javascript/suspend-continue.js

@@ -0,0 +1,39 @@
+/*
+ * @TEST-DOC: Demo suspend and continue processing from JavaScript
+ * @TEST-REQUIRES: $SCRIPTS/have-javascript
+ * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace base/protocols/http %INPUT > out
+ * @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out
+ */
+zeek.on('zeek_init', () => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} suspend_processing`);
+  zeek.invoke('suspend_processing');
+  const suspended_at = Date.now();
+
+  // Schedule a JavaScript timer (running based on wallclock)
+  // to continue execution in 333 msec.
+  setTimeout(() => {
+    const nt = zeek.invoke('network_time');
+    const continued_at = Date.now();
+    const delayed_ms = continued_at - suspended_at;
+    const delayed_enough = delayed_ms > 300;
+
+    console.log(`${nt} continue_processing (delayed_enough=${delayed_enough})`);
+    zeek.invoke('continue_processing');
+  }, 333);
+});
+
+zeek.on('http_request', (c, method, orig_URI, escaped_URI, version) => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} http_request ${c.uid} ${method} ${orig_URI} ${version}`);
+});
+
+zeek.on('Pcap::file_done', (path) => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} Pcap::file_done ${path}`);
+});
+
+zeek.on('zeek_done', () => {
+  const nt = zeek.invoke('network_time');
+  console.log(`${nt} zeek_done`);
+});

testing/btest/plugins/hooks-plugin/src/Plugin.cc

@@ -36,6 +36,7 @@ static std::set<std::string> sanitized_functions = {
 // contains any of these keywords, no log message is generated.
 static std::set<std::string> load_file_filter = {
 	"Zeek_AF_Packet",
+	"Zeek_JavaScript",
 };
 
 static bool skip_load_file_logging_for(const std::string& s)

testing/scripts/have-javascript

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if grep -q "ZEEK_HAVE_JAVASCRIPT:INTERNAL=yes" "${BUILD}"/CMakeCache.txt; then
+    exit 0
+fi
+
+exit 1