From f0b9c59adb0eaf3b303d1a7113be540bdf199e42 Mon Sep 17 00:00:00 2001 From: Arne Welzel Date: Sat, 18 Mar 2023 17:42:08 +0100 Subject: [PATCH] Add experimental JavaScript support when libnode is available zeek.on('zeek_init', () => { console.log('Hello, Zeek!'); }); For interaction with external systems and HTTP APIs, JavaScript and the Node.js ecosystem beat Zeek script. Make it more easily accessible by including ZeekJS with Zeek directly. When a recent enough libnode version is found on the build system, ZeekJS is added as a builtin plugin. This behavior can be disabled via ``--disable-javascript``. Linux distributions providing such a package are Ubuntu (22.10) and Debian (testing/bookworm) as libnode-dev. Fedora provides it as nodejs-devel. This plugin takes over loading of .js or .cjs files. When no such files are provided to Zeek, Node and the V8 engine are not initialized and should not get into the way. This should be considered experimental. --- .cirrus.yml | 1 - .gitmodules | 3 ++ CMakeLists.txt | 21 ++++++++++ NEWS | 23 +++++++++++ auxil/zeekjs | 1 + ci/debian-12/Dockerfile | 4 +- ci/fedora-37/Dockerfile | 3 +- ci/ubuntu-22.10/Dockerfile | 4 +- configure | 3 ++ testing/btest/Baseline/javascript.hello/out | 2 + .../Baseline/javascript.http-request/out | 2 + .../javascript.http-uri-sha256/http.log | 2 + .../javascript.intel/intel.log.noheader | 3 ++ .../Baseline/javascript.suspend-continue/out | 6 +++ testing/btest/btest.cfg | 2 +- .../btest/coverage/bare-load-baseline.test | 2 +- .../btest/coverage/default-load-baseline.test | 2 +- testing/btest/javascript/hello.js | 9 +++++ testing/btest/javascript/http-request.js | 9 +++++ testing/btest/javascript/http-uri-sha256.js | 30 ++++++++++++++ testing/btest/javascript/intel.js | 33 ++++++++++++++++ testing/btest/javascript/suspend-continue.js | 39 +++++++++++++++++++ .../btest/plugins/hooks-plugin/src/Plugin.cc | 1 + testing/scripts/have-javascript | 7 ++++ 24 files changed, 205 insertions(+), 7 deletions(-) create mode 160000 auxil/zeekjs create mode 100644 testing/btest/Baseline/javascript.hello/out create mode 100644 testing/btest/Baseline/javascript.http-request/out create mode 100644 testing/btest/Baseline/javascript.http-uri-sha256/http.log create mode 100644 testing/btest/Baseline/javascript.intel/intel.log.noheader create mode 100644 testing/btest/Baseline/javascript.suspend-continue/out create mode 100644 testing/btest/javascript/hello.js create mode 100644 testing/btest/javascript/http-request.js create mode 100644 testing/btest/javascript/http-uri-sha256.js create mode 100644 testing/btest/javascript/intel.js create mode 100644 testing/btest/javascript/suspend-continue.js create mode 100755 testing/scripts/have-javascript diff --git a/.cirrus.yml b/.cirrus.yml index c86576cfa6..94b17feaca 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -262,7 +262,6 @@ ubuntu2210_task: dockerfile: ci/ubuntu-22.10/Dockerfile << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE - << : *SKIP_TASK_ON_PR ubuntu22_task: container: diff --git a/.gitmodules b/.gitmodules index 13774cbbfd..e971a79201 100644 --- a/.gitmodules +++ b/.gitmodules @@ -73,3 +73,6 @@ [submodule "auxil/libunistd"] path = auxil/libunistd url = https://github.com/zeek/libunistd +[submodule "auxil/zeekjs"] + path = auxil/zeekjs + url = https://github.com/corelight/zeekjs.git diff --git a/CMakeLists.txt b/CMakeLists.txt index e0e72c27bd..03b1d66379 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1046,6 +1046,26 @@ if ( ${CMAKE_SYSTEM_NAME} MATCHES Linux ) endif () endif () +if ( NOT DISABLE_JAVASCRIPT ) + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} ${PROJECT_SOURCE_DIR}/auxil/zeekjs/cmake) + find_package(Nodejs) + + if ( NODEJS_FOUND ) + if ( ${NODEJS_VERSION} VERSION_LESS "16.13.0" ) + message(STATUS "Node.js version ${NODEJS_VERSION} is too old, need 16.13 or later. Not enabling JavaScript support.") + set(ZEEK_HAVE_JAVASCRIPT no) + else () + set(ZEEKJS_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/zeekjs) + string(APPEND ZEEK_INCLUDE_PLUGINS ";${ZEEKJS_PLUGIN_PATH}") + set(ZEEK_HAVE_JAVASCRIPT yes) + endif () + else () + set(ZEEK_HAVE_JAVASCRIPT no) + endif () +endif () + +set(ZEEK_HAVE_JAVASCRIPT ${ZEEK_HAVE_JAVASCRIPT} CACHE INTERNAL "Zeek has JavaScript support") + set(DEFAULT_ZEEKPATH_PATHS . ${ZEEK_SCRIPT_INSTALL_PATH} ${ZEEK_SCRIPT_INSTALL_PATH}/policy ${ZEEK_SCRIPT_INSTALL_PATH}/site ${ZEEK_SCRIPT_INSTALL_PATH}/builtin-plugins) if ( MSVC ) list(JOIN DEFAULT_ZEEKPATH_PATHS ";" DEFAULT_ZEEKPATH) @@ -1378,6 +1398,7 @@ message( "\nSpicy: ${_spicy}" "\nSpicy plugin: ${_spicy_plugin}" "\nSpicy analyzers: ${USE_SPICY_ANALYZERS}" + "\nJavaScript: ${ZEEK_HAVE_JAVASCRIPT}" "\n" "\nlibmaxminddb: ${USE_GEOIP}" "\nKerberos: ${USE_KRB5}" diff --git a/NEWS b/NEWS index 0996d3383a..6499ddbc52 100644 --- a/NEWS +++ b/NEWS @@ -70,6 +70,29 @@ Breaking Changes New Functionality ----------------- +- Experimental JavaScript support added: + + /* hello.js */ + zeek.on('zeek_init', () => { + console.log('Hello, Zeek!'); + }); + + $ zeek ./hello.js + Hello, Zeek! + + When a recent version of the libnode package is installed, the externally + maintained ZeekJS plugin (https://github.com/corelight/zeekjs) is automatically + included as a builtin plugin. This allows Zeek to load and execute execute + JavaScript code located in ``.js`` or ``.cjs`` files. When no such files are + passed to Zeek, the JavaScript engine and Node.js environment aren't initialized + and there is no runtime impact. + + The Linux distributions Fedora 37, Ubuntu 22.10 and the upcoming Debian 12 + release provide suitable packages. On other platforms, Node.js can be built + from source with the ``--shared`` option. + + To disable this functionality, pass ``--disable-javascript`` to configure. + - Introduce a new command-line option ``-V`` / ``--build-info``. It produces verbose output in JSON format about the repository state and any included plugins. diff --git a/auxil/zeekjs b/auxil/zeekjs new file mode 160000 index 0000000000..e4ae24051f --- /dev/null +++ b/auxil/zeekjs @@ -0,0 +1 @@ +Subproject commit e4ae24051f31620e8bd7a93e8516797d6734b6d9 diff --git a/ci/debian-12/Dockerfile b/ci/debian-12/Dockerfile index dc892c889c..2331a99a70 100644 --- a/ci/debian-12/Dockerfile +++ b/ci/debian-12/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20230405 +ENV DOCKERFILE_VERSION 20230413 RUN apt-get update && apt-get -y install \ bison \ @@ -17,8 +17,10 @@ RUN apt-get update && apt-get -y install \ gcc \ git \ libkrb5-dev \ + libnode-dev \ libpcap-dev \ libssl-dev \ + libuv1-dev \ make \ python3 \ python3-dev \ diff --git a/ci/fedora-37/Dockerfile b/ci/fedora-37/Dockerfile index 7010400611..7f278a08a9 100644 --- a/ci/fedora-37/Dockerfile +++ b/ci/fedora-37/Dockerfile @@ -2,7 +2,7 @@ FROM fedora:37 # A version field to invalidate Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20221127 +ENV DOCKERFILE_VERSION 20230413 RUN dnf -y install \ bison \ @@ -16,6 +16,7 @@ RUN dnf -y install \ git \ libpcap-devel \ make \ + nodejs-devel \ openssl \ openssl-devel \ procps-ng \ diff --git a/ci/ubuntu-22.10/Dockerfile b/ci/ubuntu-22.10/Dockerfile index 6c51387fcc..b34ae527a8 100644 --- a/ci/ubuntu-22.10/Dockerfile +++ b/ci/ubuntu-22.10/Dockerfile @@ -4,7 +4,7 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # A version field to invalide Cirrus's build cache when needed, as suggested in # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 -ENV DOCKERFILE_VERSION 20220614 +ENV DOCKERFILE_VERSION 20230413 RUN apt-get update && apt-get -y install \ bc \ @@ -20,8 +20,10 @@ RUN apt-get update && apt-get -y install \ lcov \ libkrb5-dev \ libmaxminddb-dev \ + libnode-dev \ libpcap-dev \ libssl-dev \ + libuv1-dev \ make \ python3 \ python3-dev \ diff --git a/configure b/configure index d057261f88..b9d9e16cb9 100755 --- a/configure +++ b/configure @@ -325,6 +325,9 @@ while [ $# -ne 0 ]; do --disable-cpp-tests) append_cache_entry ENABLE_ZEEK_UNIT_TESTS BOOL false ;; + --disable-javascript) + append_cache_entry DISABLE_JAVASCRIPT BOOL true + ;; --disable-port-prealloc) append_cache_entry PREALLOCATE_PORT_ARRAY BOOL false ;; diff --git a/testing/btest/Baseline/javascript.hello/out b/testing/btest/Baseline/javascript.hello/out new file mode 100644 index 0000000000..ea07615cca --- /dev/null +++ b/testing/btest/Baseline/javascript.hello/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +Hello Zeek! diff --git a/testing/btest/Baseline/javascript.http-request/out b/testing/btest/Baseline/javascript.http-request/out new file mode 100644 index 0000000000..62b431d00b --- /dev/null +++ b/testing/btest/Baseline/javascript.http-request/out @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +http_request CHhAvVGS1DHFjwGM9 GET /download/CHANGES.bro-aux.txt 1.1 diff --git a/testing/btest/Baseline/javascript.http-uri-sha256/http.log b/testing/btest/Baseline/javascript.http-uri-sha256/http.log new file mode 100644 index 0000000000..462394e5f6 --- /dev/null +++ b/testing/btest/Baseline/javascript.http-uri-sha256/http.log @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +{"ts":XXXXXXXXXX.XXXXXX,"uid":"CHhAvVGS1DHFjwGM9","id.orig_h":"141.142.228.5","id.orig_p":59856,"id.resp_h":"192.150.187.43","id.resp_p":80,"trans_depth":1,"method":"GET","host":"bro.org","uri":"/download/CHANGES.bro-aux.txt","version":"1.1","user_agent":"Wget/1.14 (darwin12.2.0)","request_body_len":0,"response_body_len":4705,"status_code":200,"status_msg":"OK","tags":[],"resp_fuids":["FMnxxt3xjVcWNS2141"],"resp_mime_types":["text/plain"],"uri_sha256":"317d15b2212888791098eeff6c021ce949d830d16f3a4b6a38c6b267c2d56317"} diff --git a/testing/btest/Baseline/javascript.intel/intel.log.noheader b/testing/btest/Baseline/javascript.intel/intel.log.noheader new file mode 100644 index 0000000000..6ba6ae139f --- /dev/null +++ b/testing/btest/Baseline/javascript.intel/intel.log.noheader @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +1362692526.939084 CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 141.142.228.5 Intel::ADDR Conn::IN_ORIG zeek Intel::ADDR json1 - - - +1362692526.939527 CHhAvVGS1DHFjwGM9 141.142.228.5 59856 192.150.187.43 80 bro.org Intel::DOMAIN HTTP::IN_HOST_HEADER zeek Intel::DOMAIN json2 - - - diff --git a/testing/btest/Baseline/javascript.suspend-continue/out b/testing/btest/Baseline/javascript.suspend-continue/out new file mode 100644 index 0000000000..3efae53419 --- /dev/null +++ b/testing/btest/Baseline/javascript.suspend-continue/out @@ -0,0 +1,6 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +0 suspend_processing +0 continue_processing (delayed_enough=true) +1362692526.939527 http_request CHhAvVGS1DHFjwGM9 GET <...>/CHANGES.bro-aux.txt 1.1 +1362692527.080972 Pcap::file_done <...>/get.trace +1362692527.080972 zeek_done diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index f7cf5fdaf5..447715534d 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -4,7 +4,7 @@ build_dir = build [btest] -TestDirs = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry +TestDirs = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry javascript TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp diff --git a/testing/btest/coverage/bare-load-baseline.test b/testing/btest/coverage/bare-load-baseline.test index f992799b5c..ba785dde8c 100644 --- a/testing/btest/coverage/bare-load-baseline.test +++ b/testing/btest/coverage/bare-load-baseline.test @@ -14,5 +14,5 @@ # @TEST-EXEC: cat loaded_scripts.log | grep -E -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log -# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log +# @TEST-EXEC: grep -E -v 'Zeek_(AF_Packet|JavaScript)' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log # @TEST-EXEC: btest-diff canonified_loaded_scripts.log diff --git a/testing/btest/coverage/default-load-baseline.test b/testing/btest/coverage/default-load-baseline.test index b94172cbc7..9b9fc1f1f0 100644 --- a/testing/btest/coverage/default-load-baseline.test +++ b/testing/btest/coverage/default-load-baseline.test @@ -13,5 +13,5 @@ # @TEST-EXEC: cat loaded_scripts.log | grep -E -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix # @TEST-EXEC: (test -L $BUILD && basename $(readlink $BUILD) || basename $BUILD) >buildprefix # @TEST-EXEC: cat loaded_scripts.log | sed "s#`cat buildprefix`#build#g" | sed "s#`cat prefix`##g" >prefix_canonified_loaded_scripts.log -# @TEST-EXEC: grep -v 'Zeek_AF_Packet' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log +# @TEST-EXEC: grep -E -v 'Zeek_(AF_Packet|JavaScript)' prefix_canonified_loaded_scripts.log > canonified_loaded_scripts.log # @TEST-EXEC: btest-diff canonified_loaded_scripts.log diff --git a/testing/btest/javascript/hello.js b/testing/btest/javascript/hello.js new file mode 100644 index 0000000000..1af791b9b1 --- /dev/null +++ b/testing/btest/javascript/hello.js @@ -0,0 +1,9 @@ +/* + * @TEST-REQUIRES: $SCRIPTS/have-javascript + * @TEST-EXEC: zeek -b %INPUT > out + * @TEST-EXEC: btest-diff out + */ + +zeek.on('zeek_init', () => { + console.log('Hello Zeek!'); +}); diff --git a/testing/btest/javascript/http-request.js b/testing/btest/javascript/http-request.js new file mode 100644 index 0000000000..92c3c8008d --- /dev/null +++ b/testing/btest/javascript/http-request.js @@ -0,0 +1,9 @@ +/* + * @TEST-REQUIRES: $SCRIPTS/have-javascript + * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace base/protocols/http %INPUT > out + * @TEST-EXEC: btest-diff out + */ + +zeek.on('http_request', (c, method, orig_URI, escaped_URI, version) => { + console.log(`http_request ${c.uid} ${method} ${orig_URI} ${version}`); +}); diff --git a/testing/btest/javascript/http-uri-sha256.js b/testing/btest/javascript/http-uri-sha256.js new file mode 100644 index 0000000000..8673db3815 --- /dev/null +++ b/testing/btest/javascript/http-uri-sha256.js @@ -0,0 +1,30 @@ +/* + * @TEST-REQUIRES: $SCRIPTS/have-javascript + * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace main.zeek LogAscii::use_json=T + * @TEST-EXEC: btest-diff http.log + */ +@TEST-START-FILE main.zeek +@load base/protocols/http + +# Extending log records only works in Zeek script. +redef record HTTP::Info += { + ## The sha256 value of the orig_URI. + uri_sha256: string &optional &log; +}; + +# Load the JavaScript pieces +@load ./main.js +@TEST-END-FILE + +@TEST-START-FILE main.js +const crypto = require('crypto'); + +/* + * We can set fields directly on c.http from JavaScript and they'll appear + * in the http.log record. In this case, we compute the sha256 hash of + * the orig_URI and log it. + */ +zeek.on('http_request', { priority: -10 }, (c, method, orig_URI, escaped_URI, version) => { + c.http.uri_sha256 = crypto.createHash('sha256').update(orig_URI).digest().toString('hex'); +}); +@TEST-END-FILE diff --git a/testing/btest/javascript/intel.js b/testing/btest/javascript/intel.js new file mode 100644 index 0000000000..67d2fd3598 --- /dev/null +++ b/testing/btest/javascript/intel.js @@ -0,0 +1,33 @@ +/* + * @TEST-DOC: Load intel data from a JSON file and populate via Intel::insert(). + * @TEST-REQUIRES: $SCRIPTS/have-javascript + * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace frameworks/intel/seen base/frameworks/intel base/protocols/http %INPUT + * @TEST-EXEC: zeek-cut < intel.log > intel.log.noheader + * @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff intel.log.noheader + * + * Following the intel file that we load via Intel::insert(). +@TEST-START-FILE intel.json_lines +{"indicator": "141.142.228.5", "indicator_type": "Intel::ADDR", "meta": {"source": "json1"}} +{"indicator": "bro.org", "indicator_type": "Intel::DOMAIN", "meta": {"source": "json2"}} +@TEST-END-FILE +*/ +const fs = require('fs'); + +zeek.on('zeek_init', () => { + // Hold the packet processing until we've read the intel file. + zeek.invoke('suspend_processing'); + + // This reads the full file into memory, but is still async. + // There's fs.createReadStream() for the piecewise consumption. + fs.readFile('./intel.json_lines', 'utf8', (err, data) => { + for (const l of data.split('\n')) { + if (l.length == 0) + continue; + + zeek.invoke('Intel::insert', [JSON.parse(l)]); + } + + /* Once all intel data is loaded, continue processing. */ + zeek.invoke('continue_processing'); + }); +}); diff --git a/testing/btest/javascript/suspend-continue.js b/testing/btest/javascript/suspend-continue.js new file mode 100644 index 0000000000..48443feeb4 --- /dev/null +++ b/testing/btest/javascript/suspend-continue.js @@ -0,0 +1,39 @@ +/* + * @TEST-DOC: Demo suspend and continue processing from JavaScript + * @TEST-REQUIRES: $SCRIPTS/have-javascript + * @TEST-EXEC: zeek -b -Cr $TRACES/http/get.trace base/protocols/http %INPUT > out + * @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-remove-abspath btest-diff out + */ +zeek.on('zeek_init', () => { + const nt = zeek.invoke('network_time'); + console.log(`${nt} suspend_processing`); + zeek.invoke('suspend_processing'); + const suspended_at = Date.now(); + + // Schedule a JavaScript timer (running based on wallclock) + // to continue execution in 333 msec. + setTimeout(() => { + const nt = zeek.invoke('network_time'); + const continued_at = Date.now(); + const delayed_ms = continued_at - suspended_at; + const delayed_enough = delayed_ms > 300; + + console.log(`${nt} continue_processing (delayed_enough=${delayed_enough})`); + zeek.invoke('continue_processing'); + }, 333); +}); + +zeek.on('http_request', (c, method, orig_URI, escaped_URI, version) => { + const nt = zeek.invoke('network_time'); + console.log(`${nt} http_request ${c.uid} ${method} ${orig_URI} ${version}`); +}); + +zeek.on('Pcap::file_done', (path) => { + const nt = zeek.invoke('network_time'); + console.log(`${nt} Pcap::file_done ${path}`); +}); + +zeek.on('zeek_done', () => { + const nt = zeek.invoke('network_time'); + console.log(`${nt} zeek_done`); +}); diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.cc b/testing/btest/plugins/hooks-plugin/src/Plugin.cc index 39efe3b8ce..accc871375 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.cc @@ -36,6 +36,7 @@ static std::set sanitized_functions = { // contains any of these keywords, no log message is generated. static std::set load_file_filter = { "Zeek_AF_Packet", + "Zeek_JavaScript", }; static bool skip_load_file_logging_for(const std::string& s) diff --git a/testing/scripts/have-javascript b/testing/scripts/have-javascript new file mode 100755 index 0000000000..c5277d8cd8 --- /dev/null +++ b/testing/scripts/have-javascript @@ -0,0 +1,7 @@ +#!/bin/sh + +if grep -q "ZEEK_HAVE_JAVASCRIPT:INTERNAL=yes" "${BUILD}"/CMakeCache.txt; then + exit 0 +fi + +exit 1