Merge remote-tracking branch 'origin/topic/bbannier/spicy'

* origin/topic/bbannier/spicy:
  Add NEWS item for new requirements due to default-enabled Spicy support.
  Include headers in binary dir before other headers.
  Increase memory in CI.
  Add tests for bundled Spicy infrastructure.
  Include spicy in build.
  Add `bare_mode` bif.
  Change test so included plugins can load notice framework.
  Do not log function arguments in test.

.cirrus.yml

@@ -8,15 +8,16 @@
 cpus: &CPUS 4
 btest_jobs: &BTEST_JOBS 4
 btest_retries: &BTEST_RETRIES 2
-memory: &MEMORY 8GB
+memory: &MEMORY 12GB
 
 config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install
-asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage
+asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy
-ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers
+ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy
-tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers
+tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy
 mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install
+homebrew_flex_bison_config: &HOMEBREW_FLEX_BISON_CONFIG --with-flex=/usr/local/opt/flex/bin/flex --with-bison=/usr/local/opt/bison/bin/bison
 
 resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS
@@ -216,6 +217,8 @@ macos_monterey_task:
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG
 
 macos_big_sur_task:
   macos_instance:
@@ -223,6 +226,8 @@ macos_big_sur_task:
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup
 freebsd14_task:

.github/workflows/generate-docs.yml

@@ -43,6 +43,8 @@ jobs:
             g++ \
             flex \
             bison \
+            libfl2 \
+            libfl-dev \
             libpcap-dev \
             libssl-dev \
             python3 \
@@ -60,7 +62,7 @@ jobs:
           sudo pip3 install -r doc/requirements.txt
 
       - name: Configure
-        run: ./configure --disable-broker-tests --disable-cpp-tests
+        run: ./configure --disable-broker-tests --disable-cpp-tests --disable-spicy
 
       - name: Build
         run: cd build && make -j $(nproc)

.gitmodules

@@ -58,3 +58,9 @@
 [submodule "auxil/out_ptr"]
 	path = auxil/out_ptr
 	url = https://github.com/soasis/out_ptr.git
+[submodule "auxil/spicy-plugin"]
+	path = auxil/spicy-plugin
+	url = https://github.com/zeek/spicy-plugin
+[submodule "auxil/spicy"]
+	path = auxil/spicy/spicy
+	url = https://github.com/zeek/spicy

.lgtm.yml

@@ -7,6 +7,8 @@ path_classifiers:
     - "auxil/libkqueue/"
     - "auxil/highwayhash/"
     - "auxil/rapidjson/"
+    - "auxil/spicy"
+    - "auxil/spicy-plugin"
     - "src/3rdparty/"
 
 # Filter out alerts that aren't concerning.
@@ -38,7 +40,9 @@ extraction:
 
     configure:
       command:
-        - ./configure --build-type=debug --generator=Ninja
+        # Disabling Spicy in LGTM build since it seems to lead to a LGTM
+        # performance edge case where analysis does not finish in 4h.
+        - ./configure --build-type=debug --generator=Ninja --disable-spicy
 
     index:
       build_command:

CHANGES

@@ -1,3 +1,49 @@
+5.0.0-dev.440 | 2022-05-17 08:50:15 -0700
+
+  * Add NEWS item for new requirements due to default-enabled Spicy support. (Benjamin Bannier, Corelight)
+
+  * Include headers in binary dir before other headers. (Benjamin Bannier, Corelight)
+
+    If e.g., optional dependencies are installed in the same prefix as a
+    previous Zeek installation, the build would have previously found
+    `zeek-config.h` from the external prefix instead of from the current
+    build.
+
+    With this patch we now include the current binary dir before any
+    previously configured include directories (here in particular: optional
+    dependencies). With that the correct `zeek-config.h` is found.
+
+  * Increase memory in CI. (Benjamin Bannier, Corelight)
+
+    Spicy requires substantial amounts of RAM to compile. In this patch we
+    bump the amount of RAM available in CI so a build including Spicy can be
+    performed.
+
+  * Add tests for bundled Spicy infrastructure. (Benjamin Bannier, Corelight)
+
+  * Include spicy in build. (Benjamin Bannier, Corelight)
+
+  * Add `bare_mode` bif. (Benjamin Bannier, Corelight)
+
+  * Change test so included plugins can load notice framework. (Benjamin Bannier, Corelight)
+
+  * Do not log function arguments in test. (Benjamin Bannier, Corelight)
+
+    This was excercising hooks to validate that they were called. For that
+    it uses pretty verbose logging. Since version numbers are not stable and
+    hard to canonify this script already unloads the version module (there
+    is e.g., code in `base/misc/version` which splits the version strings
+    and works on single components which might appear like pretty ordinary
+    floating point numbers in the log).
+
+    This test however worked under the assumption that nothing in bare mode
+    loads `base/misc/version` which is hard to guarantee, especially
+    considering that plugins can be embedded and might load that module
+    themself.
+
+    With this patch we now make the logging slightly less verbose so that
+    function call arguments are not logged anymore.
+
 5.0.0-dev.431 | 2022-05-16 17:00:24 -0700
 
   * Store location during Ascii input parsing for use in error messages (Tim Wojtulewicz, Corelight)

CMakeLists.txt

@@ -1,6 +1,6 @@
 # When changing the minimum version here, also adapt
 # auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt
-cmake_minimum_required(VERSION 3.5...3.18 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR)
 
 project(Zeek C CXX)
 
@@ -73,12 +73,16 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh
      "export ZEEKPATH=`${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n"
      "export ZEEK_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n"
-     "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+     "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n"
+     "export SPICY_PATH=\"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n"
+     "export HILTI_CXX_INCLUDE_DIRS=\"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh
      "setenv ZEEKPATH `${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n"
      "setenv ZEEK_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n"
-     "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+     "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n"
+     "setenv SPICY_PATH \"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n"
+     "setenv HILTI_CXX_INCLUDE_DIRS \"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
 execute_process(COMMAND grep "^#define  *BRO_PLUGIN_API_VERSION"
@@ -374,6 +378,54 @@ else ()
   set(broker_includes ${CMAKE_CURRENT_SOURCE_DIR}/auxil/broker/include ${CMAKE_CURRENT_BINARY_DIR}/auxil/broker/include)
 endif ()
 
+if ( NOT DISABLE_SPICY )
+    if ( NOT Spicy_ROOT )
+        add_subdirectory(auxil/spicy)
+
+        # Set variables used by the spicy-plugin build since we are building Spicy
+        # as part of Zeek so spicy-plugin cannot use `spicy-config` at configure
+        # time to set these.
+        set(SPICY_CONFIG "<builtin>")
+        set(HAVE_SPICY "YES")
+        set(SPICY_HAVE_TOOLCHAIN "YES")
+        set(SPICY_INCLUDE_DIRS_RUNTIME
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/runtime/include
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/runtime/include
+            ${PROJECT_BINARY_DIR}/auxil/spicy/spicy/include)
+        set(SPICY_INCLUDE_DIRS_TOOLCHAIN
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/toolchain/include
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/toolchain/include)
+        set(SPICY_LIBRARY spicy)
+
+        # Needed only for logging from CMake configure phase.
+        get_directory_property(
+            SPICY_VERSION
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION SPICY_VERSION)
+        get_directory_property(
+            SPICY_VERSION_NUMBER
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION SPICY_VERSION_NUMBER)
+        get_directory_property(
+            SPICY_PREFIX
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION CMAKE_INSTALL_PREFIX)
+        get_directory_property(
+            SPICY_BUILD_MODE
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION CMAKE_BUILD_TYPE)
+        set(SPICYC "<bundled>")
+    endif ()
+
+    if ( NOT SPICY_PLUGIN_PATH )
+        set(SPICY_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/spicy-plugin)
+    endif ()
+
+    # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/auxil/spicy-plugin)
+    # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin)
+    string(APPEND ZEEK_INCLUDE_PLUGINS ";${SPICY_PLUGIN_PATH}")
+endif ()
+
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
@@ -502,7 +554,7 @@ endif ()
 string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER)
 configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.h.in
                ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h)
-include_directories(${CMAKE_CURRENT_BINARY_DIR})
+include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR})
 install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h DESTINATION include/zeek)
 execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
                 "."
@@ -616,6 +668,53 @@ CheckOptionalBuildSources(auxil/zeek-aux  Zeek-Aux  INSTALL_AUX_TOOLS)
 CheckOptionalBuildSources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER)
 CheckOptionalBuildSources(auxil/zeek-client ZeekClient INSTALL_ZEEK_CLIENT)
 
+if ( NOT DISABLE_SPICY )
+    if ( NOT Spicy_ROOT )
+        list(APPEND _spicy_rt_libs spicy-rt hilti-rt)
+
+        # Use the debug runtime libraries if we are building a debug Zeek.
+        if ( SPICY_BUILD_MODE STREQUAL "Debug" )
+            list(TRANSFORM _spicy_rt_libs APPEND "-debug")
+        endif ()
+
+        # If we build spicy-plugin into Zeek we also need to build the Spicy
+        # runtime dependencies into Zeek. Since no matter how Spicy itself was
+        # linked this is always a static library, link the object files so we
+        # get all symbols and can can resolve all potential dependencies of
+        # HLTO files at runtime.
+        #
+        # TODO(bbannier): Conceptually cleaner would be linking the runtime
+        # objects into spicy-plugin and then inherit that up to zeek, but it
+        # seems this does not work reliable (due incomplete support for object
+        # libraries in CMake?).
+        list(TRANSFORM _spicy_rt_libs APPEND "-objects")
+        target_link_libraries(zeek ${_spicy_rt_libs})
+
+        # Make sure we build targets of spicy-plugin after the `spicy` target.
+        add_dependencies(plugin-Zeek-Spicy spicy)
+        add_dependencies(spicyz spicy)
+
+        # Spicy JIT relies on the path of the current executable to figure out
+        # whether it is run from a build or an install tree. This assumption
+        # gets broken for e.g., `spicyz` when running from the build tree
+        # (probably fixable), and also when JIT'ing directly from a `zeek`
+        # invocation (much harder to fix). Instead we generate shell
+        # definitions to support running and using Spicy or spicy-plugin
+        # functionality in the build tree, including JIT'ing directly from
+        # Zeek.
+        configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/spicy-path.in ${CMAKE_BINARY_DIR}/spicy-path @ONLY)
+        configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/hilti-cxx-include-dirs.in ${CMAKE_BINARY_DIR}/hilti-cxx-include-dirs @ONLY)
+
+        # Also install spicy-plugin's CMake files into Zeek's global `cmake/` folder.
+        #
+        # NOTE: We do not install spicy-plugin's `FindZeek.cmake` since another
+        # version of this file is already provided by Zeek.
+        install(FILES auxil/spicy-plugin/cmake/ZeekSpicyAnalyzerSupport.cmake
+                      auxil/spicy-plugin/cmake/FindSpicy.cmake
+                DESTINATION share/zeek/cmake)
+    endif ()
+endif ()
+
 ########################################################################
 ## Packaging Setup
 

NEWS

@@ -9,6 +9,12 @@ Zeek 5.0.0
 Breaking Changes
 ----------------
 
+- Zeek now requires at least CMake version 3.15.0.
+
+- If Zeek is configured with support for included Spicy (the default) we now
+  require at least Flex version 2.6 and its development headers, at least Bison
+  version 3.3, and GCC version 8.3 or Clang version 9.0 or higher.
+
 New Functionality
 -----------------
 
@@ -25,6 +31,10 @@ New Functionality
   and (8) how much of that memory was due to its children.  Note that profiling
   is expensive and may not be suitable for execution on live traffic.
 
+- Zeek now includes support for building Spicy and spicy-plugin as part of
+  Zeek. This feature is enabled by default, and can be turned off by passing
+  the ``--disable-spicy`` flag to ``./configure``.
+
 - Zeek now supports generation and replay of event traces via the new
   ``--event-trace`` / ``-E`` command-line options. For details, see:
   https://docs.zeek.org/en/master/quickstart.html#tracing-events

VERSION

@@ -1 +1 @@
-5.0.0-dev.431
+5.0.0-dev.440

auxil/spicy-plugin

@@ -0,0 +1 @@
+Subproject commit 0435d34acf92ff07558d72524fb2669eca78b1c9

auxil/spicy/CMakeLists.txt

@@ -0,0 +1,38 @@
+# Spicy needs the full prefix for Flex and Bison while Zeek captures only the
+# paths to the executables. Derive the prefixes from the binary paths under the
+# assumption that their bindir is under their prefix (which also implies that
+# one such prefix even exists).
+if ( NOT FLEX_EXECUTABLE )
+   find_package(FLEX REQUIRED)
+endif ()
+get_filename_component(dir ${FLEX_EXECUTABLE} DIRECTORY ABSOLUTE)
+set(FLEX_ROOT ${dir}/..)
+
+if ( NOT BISON_EXECUTABLE )
+   find_package(BISON REQUIRED)
+endif ()
+get_filename_component(dir ${BISON_EXECUTABLE} DIRECTORY ABSOLUTE)
+set(BISON_ROOT ${dir}/..)
+
+# We currently only support building against statically linked Spicy libraries.
+# This is due to global state held in the Spicy and HILTI runtime libraries. If
+# we would like Spicy dynamically this global state would end up the the final
+# binary once via the static link of the runtime libraries into Zeek (needed to
+# make sure we can e.g., load arbitrary HLTO files), and once as a link
+# dependency of spicy-plugin. This can lead to e.g. consistency issues during
+# execution or the problem during teardown where the same global might be
+# cleaned up multiple times.
+set(BUILD_SHARED_LIBS OFF)
+
+# Spicy uses slightly less strict warnings than Zeek proper. Mute a few warnings for Spicy.
+# NOTE: Compiler flags are inherited down the directory tree, so in order to
+# set these flags we do need a customizable subdirectory above the Spicy
+# sources.
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-braces")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla")
+
+# The script generating precompiled headers for Spicy expects a different build
+# system layout than provided for a bundled Spicy, disable it.
+set(HILTI_DEV_PRECOMPILE_HEADERS OFF)
+
+add_subdirectory(spicy)

auxil/spicy/hilti-cxx-include-dirs.in

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Computes CXX flags for Spicy compilation in the build tree. The value
+# computed by this script can be injected into Spicy JIT via the environment
+# variable `HILTI_CXX_INCLUDE_DIRS`.
+
+# Paths to support compiling Spicy parsers in the build tree.
+PATHS=@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/runtime/include
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/runtime/include
+PATHS=$PATHS:@CMAKE_BINARY_DIR@/auxil/spicy/spicy/include
+
+# Paths to support compiling against a build tree Zeek.
+PATHS=$PATHS:@CMAKE_BINARY_DIR@
+PATHS=$PATHS:@CMAKE_BINARY_DIR@/src
+PATHS=$PATHS:@CMAKE_BINARY_DIR@/src/include
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/src
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/broker/include/
+
+echo $PATHS

auxil/spicy/spicy

@@ -0,0 +1 @@
+Subproject commit ef859d07a137bae77ffd8f04908dbdbb5d8649ef

auxil/spicy/spicy-path.in

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Computes a value for the environment variable `SPICY_PATH` for execution of
+# Spicy tooling in the build tree.
+
+PATHS=.
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/lib
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/lib
+
+echo $PATHS

ci/alpine/Dockerfile

@@ -6,7 +6,7 @@ RUN apk add --no-cache \
   g++ \
   python3 \
   python3-dev \
-  flex \
+  flex-dev \
   bison \
   libpcap-dev \
   openssl-dev \

ci/centos-7/Dockerfile

@@ -1,5 +1,11 @@
 FROM centos:7
 
+ENV FLEX_VERSION=2.6.4
+ENV FLEX_DIR=/opt/flex
+
+# Disabled lookup of fastest mirror since the list seems to be outdated and no valid mirror can be detected.
+RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+
 # The version of git in the standard repos is 1.8 and CI needs 2.3+
 # for the use of GIT_SSH_COMMAND when cloning private repos.
 RUN yum -y install \
@@ -17,13 +23,13 @@ RUN yum -y install \
   && yum clean all && rm -rf /var/cache/yum
 
 RUN yum -y install \
-    devtoolset-7 \
+    devtoolset-8 \
   && yum clean all && rm -rf /var/cache/yum
 
 RUN yum -y install \
     cmake3 \
+    curl \
     make \
-    flex \
     bison \
     swig \
     openssl \
@@ -38,10 +44,18 @@ RUN yum -y install \
     which \
   && yum clean all && rm -rf /var/cache/yum
 
+# Install a recent flex for Spicy.
+RUN curl -sSL "https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz" | tar xzf - -C /tmp \
+ && (cd /tmp/flex-${FLEX_VERSION} \
+     && ./configure --prefix=${FLEX_DIR} \
+     && make -j`nproc` install) \
+ && rm -rf /tmp/flex-${FLEX_VERSION}
+
 RUN pip3 install junit2html
 
 RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \
-    echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env
+    echo 'source /opt/rh/devtoolset-8/enable' >> /usr/bin/zeek-ci-env && \
+    echo 'export PATH=${PATH}:${FLEX_DIR}/bin' >> /usr/bin/zeek-ci-env
 
 ENV BASH_ENV="/usr/bin/zeek-ci-env" \
     ENV="/usr/bin/zeek-ci-env" \

ci/debian-10/Dockerfile

@@ -2,9 +2,11 @@ FROM debian:10
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
     make \
     gcc \
     g++ \
@@ -23,6 +25,11 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     xz-utils \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  # Install a recent CMake to build Spicy.
+  && mkdir -p "${CMAKE_DIR}" \
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 \
 
-RUN pip3 install junit2html
+  && pip3 install junit2html
+
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"

ci/debian-9-32bit/Dockerfile

@@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
 ENV DOCKERFILE_VERSION 221001
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
+    clang-11 \
     make \
-    gcc \
-    g++ \
     flex \
     bison \
+    libc++-11-dev \
+    libc++abi-11-dev \
     libpcap-dev \
     libssl-dev \
     python3 \
@@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     xz-utils \
-    clang-7 \
+  && rm -rf /var/lib/apt/lists/* \
-    libc++-7-dev \
+  # Recent CMake.
-    libc++abi-7-dev \
+  && mkdir -p "${CMAKE_DIR}" \
-  && rm -rf /var/lib/apt/lists/*
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1
 
-RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100
+RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-11 100
-RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100
+RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-11 100
 
 RUN pip3 install junit2html
 

ci/debian-9/Dockerfile

@@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
 ENV DOCKERFILE_VERSION 221001
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
+    clang-11 \
     make \
-    gcc \
-    g++ \
     flex \
     bison \
+    libc++-11-dev \
+    libc++abi-11-dev \
     libpcap-dev \
     libssl-dev \
     python3 \
@@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     xz-utils \
-    clang-7 \
+  && rm -rf /var/lib/apt/lists/* \
-    libc++-7-dev \
+  # Recent CMake.
-    libc++abi-7-dev \
+  && mkdir -p "${CMAKE_DIR}" \
-  && rm -rf /var/lib/apt/lists/*
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1
 
 RUN pip3 install junit2html
 
-ENV CC=/usr/bin/clang-7
+ENV CC=/usr/bin/clang-11
-ENV CXX=/usr/bin/clang++-7
+ENV CXX=/usr/bin/clang++-11
 ENV CXXFLAGS=-stdlib=libc++

ci/freebsd/prepare.sh

@@ -6,9 +6,13 @@ set -e
 set -x
 
 env ASSUME_ALWAYS_YES=YES pkg bootstrap
-pkg install -y bash git cmake swig bison python3 base64
+pkg install -y bash git cmake swig bison python3 base64 flex
 pkg upgrade -y curl
 pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")')
 pkg install -y $pyver-sqlite3 $pyver-pip
 
 python -m pip install junit2html
+
+# Spicy detects whether it is run from build directory via `/proc`.
+echo "proc /proc procfs rw,noauto 0 0" >>/etc/fstab
+mount /proc

ci/macos/prepare.sh

@@ -7,4 +7,4 @@ set -x
 
 brew update
 brew upgrade cmake openssl@1.1
-brew install swig bison
+brew install swig bison flex

ci/opensuse-leap-15.3/Dockerfile

@@ -1,10 +1,12 @@
 FROM opensuse/leap:15.3
 
-RUN zypper in -y \
+RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.2:Update/standard/openSUSE:Leap:15.2:Update.repo \
+ && zypper refresh \
+ && zypper in -y \
   cmake \
   make \
-  gcc \
+  gcc9 \
-  gcc-c++ \
+  gcc9-c++ \
   python3 \
   python3-devel \
   flex \
@@ -21,5 +23,7 @@ RUN zypper in -y \
   tar \
   && rm -rf /var/cache/zypp
 
-
 RUN pip3 install junit2html
+
+ENV CXX g++-9
+ENV CC gcc-9

ci/ubuntu-18.04/Dockerfile

@@ -2,12 +2,14 @@ FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
+    clang-10 \
     make \
-    gcc \
-    g++ \
     flex \
     bison \
     libpcap-dev \
@@ -27,7 +29,13 @@ RUN apt-get update && apt-get -y install \
     ruby \
     bc \
     lcov \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  # Recent CMake.
+  && mkdir -p "${CMAKE_DIR}" \
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1
+
+ENV CC=/usr/bin/clang-10
+ENV CXX=/usr/bin/clang++-10
 
 RUN pip3 install junit2html
 RUN gem install coveralls-lcov

configure

@@ -71,6 +71,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-btest-pcaps  don't install Zeek's BTest input pcaps
     --disable-cpp-tests    don't build Zeek's C++ unit tests
     --disable-python       don't try to build python bindings for Broker
+    --disable-spicy        don't include Spicy
     --disable-zeekctl      don't install ZeekControl
     --disable-zkg          don't install zkg
 
@@ -101,6 +102,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-perftools=PATH  path to Google Perftools install root
     --with-python-inc=PATH path to Python headers
     --with-python-lib=PATH path to libpython
+    --with-spicy=PATH      path to Spicy install root
+    --with-spicy-plugin=PATH path to Spicy plugin source tree
     --with-swig=PATH       path to SWIG executable
 
   Packaging Options (for developers):
@@ -313,6 +316,9 @@ while [ $# -ne 0 ]; do
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true
             ;;
+        --disable-spicy)
+            append_cache_entry DISABLE_SPICY BOOL true
+            ;;
         --disable-zeekctl)
             append_cache_entry INSTALL_ZEEKCTL BOOL false
             ;;
@@ -375,6 +381,12 @@ while [ $# -ne 0 ]; do
         --with-python-lib=*)
             append_cache_entry PYTHON_LIBRARY PATH $optarg
             ;;
+        --with-spicy=*)
+            append_cache_entry SPICY_ROOT PATH $optarg
+            ;;
+        --with-spicy-plugin=*)
+            append_cache_entry SPICY_PLUGIN_PATH PATH $optarg
+            ;;
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE PATH $optarg
             ;;

docker/Dockerfile

@@ -12,6 +12,8 @@ RUN apt-get -q update \
      flex \
      g++ \
      gcc \
+     libfl2 \
+     libfl-dev \
      libmaxminddb-dev \
      libpcap-dev \
      libssl-dev \

src/RunState.cc

@@ -63,6 +63,7 @@ double current_wallclock = 0.0;
 double current_pseudo = 0.0;
 bool zeek_init_done = false;
 bool time_updated = false;
+bool bare_mode = false;
 
 RETSIGTYPE watchdog(int /* signo */)
 	{

src/RunState.h

@@ -65,6 +65,8 @@ extern double current_pseudo;
 
 extern bool zeek_init_done;
 
+extern bool bare_mode; // True if Zeek was started in bare mode.
+
 	} // namespace detail
 
 // Functions to temporarily suspend processing of live input (network packets

src/zeek-setup.cc

@@ -516,6 +516,8 @@ SetupResult setup(int argc, char** argv, Options* zopts)
 
 	auto options = zopts ? *zopts : parse_cmdline(argc, argv);
 
+	run_state::detail::bare_mode = options.bare_mode;
+
 	// Set up the global that facilitates access to analysis/optimization
 	// options from deep within some modules.
 	analysis_options = options.analysis_options;

src/zeek.bif

@@ -1748,6 +1748,20 @@ function zeek_version%(%): string
 	return zeek::make_intrusive<zeek::StringVal>(zeek::zeek_version());
 	%}
 
+%%{
+namespace zeek::run_state::detail {
+extern bool bare_mode;
+} // namespace zeek::run_state::detail
+%%}
+
+## Returns whether Zeek was started in bare mode.
+##
+## Returns: True if Zeek was started in bare mode, false otherwise.
+function bare_mode%(%): bool
+	%{
+	return zeek::val_mgr->Bool(zeek::run_state::detail::bare_mode);
+	%}
+
 ## Converts a record type name to a vector of strings, where each element is
 ## the name of a record field. Nested records are flattened.
 ##

testing/btest/Baseline/bifs.bare_mode/output

@@ -0,0 +1,3 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+F
+T

testing/btest/Baseline/broker.remote_event/recv.recv.out

@@ -10,4 +10,4 @@ is_remote should be T, and is, T
 receiver got ping: my-message, 4
 is_remote should be T, and is, T
 receiver got ping: my-message, 5
-[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/broker.remote_event_any/recv.recv.out

@@ -10,4 +10,4 @@ is_remote should be T, and is, T
 receiver got ping: my-message, 4
 is_remote should be T, and is, T
 receiver got ping: my-message, 5
-[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out

@@ -5,4 +5,4 @@ receiver got ping: my-message, 2
 receiver got ping: my-message, 3
 receiver got ping: my-message, 4
 receiver got ping: my-message, 5
-[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/core.check-unused-event-handlers/.stderr

@@ -7,4 +7,5 @@ warning in <params>, line 1: event handler never invoked: SupervisorControl::des
 warning in <params>, line 1: event handler never invoked: SupervisorControl::restart_request
 warning in <params>, line 1: event handler never invoked: SupervisorControl::status_request
 warning in <params>, line 1: event handler never invoked: SupervisorControl::stop_request
+warning in <params>, line 1: event handler never invoked: spicy_analyzer_for_mime_type
 warning in <params>, line 1: event handler never invoked: this_is_never_used

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -243,8 +243,18 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek
     build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
     build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek
 build/scripts/builtin-plugins/__preload__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek
 build/scripts/builtin-plugins/__load__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek
+    build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek
+      scripts/base/misc/version.zeek
+        scripts/base/frameworks/reporter/__load__.zeek
+          scripts/base/frameworks/reporter/main.zeek
+        scripts/base/utils/strings.zeek
 scripts/policy/misc/loaded-scripts.zeek
   scripts/base/utils/paths.zeek
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -243,6 +243,9 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek
     build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
     build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek
 scripts/base/init-default.zeek
   scripts/base/utils/active-http.zeek
     scripts/base/utils/exec.zeek
@@ -443,6 +446,10 @@ scripts/base/init-default.zeek
   scripts/base/misc/find-filtered-trace.zeek
   scripts/base/misc/version.zeek
 build/scripts/builtin-plugins/__preload__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek
 build/scripts/builtin-plugins/__load__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek
+    build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek
+    build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/default.zeek
 scripts/policy/misc/loaded-scripts.zeek
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/plugins.hooks/output

@@ -673,13 +673,17 @@
 0.000000   MetaHookPost  CallFunction(SumStats::register_observe_plugins, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Supervisor::__is_supervisor, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(Supervisor::is_supervisor, <frame>, ()) -> <no result>
+0.000000   MetaHookPost  CallFunction(Version::parse, ..., ...) -> <no result>
 0.000000   MetaHookPost  CallFunction(__init_primary_bifs, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(__init_secondary_bifs, <null>, ()) -> <no result>
+0.000000   MetaHookPost  CallFunction(bare_mode, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(current_time, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(filter_change_tracking, <null>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(getenv, <null>, (CLUSTER_NODE)) -> <no result>
 0.000000   MetaHookPost  CallFunction(getenv, <null>, (ZEEK_DEFAULT_LISTEN_ADDRESS)) -> <no result>
 0.000000   MetaHookPost  CallFunction(global_options, <frame>, ()) -> <no result>
+0.000000   MetaHookPost  CallFunction(gsub, ..., ...) -> <no result>
+0.000000   MetaHookPost  CallFunction(lstrip, ..., ...) -> <no result>
 0.000000   MetaHookPost  CallFunction(network_time, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(port_to_count, <frame>, (2123/udp)) -> <no result>
 0.000000   MetaHookPost  CallFunction(port_to_count, <frame>, (2152/udp)) -> <no result>
@@ -690,12 +694,15 @@
 0.000000   MetaHookPost  CallFunction(reading_live_traffic, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(reading_traces, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(set_to_regex, <frame>, ({}, (^\.?|\.)(~~)$)) -> <no result>
+0.000000   MetaHookPost  CallFunction(split_string1, ..., ...) -> <no result>
 0.000000   MetaHookPost  CallFunction(string_to_pattern, <frame>, ((^\.?|\.)()$, F)) -> <no result>
 0.000000   MetaHookPost  CallFunction(sub, <frame>, ((^\.?|\.)(~~)$, <...>/, )) -> <no result>
+0.000000   MetaHookPost  CallFunction(to_count, ..., ...) -> <no result>
 0.000000   MetaHookPost  CallFunction(x509_set_certificate_cache, <frame>, ({})) -> <no result>
 0.000000   MetaHookPost  CallFunction(x509_set_certificate_cache_hit_callback, <frame>, (X509::x509_certificate_cache_replay{ <init> X509::i{ if (X509::f$info?$x509) return event x509_certificate(X509::f, X509::e$handle, X509::e$certificate)for ([X509::i] in X509::e$extensions_cache) { X509::ext = X509::e$extensions_cache[X509::i]if (X509::ext is X509::Extension) event x509_extension(X509::f, (X509::ext as X509::Extension))elseif (X509::ext is X509::BasicConstraints) event x509_ext_basic_constraints(X509::f, (X509::ext as X509::BasicConstraints))elseif (X509::ext is X509::SubjectAlternativeName) event x509_ext_subject_alternative_name(X509::f, (X509::ext as X509::SubjectAlternativeName))elseif (X509::ext is X509::SctInfo) { X509::s = (X509::ext as X509::SctInfo)event x509_ocsp_ext_signed_certificate_timestamp(X509::f, X509::s$version, X509::s$logid, X509::s$timestamp, X509::s$hash_alg, X509::s$sig_alg, X509::s$signature)}elseReporter::error(fmt(Encountered unknown extension while replaying certificate with fuid %s, X509::f$id))}}})) -> <no result>
 0.000000   MetaHookPost  CallFunction(zeek_args, <frame>, ()) -> <no result>
 0.000000   MetaHookPost  CallFunction(zeek_init, <null>, ()) -> <no result>
+0.000000   MetaHookPost  CallFunction(zeek_version, <null>, ()) -> <no result>
 0.000000   MetaHookPost  DrainEvents() -> <void>
 0.000000   MetaHookPost  LoadFile(0, ../main, <...>/main.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ../plugin, <...>/plugin.zeek) -> -1
@@ -808,6 +815,9 @@
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> -1
@@ -924,8 +934,12 @@
 0.000000   MetaHookPost  LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base/bif, <...>/bif) -> -1
-0.000000   MetaHookPost  LoadFile(0, base/init-default, <...>/init-default.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base/packet-protocols, <...>/packet-protocols) -> -1
 0.000000   MetaHookPost  LoadFile(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) -> -1
@@ -1185,6 +1199,9 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, <no content>)
@@ -1301,8 +1318,12 @@
 0.000000   MetaHookPost  LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base/bif, <...>/bif) -> (-1, <no content>)
-0.000000   MetaHookPost  LoadFileExtended(0, base/init-default, <...>/init-default.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base/packet-protocols, <...>/packet-protocols) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) -> (-1, <no content>)
@@ -2130,13 +2151,17 @@
 0.000000   MetaHookPre   CallFunction(SumStats::register_observe_plugins, <frame>, ())
 0.000000   MetaHookPre   CallFunction(Supervisor::__is_supervisor, <frame>, ())
 0.000000   MetaHookPre   CallFunction(Supervisor::is_supervisor, <frame>, ())
+0.000000   MetaHookPre   CallFunction(Version::parse, ..., ...)
 0.000000   MetaHookPre   CallFunction(__init_primary_bifs, <null>, ())
 0.000000   MetaHookPre   CallFunction(__init_secondary_bifs, <null>, ())
+0.000000   MetaHookPre   CallFunction(bare_mode, <null>, ())
 0.000000   MetaHookPre   CallFunction(current_time, <frame>, ())
 0.000000   MetaHookPre   CallFunction(filter_change_tracking, <null>, ())
 0.000000   MetaHookPre   CallFunction(getenv, <null>, (CLUSTER_NODE))
 0.000000   MetaHookPre   CallFunction(getenv, <null>, (ZEEK_DEFAULT_LISTEN_ADDRESS))
 0.000000   MetaHookPre   CallFunction(global_options, <frame>, ())
+0.000000   MetaHookPre   CallFunction(gsub, ..., ...)
+0.000000   MetaHookPre   CallFunction(lstrip, ..., ...)
 0.000000   MetaHookPre   CallFunction(network_time, <frame>, ())
 0.000000   MetaHookPre   CallFunction(port_to_count, <frame>, (2123/udp))
 0.000000   MetaHookPre   CallFunction(port_to_count, <frame>, (2152/udp))
@@ -2147,12 +2172,15 @@
 0.000000   MetaHookPre   CallFunction(reading_live_traffic, <frame>, ())
 0.000000   MetaHookPre   CallFunction(reading_traces, <frame>, ())
 0.000000   MetaHookPre   CallFunction(set_to_regex, <frame>, ({}, (^\.?|\.)(~~)$))
+0.000000   MetaHookPre   CallFunction(split_string1, ..., ...)
 0.000000   MetaHookPre   CallFunction(string_to_pattern, <frame>, ((^\.?|\.)()$, F))
 0.000000   MetaHookPre   CallFunction(sub, <frame>, ((^\.?|\.)(~~)$, <...>/, ))
+0.000000   MetaHookPre   CallFunction(to_count, ..., ...)
 0.000000   MetaHookPre   CallFunction(x509_set_certificate_cache, <frame>, ({}))
 0.000000   MetaHookPre   CallFunction(x509_set_certificate_cache_hit_callback, <frame>, (X509::x509_certificate_cache_replay{ <init> X509::i{ if (X509::f$info?$x509) return event x509_certificate(X509::f, X509::e$handle, X509::e$certificate)for ([X509::i] in X509::e$extensions_cache) { X509::ext = X509::e$extensions_cache[X509::i]if (X509::ext is X509::Extension) event x509_extension(X509::f, (X509::ext as X509::Extension))elseif (X509::ext is X509::BasicConstraints) event x509_ext_basic_constraints(X509::f, (X509::ext as X509::BasicConstraints))elseif (X509::ext is X509::SubjectAlternativeName) event x509_ext_subject_alternative_name(X509::f, (X509::ext as X509::SubjectAlternativeName))elseif (X509::ext is X509::SctInfo) { X509::s = (X509::ext as X509::SctInfo)event x509_ocsp_ext_signed_certificate_timestamp(X509::f, X509::s$version, X509::s$logid, X509::s$timestamp, X509::s$hash_alg, X509::s$sig_alg, X509::s$signature)}elseReporter::error(fmt(Encountered unknown extension while replaying certificate with fuid %s, X509::f$id))}}}))
 0.000000   MetaHookPre   CallFunction(zeek_args, <frame>, ())
 0.000000   MetaHookPre   CallFunction(zeek_init, <null>, ())
+0.000000   MetaHookPre   CallFunction(zeek_version, <null>, ())
 0.000000   MetaHookPre   DrainEvents()
 0.000000   MetaHookPre   LoadFile(0, ../main, <...>/main.zeek)
 0.000000   MetaHookPre   LoadFile(0, ../plugin, <...>/plugin.zeek)
@@ -2265,6 +2293,9 @@
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek)
@@ -2381,8 +2412,12 @@
 0.000000   MetaHookPre   LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek)
 0.000000   MetaHookPre   LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek)
 0.000000   MetaHookPre   LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek)
+0.000000   MetaHookPre   LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek)
+0.000000   MetaHookPre   LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek)
+0.000000   MetaHookPre   LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek)
+0.000000   MetaHookPre   LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek)
 0.000000   MetaHookPre   LoadFile(0, base/bif, <...>/bif)
-0.000000   MetaHookPre   LoadFile(0, base/init-default, <...>/init-default.zeek)
+0.000000   MetaHookPre   LoadFile(0, base/init-default.zeek, <...>/init-default.zeek)
 0.000000   MetaHookPre   LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek)
 0.000000   MetaHookPre   LoadFile(0, base/packet-protocols, <...>/packet-protocols)
 0.000000   MetaHookPre   LoadFile(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek)
@@ -2642,6 +2677,9 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek)
@@ -2758,8 +2796,12 @@
 0.000000   MetaHookPre   LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base/bif, <...>/bif)
-0.000000   MetaHookPre   LoadFileExtended(0, base/init-default, <...>/init-default.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base/packet-protocols, <...>/packet-protocols)
 0.000000   MetaHookPre   LoadFileExtended(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek)
@@ -3586,13 +3628,17 @@
 0.000000 | HookCallFunction SumStats::register_observe_plugins()
 0.000000 | HookCallFunction Supervisor::__is_supervisor()
 0.000000 | HookCallFunction Supervisor::is_supervisor()
+0.000000 | HookCallFunction Version::parse(...)
 0.000000 | HookCallFunction __init_primary_bifs()
 0.000000 | HookCallFunction __init_secondary_bifs()
+0.000000 | HookCallFunction bare_mode()
 0.000000 | HookCallFunction current_time()
 0.000000 | HookCallFunction filter_change_tracking()
 0.000000 | HookCallFunction getenv(CLUSTER_NODE)
 0.000000 | HookCallFunction getenv(ZEEK_DEFAULT_LISTEN_ADDRESS)
 0.000000 | HookCallFunction global_options()
+0.000000 | HookCallFunction gsub(...)
+0.000000 | HookCallFunction lstrip(...)
 0.000000 | HookCallFunction network_time()
 0.000000 | HookCallFunction port_to_count(2123/udp)
 0.000000 | HookCallFunction port_to_count(2152/udp)
@@ -3603,12 +3649,15 @@
 0.000000 | HookCallFunction reading_live_traffic()
 0.000000 | HookCallFunction reading_traces()
 0.000000 | HookCallFunction set_to_regex({}, (^\.?|\.)(~~)$)
+0.000000 | HookCallFunction split_string1(...)
 0.000000 | HookCallFunction string_to_pattern((^\.?|\.)()$, F)
 0.000000 | HookCallFunction sub((^\.?|\.)(~~)$, <...>/, )
+0.000000 | HookCallFunction to_count(...)
 0.000000 | HookCallFunction x509_set_certificate_cache({})
 0.000000 | HookCallFunction x509_set_certificate_cache_hit_callback(X509::x509_certificate_cache_replay{ <init> X509::i{ if (X509::f$info?$x509) return event x509_certificate(X509::f, X509::e$handle, X509::e$certificate)for ([X509::i] in X509::e$extensions_cache) { X509::ext = X509::e$extensions_cache[X509::i]if (X509::ext is X509::Extension) event x509_extension(X509::f, (X509::ext as X509::Extension))elseif (X509::ext is X509::BasicConstraints) event x509_ext_basic_constraints(X509::f, (X509::ext as X509::BasicConstraints))elseif (X509::ext is X509::SubjectAlternativeName) event x509_ext_subject_alternative_name(X509::f, (X509::ext as X509::SubjectAlternativeName))elseif (X509::ext is X509::SctInfo) { X509::s = (X509::ext as X509::SctInfo)event x509_ocsp_ext_signed_certificate_timestamp(X509::f, X509::s$version, X509::s$logid, X509::s$timestamp, X509::s$hash_alg, X509::s$sig_alg, X509::s$signature)}elseReporter::error(fmt(Encountered unknown extension while replaying certificate with fuid %s, X509::f$id))}}})
 0.000000 | HookCallFunction zeek_args()
 0.000000 | HookCallFunction zeek_init()
+0.000000 | HookCallFunction zeek_version()
 0.000000 | HookDrainEvents
 0.000000 | HookLoadFile  ../main <...>/main.zeek
 0.000000 | HookLoadFile  ../plugin <...>/plugin.zeek
@@ -3721,6 +3770,9 @@
 0.000000 | HookLoadFile  ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek
@@ -3849,8 +3901,12 @@
 0.000000 | HookLoadFile  <...>/__load__.zeek <...>/__load__.zeek
 0.000000 | HookLoadFile  <...>/__preload__.zeek <...>/__preload__.zeek
 0.000000 | HookLoadFile  <...>/hooks.zeek <...>/hooks.zeek
+0.000000 | HookLoadFile  Zeek<...>/bare.zeek <...>/bare.zeek
+0.000000 | HookLoadFile  Zeek<...>/default.zeek <...>/default.zeek
+0.000000 | HookLoadFile  Zeek_Spicy/__load__.zeek <...>/__load__.zeek
+0.000000 | HookLoadFile  Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek
 0.000000 | HookLoadFile  base/bif <...>/bif
-0.000000 | HookLoadFile  base/init-default <...>/init-default.zeek
+0.000000 | HookLoadFile  base/init-default.zeek <...>/init-default.zeek
 0.000000 | HookLoadFile  base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek
 0.000000 | HookLoadFile  base/packet-protocols <...>/packet-protocols
 0.000000 | HookLoadFile  base<...>/CPP-load.bif <...>/CPP-load.bif.zeek
@@ -4098,6 +4154,9 @@
 0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek
@@ -4226,8 +4285,12 @@
 0.000000 | HookLoadFileExtended <...>/__load__.zeek <...>/__load__.zeek
 0.000000 | HookLoadFileExtended <...>/__preload__.zeek <...>/__preload__.zeek
 0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek
+0.000000 | HookLoadFileExtended Zeek<...>/bare.zeek <...>/bare.zeek
+0.000000 | HookLoadFileExtended Zeek<...>/default.zeek <...>/default.zeek
+0.000000 | HookLoadFileExtended Zeek_Spicy/__load__.zeek <...>/__load__.zeek
+0.000000 | HookLoadFileExtended Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek
 0.000000 | HookLoadFileExtended base/bif <...>/bif
-0.000000 | HookLoadFileExtended base/init-default <...>/init-default.zeek
+0.000000 | HookLoadFileExtended base/init-default.zeek <...>/init-default.zeek
 0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek
 0.000000 | HookLoadFileExtended base/packet-protocols <...>/packet-protocols
 0.000000 | HookLoadFileExtended base<...>/CPP-load.bif <...>/CPP-load.bif.zeek

testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr

@@ -1,2 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension
+XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension

testing/btest/Baseline/spicy.spicy-dump/output

@@ -0,0 +1,10 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+test::X {
+  xs: [
+    11
+    22
+    33
+    44
+    55
+  ]
+}

testing/btest/Baseline/spicy.spicyz-aot/output

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+8, [$data=[b"POST /post HTTP/1.1", b"User-Agent: curl/7.29.0", b"Host: httpbin.org", b"Accept: */*", b"Content-Length: 11", b"Content-Type: application/x-www-form-urlencoded", b"", b"hello world"]]
+Event:, [POST /post HTTP/1.1, User-Agent: curl/7.29.0, Host: httpbin.org, Accept: */*, Content-Length: 11, Content-Type: application/x-www-form-urlencoded, , hello world]
+8, [$data=[b"HTTP/1.1 200 OK", b"Server: gunicorn/0.16.1", b"Date: Tue, 19 Mar 2013 16:05:11 GMT", b"Content-Type: application/json", b"Content-Length: 366", b"Connection: close", b"", b"{\x0a  \"origin\": \"10.142.133.148\",\x0a  \"files\": {},\x0a  \"form\": null,\x0a  \"url\": \"http://httpbin.org/post\",\x0a  \"args\": {},\x0a  \"headers\": {\x0a    \"Content-Length\": \"11\",\x0a    \"Connection\": \"close\",\x0a    \"Accept\": \"*/*\",\x0a    \"User-Agent\": \"curl/7.29.0\",\x0a    \"Host\": \"httpbin.org\",\x0a    \"Content-Type\": \"application/x-www-form-urlencoded\"\x0a  },\x0a  \"json\": null,\x0a  \"data\": \"hello world\"\x0a}"]]
+Event:, [HTTP/1.1 200 OK, Server: gunicorn/0.16.1, Date: Tue, 19 Mar 2013 16:05:11 GMT, Content-Type: application/json, Content-Length: 366, Connection: close, , {\x0a  "origin": "10.142.133.148",\x0a  "files": {},\x0a  "form": null,\x0a  "url": "http://httpbin.org/post",\x0a  "args": {},\x0a  "headers": {\x0a    "Content-Length": "11",\x0a    "Connection": "close",\x0a    "Accept": "*/*",\x0a    "User-Agent": "curl/7.29.0",\x0a    "Host": "httpbin.org",\x0a    "Content-Type": "application/x-www-form-urlencoded"\x0a  },\x0a  "json": null,\x0a  "data": "hello world"\x0a}]

testing/btest/Baseline/spicy.spicyz-jit/output

@@ -0,0 +1,5 @@
+### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
+8, [$data=[b"POST /post HTTP/1.1", b"User-Agent: curl/7.29.0", b"Host: httpbin.org", b"Accept: */*", b"Content-Length: 11", b"Content-Type: application/x-www-form-urlencoded", b"", b"hello world"]]
+Event:, [POST /post HTTP/1.1, User-Agent: curl/7.29.0, Host: httpbin.org, Accept: */*, Content-Length: 11, Content-Type: application/x-www-form-urlencoded, , hello world]
+8, [$data=[b"HTTP/1.1 200 OK", b"Server: gunicorn/0.16.1", b"Date: Tue, 19 Mar 2013 16:05:11 GMT", b"Content-Type: application/json", b"Content-Length: 366", b"Connection: close", b"", b"{\x0a  \"origin\": \"10.142.133.148\",\x0a  \"files\": {},\x0a  \"form\": null,\x0a  \"url\": \"http://httpbin.org/post\",\x0a  \"args\": {},\x0a  \"headers\": {\x0a    \"Content-Length\": \"11\",\x0a    \"Connection\": \"close\",\x0a    \"Accept\": \"*/*\",\x0a    \"User-Agent\": \"curl/7.29.0\",\x0a    \"Host\": \"httpbin.org\",\x0a    \"Content-Type\": \"application/x-www-form-urlencoded\"\x0a  },\x0a  \"json\": null,\x0a  \"data\": \"hello world\"\x0a}"]]
+Event:, [HTTP/1.1 200 OK, Server: gunicorn/0.16.1, Date: Tue, 19 Mar 2013 16:05:11 GMT, Content-Type: application/json, Content-Length: 366, Connection: close, , {\x0a  "origin": "10.142.133.148",\x0a  "files": {},\x0a  "form": null,\x0a  "url": "http://httpbin.org/post",\x0a  "args": {},\x0a  "headers": {\x0a    "Content-Length": "11",\x0a    "Connection": "close",\x0a    "Accept": "*/*",\x0a    "User-Agent": "curl/7.29.0",\x0a    "Host": "httpbin.org",\x0a    "Content-Type": "application/x-www-form-urlencoded"\x0a  },\x0a  "json": null,\x0a  "data": "hello world"\x0a}]

testing/btest/bifs/bare_mode.zeek

@@ -0,0 +1,8 @@
+# @TEST-EXEC: zeek %INPUT >>output
+# @TEST-EXEC: zeek -b %INPUT >>output
+# @TEST-EXEC: btest-diff output
+
+event zeek_init()
+	{
+	print bare_mode();
+	}

testing/btest/broker/remote_event.zeek

@@ -1,5 +1,7 @@
 # @TEST-GROUP: broker
 #
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not.
+#
 # @TEST-PORT: BROKER_PORT
 #
 # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"

testing/btest/broker/remote_event_any.zeek

@@ -1,5 +1,7 @@
 # @TEST-GROUP: broker
 #
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not.
+#
 # @TEST-PORT: BROKER_PORT
 #
 # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"

testing/btest/broker/remote_event_ssl_auth.zeek

@@ -1,5 +1,7 @@
 # @TEST-GROUP: broker
 #
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not.
+#
 # @TEST-PORT: BROKER_PORT
 #
 # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"

testing/btest/btest.cfg

@@ -1,5 +1,5 @@
 [btest]
-TestDirs    = doc bifs language core scripts coverage signatures plugins broker supervisor telemetry
+TestDirs    = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry
 TmpDir      = %(testbase)s/.tmp
 BaselineDir = %(testbase)s/Baseline
 IgnoreDirs  = .svn CVS .tmp
@@ -32,6 +32,8 @@ ZEEK_DISABLE_ZEEKYGEN=1
 ZEEK_ALLOW_INIT_ERRORS=1
 ZEEK_SUPERVISOR_NO_SIGKILL=1
 UBSAN_OPTIONS=print_stacktrace=1
+SPICY_PATH=`bash -c %(testbase)s/../../build/spicy-path`
+HILTI_CXX_INCLUDE_DIRS=`bash -c %(testbase)s/../../build/hilti-cxx-include-dirs`
 
 [environment-AST-dup]
 # Environment for testing AST duplication functionality, which is

testing/btest/core/check-unused-event-handlers.test

@@ -1,4 +1,5 @@
 # This test should print a warning that the event handler is never invoked.
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # This test logs uninvoked event handlers, so disable it if Spicy and its plugin is unavailable.
 # @TEST-EXEC: zeek -b %INPUT check_for_unused_event_handlers=T
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
 

testing/btest/coverage/bare-load-baseline.test

@@ -7,6 +7,7 @@
 # prefix to make the test work everywhere. That's what the sed magic
 # below does. Don't ask. :-)
 
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable.
 # @TEST-EXEC: zeek -b misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
 # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix

testing/btest/coverage/default-load-baseline.test

@@ -7,6 +7,7 @@
 # prefix to make the test work everywhere. That's what the sed magic
 # below does. Don't ask. :-)
 
+# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config
 # @TEST-EXEC: zeek misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
 # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix

testing/btest/plugins/hooks-plugin/src/Plugin.cc

@@ -1,6 +1,8 @@
 
 #include "Plugin.h"
 
+#include <cstring>
+
 #include <Func.h>
 #include <Event.h>
 #include <Conn.h>
@@ -40,13 +42,36 @@ zeek::plugin::Configuration Plugin::Configure()
 static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek::ODesc* d)
 	{
 	bool first = true;
+	bool serialize_args = true;
 
 	for ( zeek::plugin::HookArgumentList::const_iterator i = args.begin(); i != args.end(); i++ )
 		{
-		if ( ! first )
+		if ( first )
-			d->Add(", ");
+			{
+			first = false;
+
+			i->Describe(d);
+
+			// For function calls we remove args for unstable arguments
+			// from parsing the version in `base/misc/version`.
+			if ( i->GetType() == zeek::plugin::HookArgument::FUNC &&
+			    (::strcmp(d->Description(), "Version::parse") == 0 ||
+			     ::strcmp(d->Description(), "gsub") == 0 ||
+			     ::strcmp(d->Description(), "split_string1") == 0 ||
+			     ::strcmp(d->Description(), "lstrip") == 0 ||
+			     ::strcmp(d->Description(), "to_count") == 0))
+				serialize_args = false;
+
+			continue;
+			}
+
+		d->Add(", ");
+
+		if ( serialize_args )
+			i->Describe(d);
+		else
+			d->Add("...");
 
-		i->Describe(d);
 		first = false;
 		}
 	}
@@ -70,8 +95,20 @@ std::pair<bool, zeek::ValPtr> Plugin::HookFunctionCall(const zeek::Func* func, z
 	{
 	zeek::ODesc d;
 	d.SetShort();
+
 	zeek::plugin::HookArgument(func).Describe(&d);
-	zeek::plugin::HookArgument(args).Describe(&d);
+
+	// For function calls we remove args for unstable arguments
+	// from parsing the version in `base/misc/version`.
+	if ( ::strcmp(d.Description(), "Version::parse") == 0 ||
+	     ::strcmp(d.Description(), "gsub") == 0 ||
+	     ::strcmp(d.Description(), "split_string1") == 0 ||
+	     ::strcmp(d.Description(), "lstrip") == 0 ||
+	     ::strcmp(d.Description(), "to_count") == 0)
+		d.Add("(...)");
+	else
+		zeek::plugin::HookArgument(args).Describe(&d);
+
 	fprintf(stderr, "%.6f %-15s %s\n", zeek::run_state::network_time, "| HookCallFunction",
 		d.Description());
 

testing/btest/plugins/hooks.zeek

@@ -1,13 +1,11 @@
 # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1"
+# @TEST-REQUIRES: ${SCRIPTS}/have-spicy  # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable.
 # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks
 # @TEST-EXEC: cp -r %DIR/hooks-plugin/* .
 # @TEST-EXEC: ./configure --zeek-dist=${DIST} && make
-# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT s1.sig 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq  >output
+# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -r $TRACES/http/get.trace %INPUT s1.sig 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq  >output
 # @TEST-EXEC: btest-diff output
 
-@unload base/misc/version
-@load base/init-default
-
 @load-sigs s2
 
 @TEST-START-FILE s1.sig

testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek

@@ -1,3 +1,4 @@
+# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config
 # @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT
 # @TEST-EXEC: btest-diff conn.log
 # @TEST-EXEC: btest-diff .stderr

testing/btest/spicy/spicy-dump.spicy

@@ -0,0 +1,11 @@
+# @TEST-DOC: Smoke test for a bundled Spicy.
+#
+# @TEST-REQUIRES: $SCRIPTS/have-spicy
+# @TEST-EXEC: printf 12345 | ${BUILD}/auxil/spicy/spicy/bin/spicy-dump -d %INPUT >output 2>&1
+# @TEST-EXEC: btest-diff output
+
+module test;
+
+public type X = unit {
+    xs: (bytes &size=1 &convert=$$.to_int()*11)[];
+};

testing/btest/spicy/spicyz-aot.test

@@ -0,0 +1,37 @@
+# @TEST-DOC: Smoke test for a custom ahead-of-time compiled Spicy analyzer hooked into Zeek.
+#
+# @TEST-REQUIRES: $SCRIPTS/have-spicy
+# @TEST-EXEC: ${BUILD}/src/builtin-plugins/spicy-plugin/bin/spicyz test.spicy test.evt -o test.hlto
+# @TEST-EXEC: zeek -NN test.hlto | grep -q ANALYZER_SPICY_TEST
+# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.zeek test.hlto "Spicy::enable_print = T;" >>output 2>&1
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE test.spicy
+module test;
+
+import zeek;
+
+public type Dummy = unit {
+    # Consume all data. We split data into lines and log the number of lines and the lines when done.
+    data: bytes &eod &convert=$$.split(b"\r\n");
+
+    on %done { print |self.data|, self; }
+};
+# @TEST-END-FILE
+
+# @TEST-START-FILE test.evt
+protocol analyzer spicy::Test over TCP:
+    parse with test::Dummy,
+    port 80/tcp;
+
+on test::Dummy -> event test::dummy(self.data);
+# @TEST-END-FILE
+
+# @TEST-START-FILE test.zeek
+module test;
+
+event test::dummy(data: vector of string)
+{
+	print "Event:", data;
+}
+# @TEST-END-FILE

testing/btest/spicy/spicyz-jit.test

@@ -0,0 +1,37 @@
+# @TEST-DOC: Smoke test for a custom Spicy analyzer hooked into Zeek with JIT via Zeek itself.
+#
+# @TEST-REQUIRES: $SCRIPTS/have-spicy
+# @TEST-EXEC: ${BUILD}/src/builtin-plugins/spicy-plugin/bin/spicyz test.spicy test.evt -o test.hlto
+# @TEST-EXEC: zeek -NN test.hlto | grep -q ANALYZER_SPICY_TEST
+# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.zeek test.spicy test.evt "Spicy::enable_print = T;" >>output 2>&1
+# @TEST-EXEC: btest-diff output
+
+# @TEST-START-FILE test.spicy
+module test;
+
+import zeek;
+
+public type Dummy = unit {
+    # Consume all data. We split data into lines and log the number of lines and the lines when done.
+    data: bytes &eod &convert=$$.split(b"\r\n");
+
+    on %done { print |self.data|, self; }
+};
+# @TEST-END-FILE
+
+# @TEST-START-FILE test.evt
+protocol analyzer spicy::Test over TCP:
+    parse with test::Dummy,
+    port 80/tcp;
+
+on test::Dummy -> event test::dummy(self.data);
+# @TEST-END-FILE
+
+# @TEST-START-FILE test.zeek
+module test;
+
+event test::dummy(data: vector of string)
+{
+	print "Event:", data;
+}
+# @TEST-END-FILE

testing/btest/supervisor/config-bare-mode.zeek

@@ -53,7 +53,7 @@ event zeek_init()
 		print node_output_file, "supervised node zeek_init()";
 
 # This is only defined when we're loading init-default.zeek:
-@ifdef ( Notice::Info )
+@ifdef ( Conn::LOG )
 		print node_output_file, "default mode";
 @else
 		print node_output_file, "bare mode";

testing/scripts/have-spicy

@@ -0,0 +1,6 @@
+#!/bin/sh
+if grep -q "DISABLE_SPICY:BOOL=true" "${BUILD}"/CMakeCache.txt; then
+    return 1
+else
+    true
+fi