Include spicy in build.

.cirrus.yml

@@ -12,11 +12,12 @@ memory: &MEMORY 8GB
 
 config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install
-asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage
+asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy
-ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers
+ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy
-tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers
+tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy
 mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install
 openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install
+homebrew_flex_bison_config: &HOMEBREW_FLEX_BISON_CONFIG --with-flex=/usr/local/opt/flex/bin/flex --with-bison=/usr/local/opt/bison/bin/bison
 
 resources_template: &RESOURCES_TEMPLATE
   cpu: *CPUS
@@ -216,6 +217,8 @@ macos_monterey_task:
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG
 
 macos_big_sur_task:
   macos_instance:
@@ -223,6 +226,8 @@ macos_big_sur_task:
   prepare_script: ./ci/macos/prepare.sh
   << : *CI_TEMPLATE
   << : *MACOS_RESOURCES_TEMPLATE
+  env:
+    ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG
 
 # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup
 freebsd14_task:

.github/workflows/generate-docs.yml

@@ -43,6 +43,8 @@ jobs:
             g++ \
             flex \
             bison \
+            libfl2 \
+            libfl-dev \
             libpcap-dev \
             libssl-dev \
             python3 \
@@ -60,7 +62,7 @@ jobs:
           sudo pip3 install -r doc/requirements.txt
 
       - name: Configure
-        run: ./configure --disable-broker-tests --disable-cpp-tests
+        run: ./configure --disable-broker-tests --disable-cpp-tests --disable-spicy
 
       - name: Build
         run: cd build && make -j $(nproc)

.gitmodules

@@ -58,3 +58,9 @@
 [submodule "auxil/out_ptr"]
 	path = auxil/out_ptr
 	url = https://github.com/soasis/out_ptr.git
+[submodule "auxil/spicy-plugin"]
+	path = auxil/spicy-plugin
+	url = https://github.com/zeek/spicy-plugin
+[submodule "auxil/spicy"]
+	path = auxil/spicy/spicy
+	url = https://github.com/zeek/spicy

.lgtm.yml

@@ -7,6 +7,8 @@ path_classifiers:
     - "auxil/libkqueue/"
     - "auxil/highwayhash/"
     - "auxil/rapidjson/"
+    - "auxil/spicy"
+    - "auxil/spicy-plugin"
     - "src/3rdparty/"
 
 # Filter out alerts that aren't concerning.
@@ -38,7 +40,9 @@ extraction:
 
     configure:
       command:
-        - ./configure --build-type=debug --generator=Ninja
+        # Disabling Spicy in LGTM build since it seems to lead to a LGTM
+        # performance edge case where analysis does not finish in 4h.
+        - ./configure --build-type=debug --generator=Ninja --disable-spicy
 
     index:
       build_command:

CMakeLists.txt

@@ -1,6 +1,6 @@
 # When changing the minimum version here, also adapt
 # auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt
-cmake_minimum_required(VERSION 3.5...3.18 FATAL_ERROR)
+cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR)
 
 project(Zeek C CXX)
 
@@ -73,12 +73,16 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh
      "export ZEEKPATH=`${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n"
      "export ZEEK_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n"
-     "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+     "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n"
+     "export SPICY_PATH=\"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n"
+     "export HILTI_CXX_INCLUDE_DIRS=\"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n")
 
 file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh
      "setenv ZEEKPATH `${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n"
      "setenv ZEEK_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n"
-     "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n")
+     "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n"
+     "setenv SPICY_PATH \"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n"
+     "setenv HILTI_CXX_INCLUDE_DIRS \"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n")
 
 file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1)
 execute_process(COMMAND grep "^#define  *BRO_PLUGIN_API_VERSION"
@@ -374,6 +378,54 @@ else ()
   set(broker_includes ${CMAKE_CURRENT_SOURCE_DIR}/auxil/broker/include ${CMAKE_CURRENT_BINARY_DIR}/auxil/broker/include)
 endif ()
 
+if ( NOT DISABLE_SPICY )
+    if ( NOT Spicy_ROOT )
+        add_subdirectory(auxil/spicy)
+
+        # Set variables used by the spicy-plugin build since we are building Spicy
+        # as part of Zeek so spicy-plugin cannot use `spicy-config` at configure
+        # time to set these.
+        set(SPICY_CONFIG "<builtin>")
+        set(HAVE_SPICY "YES")
+        set(SPICY_HAVE_TOOLCHAIN "YES")
+        set(SPICY_INCLUDE_DIRS_RUNTIME
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/runtime/include
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/runtime/include
+            ${PROJECT_BINARY_DIR}/auxil/spicy/spicy/include)
+        set(SPICY_INCLUDE_DIRS_TOOLCHAIN
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/toolchain/include
+            ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/toolchain/include)
+        set(SPICY_LIBRARY spicy)
+
+        # Needed only for logging from CMake configure phase.
+        get_directory_property(
+            SPICY_VERSION
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION SPICY_VERSION)
+        get_directory_property(
+            SPICY_VERSION_NUMBER
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION SPICY_VERSION_NUMBER)
+        get_directory_property(
+            SPICY_PREFIX
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION CMAKE_INSTALL_PREFIX)
+        get_directory_property(
+            SPICY_BUILD_MODE
+            DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy
+            DEFINITION CMAKE_BUILD_TYPE)
+        set(SPICYC "<bundled>")
+    endif ()
+
+    if ( NOT SPICY_PLUGIN_PATH )
+        set(SPICY_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/spicy-plugin)
+    endif ()
+
+    # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/auxil/spicy-plugin)
+    # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin)
+    string(APPEND ZEEK_INCLUDE_PLUGINS ";${SPICY_PLUGIN_PATH}")
+endif ()
+
 include_directories(BEFORE
                     ${PCAP_INCLUDE_DIR}
                     ${BIND_INCLUDE_DIR}
@@ -616,6 +668,53 @@ CheckOptionalBuildSources(auxil/zeek-aux  Zeek-Aux  INSTALL_AUX_TOOLS)
 CheckOptionalBuildSources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER)
 CheckOptionalBuildSources(auxil/zeek-client ZeekClient INSTALL_ZEEK_CLIENT)
 
+if ( NOT DISABLE_SPICY )
+    if ( NOT Spicy_ROOT )
+        list(APPEND _spicy_rt_libs spicy-rt hilti-rt)
+
+        # Use the debug runtime libraries if we are building a debug Zeek.
+        if ( SPICY_BUILD_MODE STREQUAL "Debug" )
+            list(TRANSFORM _spicy_rt_libs APPEND "-debug")
+        endif ()
+
+        # If we build spicy-plugin into Zeek we also need to build the Spicy
+        # runtime dependencies into Zeek. Since no matter how Spicy itself was
+        # linked this is always a static library, link the object files so we
+        # get all symbols and can can resolve all potential dependencies of
+        # HLTO files at runtime.
+        #
+        # TODO(bbannier): Conceptually cleaner would be linking the runtime
+        # objects into spicy-plugin and then inherit that up to zeek, but it
+        # seems this does not work reliable (due incomplete support for object
+        # libraries in CMake?).
+        list(TRANSFORM _spicy_rt_libs APPEND "-objects")
+        target_link_libraries(zeek ${_spicy_rt_libs})
+
+        # Make sure we build targets of spicy-plugin after the `spicy` target.
+        add_dependencies(plugin-Zeek-Spicy spicy)
+        add_dependencies(spicyz spicy)
+
+        # Spicy JIT relies on the path of the current executable to figure out
+        # whether it is run from a build or an install tree. This assumption
+        # gets broken for e.g., `spicyz` when running from the build tree
+        # (probably fixable), and also when JIT'ing directly from a `zeek`
+        # invocation (much harder to fix). Instead we generate shell
+        # definitions to support running and using Spicy or spicy-plugin
+        # functionality in the build tree, including JIT'ing directly from
+        # Zeek.
+        configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/spicy-path.in ${CMAKE_BINARY_DIR}/spicy-path @ONLY)
+        configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/hilti-cxx-include-dirs.in ${CMAKE_BINARY_DIR}/hilti-cxx-include-dirs @ONLY)
+
+        # Also install spicy-plugin's CMake files into Zeek's global `cmake/` folder.
+        #
+        # NOTE: We do not install spicy-plugin's `FindZeek.cmake` since another
+        # version of this file is already provided by Zeek.
+        install(FILES auxil/spicy-plugin/cmake/ZeekSpicyAnalyzerSupport.cmake
+                      auxil/spicy-plugin/cmake/FindSpicy.cmake
+                DESTINATION share/zeek/cmake)
+    endif ()
+endif ()
+
 ########################################################################
 ## Packaging Setup
 

auxil/spicy-plugin

@@ -0,0 +1 @@
+Subproject commit 0435d34acf92ff07558d72524fb2669eca78b1c9

auxil/spicy/CMakeLists.txt

@@ -0,0 +1,38 @@
+# Spicy needs the full prefix for Flex and Bison while Zeek captures only the
+# paths to the executables. Derive the prefixes from the binary paths under the
+# assumption that their bindir is under their prefix (which also implies that
+# one such prefix even exists).
+if ( NOT FLEX_EXECUTABLE )
+   find_package(FLEX REQUIRED)
+endif ()
+get_filename_component(dir ${FLEX_EXECUTABLE} DIRECTORY ABSOLUTE)
+set(FLEX_ROOT ${dir}/..)
+
+if ( NOT BISON_EXECUTABLE )
+   find_package(BISON REQUIRED)
+endif ()
+get_filename_component(dir ${BISON_EXECUTABLE} DIRECTORY ABSOLUTE)
+set(BISON_ROOT ${dir}/..)
+
+# We currently only support building against statically linked Spicy libraries.
+# This is due to global state held in the Spicy and HILTI runtime libraries. If
+# we would like Spicy dynamically this global state would end up the the final
+# binary once via the static link of the runtime libraries into Zeek (needed to
+# make sure we can e.g., load arbitrary HLTO files), and once as a link
+# dependency of spicy-plugin. This can lead to e.g. consistency issues during
+# execution or the problem during teardown where the same global might be
+# cleaned up multiple times.
+set(BUILD_SHARED_LIBS OFF)
+
+# Spicy uses slightly less strict warnings than Zeek proper. Mute a few warnings for Spicy.
+# NOTE: Compiler flags are inherited down the directory tree, so in order to
+# set these flags we do need a customizable subdirectory above the Spicy
+# sources.
+set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-braces")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla")
+
+# The script generating precompiled headers for Spicy expects a different build
+# system layout than provided for a bundled Spicy, disable it.
+set(HILTI_DEV_PRECOMPILE_HEADERS OFF)
+
+add_subdirectory(spicy)

auxil/spicy/hilti-cxx-include-dirs.in

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# Computes CXX flags for Spicy compilation in the build tree. The value
+# computed by this script can be injected into Spicy JIT via the environment
+# variable `HILTI_CXX_INCLUDE_DIRS`.
+
+# Paths to support compiling Spicy parsers in the build tree.
+PATHS=@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/runtime/include
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/runtime/include
+PATHS=$PATHS:@CMAKE_BINARY_DIR@/auxil/spicy/spicy/include
+
+# Paths to support compiling against a build tree Zeek.
+PATHS=$PATHS:@CMAKE_BINARY_DIR@
+PATHS=$PATHS:@CMAKE_BINARY_DIR@/src
+PATHS=$PATHS:@CMAKE_BINARY_DIR@/src/include
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/src
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/broker/include/
+
+echo $PATHS

auxil/spicy/spicy

@@ -0,0 +1 @@
+Subproject commit ef859d07a137bae77ffd8f04908dbdbb5d8649ef

auxil/spicy/spicy-path.in

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+# Computes a value for the environment variable `SPICY_PATH` for execution of
+# Spicy tooling in the build tree.
+
+PATHS=.
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/lib
+PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/lib
+
+echo $PATHS

ci/alpine/Dockerfile

@@ -6,7 +6,7 @@ RUN apk add --no-cache \
   g++ \
   python3 \
   python3-dev \
-  flex \
+  flex-dev \
   bison \
   libpcap-dev \
   openssl-dev \

ci/centos-7/Dockerfile

@@ -1,5 +1,11 @@
 FROM centos:7
 
+ENV FLEX_VERSION=2.6.4
+ENV FLEX_DIR=/opt/flex
+
+# Disabled lookup of fastest mirror since the list seems to be outdated and no valid mirror can be detected.
+RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf
+
 # The version of git in the standard repos is 1.8 and CI needs 2.3+
 # for the use of GIT_SSH_COMMAND when cloning private repos.
 RUN yum -y install \
@@ -17,13 +23,13 @@ RUN yum -y install \
   && yum clean all && rm -rf /var/cache/yum
 
 RUN yum -y install \
-    devtoolset-7 \
+    devtoolset-8 \
   && yum clean all && rm -rf /var/cache/yum
 
 RUN yum -y install \
     cmake3 \
+    curl \
     make \
-    flex \
     bison \
     swig \
     openssl \
@@ -38,10 +44,18 @@ RUN yum -y install \
     which \
   && yum clean all && rm -rf /var/cache/yum
 
+# Install a recent flex for Spicy.
+RUN curl -sSL "https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz" | tar xzf - -C /tmp \
+ && (cd /tmp/flex-${FLEX_VERSION} \
+     && ./configure --prefix=${FLEX_DIR} \
+     && make -j`nproc` install) \
+ && rm -rf /tmp/flex-${FLEX_VERSION}
+
 RUN pip3 install junit2html
 
 RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \
-    echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env
+    echo 'source /opt/rh/devtoolset-8/enable' >> /usr/bin/zeek-ci-env && \
+    echo 'export PATH=${PATH}:${FLEX_DIR}/bin' >> /usr/bin/zeek-ci-env
 
 ENV BASH_ENV="/usr/bin/zeek-ci-env" \
     ENV="/usr/bin/zeek-ci-env" \

ci/debian-10/Dockerfile

@@ -2,9 +2,11 @@ FROM debian:10
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
     make \
     gcc \
     g++ \
@@ -23,6 +25,11 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     xz-utils \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  # Install a recent CMake to build Spicy.
+  && mkdir -p "${CMAKE_DIR}" \
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 \
 
-RUN pip3 install junit2html
+  && pip3 install junit2html
+
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"

ci/debian-9-32bit/Dockerfile

@@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
 ENV DOCKERFILE_VERSION 221001
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
+    clang-11 \
     make \
-    gcc \
-    g++ \
     flex \
     bison \
+    libc++-11-dev \
+    libc++abi-11-dev \
     libpcap-dev \
     libssl-dev \
     python3 \
@@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     xz-utils \
-    clang-7 \
+  && rm -rf /var/lib/apt/lists/* \
-    libc++-7-dev \
+  # Recent CMake.
-    libc++abi-7-dev \
+  && mkdir -p "${CMAKE_DIR}" \
-  && rm -rf /var/lib/apt/lists/*
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1
 
-RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100
+RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-11 100
-RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100
+RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-11 100
 
 RUN pip3 install junit2html
 

ci/debian-9/Dockerfile

@@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822
 ENV DOCKERFILE_VERSION 221001
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
+    clang-11 \
     make \
-    gcc \
-    g++ \
     flex \
     bison \
+    libc++-11-dev \
+    libc++abi-11-dev \
     libpcap-dev \
     libssl-dev \
     python3 \
@@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \
     curl \
     wget \
     xz-utils \
-    clang-7 \
+  && rm -rf /var/lib/apt/lists/* \
-    libc++-7-dev \
+  # Recent CMake.
-    libc++abi-7-dev \
+  && mkdir -p "${CMAKE_DIR}" \
-  && rm -rf /var/lib/apt/lists/*
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1
 
 RUN pip3 install junit2html
 
-ENV CC=/usr/bin/clang-7
+ENV CC=/usr/bin/clang-11
-ENV CXX=/usr/bin/clang++-7
+ENV CXX=/usr/bin/clang++-11
 ENV CXXFLAGS=-stdlib=libc++

ci/freebsd/prepare.sh

@@ -6,8 +6,12 @@ set -e
 set -x
 
 env ASSUME_ALWAYS_YES=YES pkg bootstrap
-pkg install -y bash git cmake swig bison python3 base64
+pkg install -y bash git cmake swig bison python3 base64 flex
 pkg upgrade -y curl
 pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")')
 pkg install -y $pyver-sqlite3 $pyver-pip
 pip install junit2html
+
+# Spicy detects whether it is run from build directory via `/proc`.
+echo "proc /proc procfs rw,noauto 0 0" >>/etc/fstab
+mount /proc

ci/macos/prepare.sh

@@ -7,4 +7,4 @@ set -x
 
 brew update
 brew upgrade cmake openssl@1.1
-brew install swig bison
+brew install swig bison flex

ci/opensuse-leap-15.3/Dockerfile

@@ -1,10 +1,12 @@
 FROM opensuse/leap:15.3
 
-RUN zypper in -y \
+RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.2:Update/standard/openSUSE:Leap:15.2:Update.repo \
+ && zypper refresh \
+ && zypper in -y \
   cmake \
   make \
-  gcc \
+  gcc9 \
-  gcc-c++ \
+  gcc9-c++ \
   python3 \
   python3-devel \
   flex \
@@ -21,5 +23,7 @@ RUN zypper in -y \
   tar \
   && rm -rf /var/cache/zypp
 
-
 RUN pip3 install junit2html
+
+ENV CXX g++-9
+ENV CC gcc-9

ci/ubuntu-18.04/Dockerfile

@@ -2,12 +2,14 @@ FROM ubuntu:18.04
 
 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles"
 
+ENV CMAKE_DIR "/opt/cmake"
+ENV CMAKE_VERSION "3.19.1"
+ENV PATH "${CMAKE_DIR}/bin:${PATH}"
+
 RUN apt-get update && apt-get -y install \
     git \
-    cmake \
+    clang-10 \
     make \
-    gcc \
-    g++ \
     flex \
     bison \
     libpcap-dev \
@@ -27,7 +29,13 @@ RUN apt-get update && apt-get -y install \
     ruby \
     bc \
     lcov \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  # Recent CMake.
+  && mkdir -p "${CMAKE_DIR}" \
+  && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1
+
+ENV CC=/usr/bin/clang-10
+ENV CXX=/usr/bin/clang++-10
 
 RUN pip3 install junit2html
 RUN gem install coveralls-lcov

configure

@@ -71,6 +71,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --disable-btest-pcaps  don't install Zeek's BTest input pcaps
     --disable-cpp-tests    don't build Zeek's C++ unit tests
     --disable-python       don't try to build python bindings for Broker
+    --disable-spicy        don't include Spicy
     --disable-zeekctl      don't install ZeekControl
     --disable-zkg          don't install zkg
 
@@ -101,6 +102,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]...
     --with-perftools=PATH  path to Google Perftools install root
     --with-python-inc=PATH path to Python headers
     --with-python-lib=PATH path to libpython
+    --with-spicy=PATH      path to Spicy install root
+    --with-spicy-plugin=PATH path to Spicy plugin source tree
     --with-swig=PATH       path to SWIG executable
 
   Packaging Options (for developers):
@@ -313,6 +316,9 @@ while [ $# -ne 0 ]; do
         --disable-python)
             append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true
             ;;
+        --disable-spicy)
+            append_cache_entry DISABLE_SPICY BOOL true
+            ;;
         --disable-zeekctl)
             append_cache_entry INSTALL_ZEEKCTL BOOL false
             ;;
@@ -375,6 +381,12 @@ while [ $# -ne 0 ]; do
         --with-python-lib=*)
             append_cache_entry PYTHON_LIBRARY PATH $optarg
             ;;
+        --with-spicy=*)
+            append_cache_entry SPICY_ROOT PATH $optarg
+            ;;
+        --with-spicy-plugin=*)
+            append_cache_entry SPICY_PLUGIN_PATH PATH $optarg
+            ;;
         --with-swig=*)
             append_cache_entry SWIG_EXECUTABLE PATH $optarg
             ;;

docker/Dockerfile

@@ -12,6 +12,8 @@ RUN apt-get -q update \
      flex \
      g++ \
      gcc \
+     libfl2 \
+     libfl-dev \
      libmaxminddb-dev \
      libpcap-dev \
      libssl-dev \

testing/btest/Baseline/broker.remote_event/recv.recv.out

@@ -10,4 +10,4 @@ is_remote should be T, and is, T
 receiver got ping: my-message, 4
 is_remote should be T, and is, T
 receiver got ping: my-message, 5
-[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/broker.remote_event_any/recv.recv.out

@@ -10,4 +10,4 @@ is_remote should be T, and is, T
 receiver got ping: my-message, 4
 is_remote should be T, and is, T
 receiver got ping: my-message, 5
-[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out

@@ -5,4 +5,4 @@ receiver got ping: my-message, 2
 receiver got ping: my-message, 3
 receiver got ping: my-message, 4
 receiver got ping: my-message, 5
-[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0]
+[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0]

testing/btest/Baseline/core.check-unused-event-handlers/.stderr

@@ -7,4 +7,5 @@ warning in <params>, line 1: event handler never invoked: SupervisorControl::des
 warning in <params>, line 1: event handler never invoked: SupervisorControl::restart_request
 warning in <params>, line 1: event handler never invoked: SupervisorControl::status_request
 warning in <params>, line 1: event handler never invoked: SupervisorControl::stop_request
+warning in <params>, line 1: event handler never invoked: spicy_analyzer_for_mime_type
 warning in <params>, line 1: event handler never invoked: this_is_never_used

testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log

@@ -243,8 +243,18 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek
     build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
     build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek
 build/scripts/builtin-plugins/__preload__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek
 build/scripts/builtin-plugins/__load__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek
+    build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek
+      scripts/base/misc/version.zeek
+        scripts/base/frameworks/reporter/__load__.zeek
+          scripts/base/frameworks/reporter/main.zeek
+        scripts/base/utils/strings.zeek
 scripts/policy/misc/loaded-scripts.zeek
   scripts/base/utils/paths.zeek
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log

@@ -243,6 +243,9 @@ scripts/base/init-frameworks-and-bifs.zeek
     build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek
     build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek
     build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek
+    build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek
 scripts/base/init-default.zeek
   scripts/base/utils/active-http.zeek
     scripts/base/utils/exec.zeek
@@ -443,6 +446,10 @@ scripts/base/init-default.zeek
   scripts/base/misc/find-filtered-trace.zeek
   scripts/base/misc/version.zeek
 build/scripts/builtin-plugins/__preload__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek
 build/scripts/builtin-plugins/__load__.zeek
+  build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek
+    build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek
+    build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/default.zeek
 scripts/policy/misc/loaded-scripts.zeek
 #close XXXX-XX-XX-XX-XX-XX

testing/btest/Baseline/plugins.hooks/output

@@ -815,6 +815,9 @@
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> -1
@@ -830,9 +833,6 @@
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -> -1
-0.000000   MetaHookPost  LoadFile(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek) -> -1
-0.000000   MetaHookPost  LoadFile(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek) -> -1
-0.000000   MetaHookPost  LoadFile(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./acld, <...>/acld.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./addrs, <...>/addrs.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> -1
@@ -936,8 +936,8 @@
 0.000000   MetaHookPost  LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -> -1
-0.000000   MetaHookPost  LoadFile(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1
-0.000000   MetaHookPost  LoadFile(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1
+0.000000   MetaHookPost  LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base/bif, <...>/bif) -> -1
 0.000000   MetaHookPost  LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) -> -1
 0.000000   MetaHookPost  LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> -1
@@ -1199,6 +1199,9 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, <no content>)
@@ -1214,9 +1217,6 @@
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -> (-1, <no content>)
-0.000000   MetaHookPost  LoadFileExtended(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek) -> (-1, <no content>)
-0.000000   MetaHookPost  LoadFileExtended(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek) -> (-1, <no content>)
-0.000000   MetaHookPost  LoadFileExtended(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./acld, <...>/acld.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./addrs, <...>/addrs.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> (-1, <no content>)
@@ -1320,8 +1320,8 @@
 0.000000   MetaHookPost  LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -> (-1, <no content>)
-0.000000   MetaHookPost  LoadFileExtended(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, <no content>)
-0.000000   MetaHookPost  LoadFileExtended(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, <no content>)
+0.000000   MetaHookPost  LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base/bif, <...>/bif) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) -> (-1, <no content>)
 0.000000   MetaHookPost  LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, <no content>)
@@ -2293,6 +2293,9 @@
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek)
+0.000000   MetaHookPre   LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek)
@@ -2308,9 +2311,6 @@
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek)
-0.000000   MetaHookPre   LoadFile(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek)
-0.000000   MetaHookPre   LoadFile(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek)
-0.000000   MetaHookPre   LoadFile(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./acld, <...>/acld.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./addrs, <...>/addrs.zeek)
 0.000000   MetaHookPre   LoadFile(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek)
@@ -2414,8 +2414,8 @@
 0.000000   MetaHookPre   LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek)
 0.000000   MetaHookPre   LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek)
 0.000000   MetaHookPre   LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek)
-0.000000   MetaHookPre   LoadFile(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek)
+0.000000   MetaHookPre   LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek)
-0.000000   MetaHookPre   LoadFile(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek)
+0.000000   MetaHookPre   LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek)
 0.000000   MetaHookPre   LoadFile(0, base/bif, <...>/bif)
 0.000000   MetaHookPre   LoadFile(0, base/init-default.zeek, <...>/init-default.zeek)
 0.000000   MetaHookPre   LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek)
@@ -2677,6 +2677,9 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek)
@@ -2692,9 +2695,6 @@
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek)
-0.000000   MetaHookPre   LoadFileExtended(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek)
-0.000000   MetaHookPre   LoadFileExtended(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek)
-0.000000   MetaHookPre   LoadFileExtended(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./acld, <...>/acld.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./addrs, <...>/addrs.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek)
@@ -2798,8 +2798,8 @@
 0.000000   MetaHookPre   LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek)
-0.000000   MetaHookPre   LoadFileExtended(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek)
-0.000000   MetaHookPre   LoadFileExtended(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek)
+0.000000   MetaHookPre   LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base/bif, <...>/bif)
 0.000000   MetaHookPre   LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek)
 0.000000   MetaHookPre   LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek)
@@ -3770,6 +3770,9 @@
 0.000000 | HookLoadFile  ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek
+0.000000 | HookLoadFile  ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek
@@ -3785,9 +3788,6 @@
 0.000000 | HookLoadFile  ./Zeek_X509.ocsp_events.bif.zeek <...>/Zeek_X509.ocsp_events.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_X509.types.bif.zeek <...>/Zeek_X509.types.bif.zeek
 0.000000 | HookLoadFile  ./Zeek_XMPP.events.bif.zeek <...>/Zeek_XMPP.events.bif.zeek
-0.000000 | HookLoadFile  ./_Zeek_Spicy.consts.bif.zeek <...>/_Zeek_Spicy.consts.bif.zeek
-0.000000 | HookLoadFile  ./_Zeek_Spicy.events.bif.zeek <...>/_Zeek_Spicy.events.bif.zeek
-0.000000 | HookLoadFile  ./_Zeek_Spicy.functions.bif.zeek <...>/_Zeek_Spicy.functions.bif.zeek
 0.000000 | HookLoadFile  ./acld <...>/acld.zeek
 0.000000 | HookLoadFile  ./addrs <...>/addrs.zeek
 0.000000 | HookLoadFile  ./analyzer.bif.zeek <...>/analyzer.bif.zeek
@@ -3903,8 +3903,8 @@
 0.000000 | HookLoadFile  <...>/hooks.zeek <...>/hooks.zeek
 0.000000 | HookLoadFile  Zeek<...>/bare.zeek <...>/bare.zeek
 0.000000 | HookLoadFile  Zeek<...>/default.zeek <...>/default.zeek
-0.000000 | HookLoadFile  _Zeek_Spicy/__load__.zeek <...>/__load__.zeek
+0.000000 | HookLoadFile  Zeek_Spicy/__load__.zeek <...>/__load__.zeek
-0.000000 | HookLoadFile  _Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek
+0.000000 | HookLoadFile  Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek
 0.000000 | HookLoadFile  base/bif <...>/bif
 0.000000 | HookLoadFile  base/init-default.zeek <...>/init-default.zeek
 0.000000 | HookLoadFile  base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek
@@ -4154,6 +4154,9 @@
 0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek
+0.000000 | HookLoadFileExtended ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek
@@ -4169,9 +4172,6 @@
 0.000000 | HookLoadFileExtended ./Zeek_X509.ocsp_events.bif.zeek <...>/Zeek_X509.ocsp_events.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_X509.types.bif.zeek <...>/Zeek_X509.types.bif.zeek
 0.000000 | HookLoadFileExtended ./Zeek_XMPP.events.bif.zeek <...>/Zeek_XMPP.events.bif.zeek
-0.000000 | HookLoadFileExtended ./_Zeek_Spicy.consts.bif.zeek <...>/_Zeek_Spicy.consts.bif.zeek
-0.000000 | HookLoadFileExtended ./_Zeek_Spicy.events.bif.zeek <...>/_Zeek_Spicy.events.bif.zeek
-0.000000 | HookLoadFileExtended ./_Zeek_Spicy.functions.bif.zeek <...>/_Zeek_Spicy.functions.bif.zeek
 0.000000 | HookLoadFileExtended ./acld <...>/acld.zeek
 0.000000 | HookLoadFileExtended ./addrs <...>/addrs.zeek
 0.000000 | HookLoadFileExtended ./analyzer.bif.zeek <...>/analyzer.bif.zeek
@@ -4287,8 +4287,8 @@
 0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek
 0.000000 | HookLoadFileExtended Zeek<...>/bare.zeek <...>/bare.zeek
 0.000000 | HookLoadFileExtended Zeek<...>/default.zeek <...>/default.zeek
-0.000000 | HookLoadFileExtended _Zeek_Spicy/__load__.zeek <...>/__load__.zeek
+0.000000 | HookLoadFileExtended Zeek_Spicy/__load__.zeek <...>/__load__.zeek
-0.000000 | HookLoadFileExtended _Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek
+0.000000 | HookLoadFileExtended Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek
 0.000000 | HookLoadFileExtended base/bif <...>/bif
 0.000000 | HookLoadFileExtended base/init-default.zeek <...>/init-default.zeek
 0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek

testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr

@@ -1,2 +1,3 @@
 ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63.
 XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension
+XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension

testing/btest/broker/remote_event.zeek

@@ -1,5 +1,7 @@
 # @TEST-GROUP: broker
 #
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not.
+#
 # @TEST-PORT: BROKER_PORT
 #
 # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"

testing/btest/broker/remote_event_any.zeek

@@ -1,5 +1,7 @@
 # @TEST-GROUP: broker
 #
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not.
+#
 # @TEST-PORT: BROKER_PORT
 #
 # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"

testing/btest/broker/remote_event_ssl_auth.zeek

@@ -1,5 +1,7 @@
 # @TEST-GROUP: broker
 #
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not.
+#
 # @TEST-PORT: BROKER_PORT
 #
 # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out"

testing/btest/core/check-unused-event-handlers.test

@@ -1,4 +1,5 @@
 # This test should print a warning that the event handler is never invoked.
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # This test logs uninvoked event handlers, so disable it if Spicy and its plugin is unavailable.
 # @TEST-EXEC: zeek -b %INPUT check_for_unused_event_handlers=T
 # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr
 

testing/btest/coverage/bare-load-baseline.test

@@ -7,6 +7,7 @@
 # prefix to make the test work everywhere. That's what the sed magic
 # below does. Don't ask. :-)
 
+# @TEST-REQUIRES: $SCRIPTS/have-spicy  # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable.
 # @TEST-EXEC: zeek -b misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
 # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix

testing/btest/coverage/default-load-baseline.test

@@ -7,6 +7,7 @@
 # prefix to make the test work everywhere. That's what the sed magic
 # below does. Don't ask. :-)
 
+# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config
 # @TEST-EXEC: zeek misc/loaded-scripts
 # @TEST-EXEC: test -e loaded_scripts.log
 # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix

testing/btest/plugins/hooks.zeek

@@ -1,4 +1,5 @@
 # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1"
+# @TEST-REQUIRES: ${SCRIPTS}/have-spicy  # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable.
 # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks
 # @TEST-EXEC: cp -r %DIR/hooks-plugin/* .
 # @TEST-EXEC: ./configure --zeek-dist=${DIST} && make

testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek

@@ -1,3 +1,4 @@
+# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config
 # @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT
 # @TEST-EXEC: btest-diff conn.log
 # @TEST-EXEC: btest-diff .stderr