From 95aff9a1e38c63d57d2bbcbce038718a550a12bf Mon Sep 17 00:00:00 2001 From: Benjamin Bannier Date: Tue, 22 Mar 2022 09:27:31 +0100 Subject: [PATCH] Include spicy in build. --- .cirrus.yml | 11 +- .github/workflows/generate-docs.yml | 4 +- .gitmodules | 6 + .lgtm.yml | 6 +- CMakeLists.txt | 105 +++++++++++++++++- auxil/spicy-plugin | 1 + auxil/spicy/CMakeLists.txt | 38 +++++++ auxil/spicy/hilti-cxx-include-dirs.in | 19 ++++ auxil/spicy/spicy | 1 + auxil/spicy/spicy-path.in | 10 ++ ci/alpine/Dockerfile | 2 +- ci/centos-7/Dockerfile | 20 +++- ci/debian-10/Dockerfile | 13 ++- ci/debian-9-32bit/Dockerfile | 22 ++-- ci/debian-9/Dockerfile | 22 ++-- ci/freebsd/prepare.sh | 6 +- ci/macos/prepare.sh | 2 +- ci/opensuse-leap-15.3/Dockerfile | 12 +- ci/ubuntu-18.04/Dockerfile | 16 ++- configure | 12 ++ docker/Dockerfile | 2 + .../broker.remote_event/recv.recv.out | 2 +- .../broker.remote_event_any/recv.recv.out | 2 +- .../recv.recv.out | 2 +- .../core.check-unused-event-handlers/.stderr | 1 + .../canonified_loaded_scripts.log | 10 ++ .../canonified_loaded_scripts.log | 7 ++ testing/btest/Baseline/plugins.hooks/output | 60 +++++----- .../.stderr | 1 + testing/btest/broker/remote_event.zeek | 2 + testing/btest/broker/remote_event_any.zeek | 2 + .../btest/broker/remote_event_ssl_auth.zeek | 2 + .../core/check-unused-event-handlers.test | 1 + .../btest/coverage/bare-load-baseline.test | 1 + .../btest/coverage/default-load-baseline.test | 1 + testing/btest/plugins/hooks.zeek | 1 + .../logging/field-extension-invalid.zeek | 1 + 37 files changed, 350 insertions(+), 76 deletions(-) create mode 160000 auxil/spicy-plugin create mode 100644 auxil/spicy/CMakeLists.txt create mode 100755 auxil/spicy/hilti-cxx-include-dirs.in create mode 160000 auxil/spicy/spicy create mode 100755 auxil/spicy/spicy-path.in diff --git a/.cirrus.yml b/.cirrus.yml index 5b48bdce14..70336514d2 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -12,11 +12,12 @@ memory: &MEMORY 8GB config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install -asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage -ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers -tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers +asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy +ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy +tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install +homebrew_flex_bison_config: &HOMEBREW_FLEX_BISON_CONFIG --with-flex=/usr/local/opt/flex/bin/flex --with-bison=/usr/local/opt/bison/bin/bison resources_template: &RESOURCES_TEMPLATE cpu: *CPUS @@ -216,6 +217,8 @@ macos_monterey_task: prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE << : *MACOS_RESOURCES_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG macos_big_sur_task: macos_instance: @@ -223,6 +226,8 @@ macos_big_sur_task: prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE << : *MACOS_RESOURCES_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup freebsd14_task: diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 6159ddc528..8ddf861401 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -43,6 +43,8 @@ jobs: g++ \ flex \ bison \ + libfl2 \ + libfl-dev \ libpcap-dev \ libssl-dev \ python3 \ @@ -60,7 +62,7 @@ jobs: sudo pip3 install -r doc/requirements.txt - name: Configure - run: ./configure --disable-broker-tests --disable-cpp-tests + run: ./configure --disable-broker-tests --disable-cpp-tests --disable-spicy - name: Build run: cd build && make -j $(nproc) diff --git a/.gitmodules b/.gitmodules index cb20683187..d84fc4118a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,3 +58,9 @@ [submodule "auxil/out_ptr"] path = auxil/out_ptr url = https://github.com/soasis/out_ptr.git +[submodule "auxil/spicy-plugin"] + path = auxil/spicy-plugin + url = https://github.com/zeek/spicy-plugin +[submodule "auxil/spicy"] + path = auxil/spicy/spicy + url = https://github.com/zeek/spicy diff --git a/.lgtm.yml b/.lgtm.yml index 15857c99e2..13772c6e87 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -7,6 +7,8 @@ path_classifiers: - "auxil/libkqueue/" - "auxil/highwayhash/" - "auxil/rapidjson/" + - "auxil/spicy" + - "auxil/spicy-plugin" - "src/3rdparty/" # Filter out alerts that aren't concerning. @@ -38,7 +40,9 @@ extraction: configure: command: - - ./configure --build-type=debug --generator=Ninja + # Disabling Spicy in LGTM build since it seems to lead to a LGTM + # performance edge case where analysis does not finish in 4h. + - ./configure --build-type=debug --generator=Ninja --disable-spicy index: build_command: diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f7df8ffdb..721d710a35 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ # When changing the minimum version here, also adapt # auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt -cmake_minimum_required(VERSION 3.5...3.18 FATAL_ERROR) +cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) project(Zeek C CXX) @@ -73,12 +73,16 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh "export ZEEKPATH=`${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n" "export ZEEK_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n" - "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") + "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n" + "export SPICY_PATH=\"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n" + "export HILTI_CXX_INCLUDE_DIRS=\"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n") file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh "setenv ZEEKPATH `${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n" "setenv ZEEK_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n" - "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") + "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n" + "setenv SPICY_PATH \"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n" + "setenv HILTI_CXX_INCLUDE_DIRS \"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) execute_process(COMMAND grep "^#define *BRO_PLUGIN_API_VERSION" @@ -374,6 +378,54 @@ else () set(broker_includes ${CMAKE_CURRENT_SOURCE_DIR}/auxil/broker/include ${CMAKE_CURRENT_BINARY_DIR}/auxil/broker/include) endif () +if ( NOT DISABLE_SPICY ) + if ( NOT Spicy_ROOT ) + add_subdirectory(auxil/spicy) + + # Set variables used by the spicy-plugin build since we are building Spicy + # as part of Zeek so spicy-plugin cannot use `spicy-config` at configure + # time to set these. + set(SPICY_CONFIG "") + set(HAVE_SPICY "YES") + set(SPICY_HAVE_TOOLCHAIN "YES") + set(SPICY_INCLUDE_DIRS_RUNTIME + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/runtime/include + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/runtime/include + ${PROJECT_BINARY_DIR}/auxil/spicy/spicy/include) + set(SPICY_INCLUDE_DIRS_TOOLCHAIN + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/toolchain/include + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/toolchain/include) + set(SPICY_LIBRARY spicy) + + # Needed only for logging from CMake configure phase. + get_directory_property( + SPICY_VERSION + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION SPICY_VERSION) + get_directory_property( + SPICY_VERSION_NUMBER + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION SPICY_VERSION_NUMBER) + get_directory_property( + SPICY_PREFIX + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION CMAKE_INSTALL_PREFIX) + get_directory_property( + SPICY_BUILD_MODE + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION CMAKE_BUILD_TYPE) + set(SPICYC "") + endif () + + if ( NOT SPICY_PLUGIN_PATH ) + set(SPICY_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/spicy-plugin) + endif () + + # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/auxil/spicy-plugin) + # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin) + string(APPEND ZEEK_INCLUDE_PLUGINS ";${SPICY_PLUGIN_PATH}") +endif () + include_directories(BEFORE ${PCAP_INCLUDE_DIR} ${BIND_INCLUDE_DIR} @@ -616,6 +668,53 @@ CheckOptionalBuildSources(auxil/zeek-aux Zeek-Aux INSTALL_AUX_TOOLS) CheckOptionalBuildSources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER) CheckOptionalBuildSources(auxil/zeek-client ZeekClient INSTALL_ZEEK_CLIENT) +if ( NOT DISABLE_SPICY ) + if ( NOT Spicy_ROOT ) + list(APPEND _spicy_rt_libs spicy-rt hilti-rt) + + # Use the debug runtime libraries if we are building a debug Zeek. + if ( SPICY_BUILD_MODE STREQUAL "Debug" ) + list(TRANSFORM _spicy_rt_libs APPEND "-debug") + endif () + + # If we build spicy-plugin into Zeek we also need to build the Spicy + # runtime dependencies into Zeek. Since no matter how Spicy itself was + # linked this is always a static library, link the object files so we + # get all symbols and can can resolve all potential dependencies of + # HLTO files at runtime. + # + # TODO(bbannier): Conceptually cleaner would be linking the runtime + # objects into spicy-plugin and then inherit that up to zeek, but it + # seems this does not work reliable (due incomplete support for object + # libraries in CMake?). + list(TRANSFORM _spicy_rt_libs APPEND "-objects") + target_link_libraries(zeek ${_spicy_rt_libs}) + + # Make sure we build targets of spicy-plugin after the `spicy` target. + add_dependencies(plugin-Zeek-Spicy spicy) + add_dependencies(spicyz spicy) + + # Spicy JIT relies on the path of the current executable to figure out + # whether it is run from a build or an install tree. This assumption + # gets broken for e.g., `spicyz` when running from the build tree + # (probably fixable), and also when JIT'ing directly from a `zeek` + # invocation (much harder to fix). Instead we generate shell + # definitions to support running and using Spicy or spicy-plugin + # functionality in the build tree, including JIT'ing directly from + # Zeek. + configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/spicy-path.in ${CMAKE_BINARY_DIR}/spicy-path @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/hilti-cxx-include-dirs.in ${CMAKE_BINARY_DIR}/hilti-cxx-include-dirs @ONLY) + + # Also install spicy-plugin's CMake files into Zeek's global `cmake/` folder. + # + # NOTE: We do not install spicy-plugin's `FindZeek.cmake` since another + # version of this file is already provided by Zeek. + install(FILES auxil/spicy-plugin/cmake/ZeekSpicyAnalyzerSupport.cmake + auxil/spicy-plugin/cmake/FindSpicy.cmake + DESTINATION share/zeek/cmake) + endif () +endif () + ######################################################################## ## Packaging Setup diff --git a/auxil/spicy-plugin b/auxil/spicy-plugin new file mode 160000 index 0000000000..0435d34acf --- /dev/null +++ b/auxil/spicy-plugin @@ -0,0 +1 @@ +Subproject commit 0435d34acf92ff07558d72524fb2669eca78b1c9 diff --git a/auxil/spicy/CMakeLists.txt b/auxil/spicy/CMakeLists.txt new file mode 100644 index 0000000000..050ee9753e --- /dev/null +++ b/auxil/spicy/CMakeLists.txt @@ -0,0 +1,38 @@ +# Spicy needs the full prefix for Flex and Bison while Zeek captures only the +# paths to the executables. Derive the prefixes from the binary paths under the +# assumption that their bindir is under their prefix (which also implies that +# one such prefix even exists). +if ( NOT FLEX_EXECUTABLE ) + find_package(FLEX REQUIRED) +endif () +get_filename_component(dir ${FLEX_EXECUTABLE} DIRECTORY ABSOLUTE) +set(FLEX_ROOT ${dir}/..) + +if ( NOT BISON_EXECUTABLE ) + find_package(BISON REQUIRED) +endif () +get_filename_component(dir ${BISON_EXECUTABLE} DIRECTORY ABSOLUTE) +set(BISON_ROOT ${dir}/..) + +# We currently only support building against statically linked Spicy libraries. +# This is due to global state held in the Spicy and HILTI runtime libraries. If +# we would like Spicy dynamically this global state would end up the the final +# binary once via the static link of the runtime libraries into Zeek (needed to +# make sure we can e.g., load arbitrary HLTO files), and once as a link +# dependency of spicy-plugin. This can lead to e.g. consistency issues during +# execution or the problem during teardown where the same global might be +# cleaned up multiple times. +set(BUILD_SHARED_LIBS OFF) + +# Spicy uses slightly less strict warnings than Zeek proper. Mute a few warnings for Spicy. +# NOTE: Compiler flags are inherited down the directory tree, so in order to +# set these flags we do need a customizable subdirectory above the Spicy +# sources. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-braces") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla") + +# The script generating precompiled headers for Spicy expects a different build +# system layout than provided for a bundled Spicy, disable it. +set(HILTI_DEV_PRECOMPILE_HEADERS OFF) + +add_subdirectory(spicy) diff --git a/auxil/spicy/hilti-cxx-include-dirs.in b/auxil/spicy/hilti-cxx-include-dirs.in new file mode 100755 index 0000000000..06ae06a806 --- /dev/null +++ b/auxil/spicy/hilti-cxx-include-dirs.in @@ -0,0 +1,19 @@ +#!/bin/sh + +# Computes CXX flags for Spicy compilation in the build tree. The value +# computed by this script can be injected into Spicy JIT via the environment +# variable `HILTI_CXX_INCLUDE_DIRS`. + +# Paths to support compiling Spicy parsers in the build tree. +PATHS=@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/runtime/include +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/runtime/include +PATHS=$PATHS:@CMAKE_BINARY_DIR@/auxil/spicy/spicy/include + +# Paths to support compiling against a build tree Zeek. +PATHS=$PATHS:@CMAKE_BINARY_DIR@ +PATHS=$PATHS:@CMAKE_BINARY_DIR@/src +PATHS=$PATHS:@CMAKE_BINARY_DIR@/src/include +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/src +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/broker/include/ + +echo $PATHS diff --git a/auxil/spicy/spicy b/auxil/spicy/spicy new file mode 160000 index 0000000000..ef859d07a1 --- /dev/null +++ b/auxil/spicy/spicy @@ -0,0 +1 @@ +Subproject commit ef859d07a137bae77ffd8f04908dbdbb5d8649ef diff --git a/auxil/spicy/spicy-path.in b/auxil/spicy/spicy-path.in new file mode 100755 index 0000000000..3fab168230 --- /dev/null +++ b/auxil/spicy/spicy-path.in @@ -0,0 +1,10 @@ +#!/bin/sh + +# Computes a value for the environment variable `SPICY_PATH` for execution of +# Spicy tooling in the build tree. + +PATHS=. +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/lib +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/lib + +echo $PATHS diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile index be93e0c3d0..dcbbbe9146 100644 --- a/ci/alpine/Dockerfile +++ b/ci/alpine/Dockerfile @@ -6,7 +6,7 @@ RUN apk add --no-cache \ g++ \ python3 \ python3-dev \ - flex \ + flex-dev \ bison \ libpcap-dev \ openssl-dev \ diff --git a/ci/centos-7/Dockerfile b/ci/centos-7/Dockerfile index 5ab9df2d68..7928da7727 100644 --- a/ci/centos-7/Dockerfile +++ b/ci/centos-7/Dockerfile @@ -1,5 +1,11 @@ FROM centos:7 +ENV FLEX_VERSION=2.6.4 +ENV FLEX_DIR=/opt/flex + +# Disabled lookup of fastest mirror since the list seems to be outdated and no valid mirror can be detected. +RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf + # The version of git in the standard repos is 1.8 and CI needs 2.3+ # for the use of GIT_SSH_COMMAND when cloning private repos. RUN yum -y install \ @@ -17,13 +23,13 @@ RUN yum -y install \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ - devtoolset-7 \ + devtoolset-8 \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ cmake3 \ + curl \ make \ - flex \ bison \ swig \ openssl \ @@ -38,10 +44,18 @@ RUN yum -y install \ which \ && yum clean all && rm -rf /var/cache/yum +# Install a recent flex for Spicy. +RUN curl -sSL "https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz" | tar xzf - -C /tmp \ + && (cd /tmp/flex-${FLEX_VERSION} \ + && ./configure --prefix=${FLEX_DIR} \ + && make -j`nproc` install) \ + && rm -rf /tmp/flex-${FLEX_VERSION} + RUN pip3 install junit2html RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \ - echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env + echo 'source /opt/rh/devtoolset-8/enable' >> /usr/bin/zeek-ci-env && \ + echo 'export PATH=${PATH}:${FLEX_DIR}/bin' >> /usr/bin/zeek-ci-env ENV BASH_ENV="/usr/bin/zeek-ci-env" \ ENV="/usr/bin/zeek-ci-env" \ diff --git a/ci/debian-10/Dockerfile b/ci/debian-10/Dockerfile index a0f05b6f6a..77d0c7deec 100644 --- a/ci/debian-10/Dockerfile +++ b/ci/debian-10/Dockerfile @@ -2,9 +2,11 @@ FROM debian:10 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" + RUN apt-get update && apt-get -y install \ git \ - cmake \ make \ gcc \ g++ \ @@ -23,6 +25,11 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ xz-utils \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Install a recent CMake to build Spicy. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 \ -RUN pip3 install junit2html + && pip3 install junit2html + +ENV PATH "${CMAKE_DIR}/bin:${PATH}" diff --git a/ci/debian-9-32bit/Dockerfile b/ci/debian-9-32bit/Dockerfile index a0078c3c30..6f63995a8d 100644 --- a/ci/debian-9-32bit/Dockerfile +++ b/ci/debian-9-32bit/Dockerfile @@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 ENV DOCKERFILE_VERSION 221001 +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" +ENV PATH "${CMAKE_DIR}/bin:${PATH}" + RUN apt-get update && apt-get -y install \ git \ - cmake \ + clang-11 \ make \ - gcc \ - g++ \ flex \ bison \ + libc++-11-dev \ + libc++abi-11-dev \ libpcap-dev \ libssl-dev \ python3 \ @@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ xz-utils \ - clang-7 \ - libc++-7-dev \ - libc++abi-7-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Recent CMake. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 -RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100 -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100 +RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-11 100 +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-11 100 RUN pip3 install junit2html diff --git a/ci/debian-9/Dockerfile b/ci/debian-9/Dockerfile index 1087378337..27b16db3b0 100644 --- a/ci/debian-9/Dockerfile +++ b/ci/debian-9/Dockerfile @@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 ENV DOCKERFILE_VERSION 221001 +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" +ENV PATH "${CMAKE_DIR}/bin:${PATH}" + RUN apt-get update && apt-get -y install \ git \ - cmake \ + clang-11 \ make \ - gcc \ - g++ \ flex \ bison \ + libc++-11-dev \ + libc++abi-11-dev \ libpcap-dev \ libssl-dev \ python3 \ @@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ xz-utils \ - clang-7 \ - libc++-7-dev \ - libc++abi-7-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Recent CMake. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 RUN pip3 install junit2html -ENV CC=/usr/bin/clang-7 -ENV CXX=/usr/bin/clang++-7 +ENV CC=/usr/bin/clang-11 +ENV CXX=/usr/bin/clang++-11 ENV CXXFLAGS=-stdlib=libc++ diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index 8a51a49461..0eb6817449 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -6,8 +6,12 @@ set -e set -x env ASSUME_ALWAYS_YES=YES pkg bootstrap -pkg install -y bash git cmake swig bison python3 base64 +pkg install -y bash git cmake swig bison python3 base64 flex pkg upgrade -y curl pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")') pkg install -y $pyver-sqlite3 $pyver-pip pip install junit2html + +# Spicy detects whether it is run from build directory via `/proc`. +echo "proc /proc procfs rw,noauto 0 0" >>/etc/fstab +mount /proc diff --git a/ci/macos/prepare.sh b/ci/macos/prepare.sh index 5e1efa0eef..98a772aed9 100755 --- a/ci/macos/prepare.sh +++ b/ci/macos/prepare.sh @@ -7,4 +7,4 @@ set -x brew update brew upgrade cmake openssl@1.1 -brew install swig bison +brew install swig bison flex diff --git a/ci/opensuse-leap-15.3/Dockerfile b/ci/opensuse-leap-15.3/Dockerfile index 69f51c50f2..1ffdbfa1f5 100644 --- a/ci/opensuse-leap-15.3/Dockerfile +++ b/ci/opensuse-leap-15.3/Dockerfile @@ -1,10 +1,12 @@ FROM opensuse/leap:15.3 -RUN zypper in -y \ +RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.2:Update/standard/openSUSE:Leap:15.2:Update.repo \ + && zypper refresh \ + && zypper in -y \ cmake \ make \ - gcc \ - gcc-c++ \ + gcc9 \ + gcc9-c++ \ python3 \ python3-devel \ flex \ @@ -21,5 +23,7 @@ RUN zypper in -y \ tar \ && rm -rf /var/cache/zypp - RUN pip3 install junit2html + +ENV CXX g++-9 +ENV CC gcc-9 diff --git a/ci/ubuntu-18.04/Dockerfile b/ci/ubuntu-18.04/Dockerfile index ed5ecb3b8e..cb1dec0d5f 100644 --- a/ci/ubuntu-18.04/Dockerfile +++ b/ci/ubuntu-18.04/Dockerfile @@ -2,12 +2,14 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" +ENV PATH "${CMAKE_DIR}/bin:${PATH}" + RUN apt-get update && apt-get -y install \ git \ - cmake \ + clang-10 \ make \ - gcc \ - g++ \ flex \ bison \ libpcap-dev \ @@ -27,7 +29,13 @@ RUN apt-get update && apt-get -y install \ ruby \ bc \ lcov \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Recent CMake. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 + +ENV CC=/usr/bin/clang-10 +ENV CXX=/usr/bin/clang++-10 RUN pip3 install junit2html RUN gem install coveralls-lcov diff --git a/configure b/configure index 0565fd1dff..56adde85cb 100755 --- a/configure +++ b/configure @@ -71,6 +71,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-btest-pcaps don't install Zeek's BTest input pcaps --disable-cpp-tests don't build Zeek's C++ unit tests --disable-python don't try to build python bindings for Broker + --disable-spicy don't include Spicy --disable-zeekctl don't install ZeekControl --disable-zkg don't install zkg @@ -101,6 +102,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-perftools=PATH path to Google Perftools install root --with-python-inc=PATH path to Python headers --with-python-lib=PATH path to libpython + --with-spicy=PATH path to Spicy install root + --with-spicy-plugin=PATH path to Spicy plugin source tree --with-swig=PATH path to SWIG executable Packaging Options (for developers): @@ -313,6 +316,9 @@ while [ $# -ne 0 ]; do --disable-python) append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true ;; + --disable-spicy) + append_cache_entry DISABLE_SPICY BOOL true + ;; --disable-zeekctl) append_cache_entry INSTALL_ZEEKCTL BOOL false ;; @@ -375,6 +381,12 @@ while [ $# -ne 0 ]; do --with-python-lib=*) append_cache_entry PYTHON_LIBRARY PATH $optarg ;; + --with-spicy=*) + append_cache_entry SPICY_ROOT PATH $optarg + ;; + --with-spicy-plugin=*) + append_cache_entry SPICY_PLUGIN_PATH PATH $optarg + ;; --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; diff --git a/docker/Dockerfile b/docker/Dockerfile index 5957575213..89db556e72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,8 @@ RUN apt-get -q update \ flex \ g++ \ gcc \ + libfl2 \ + libfl-dev \ libmaxminddb-dev \ libpcap-dev \ libssl-dev \ diff --git a/testing/btest/Baseline/broker.remote_event/recv.recv.out b/testing/btest/Baseline/broker.remote_event/recv.recv.out index c13ab4ea41..dacc55938b 100644 --- a/testing/btest/Baseline/broker.remote_event/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event/recv.recv.out @@ -10,4 +10,4 @@ is_remote should be T, and is, T receiver got ping: my-message, 4 is_remote should be T, and is, T receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/broker.remote_event_any/recv.recv.out b/testing/btest/Baseline/broker.remote_event_any/recv.recv.out index c13ab4ea41..dacc55938b 100644 --- a/testing/btest/Baseline/broker.remote_event_any/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event_any/recv.recv.out @@ -10,4 +10,4 @@ is_remote should be T, and is, T receiver got ping: my-message, 4 is_remote should be T, and is, T receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out b/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out index 7a0dc495f2..b25425641f 100644 --- a/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out @@ -5,4 +5,4 @@ receiver got ping: my-message, 2 receiver got ping: my-message, 3 receiver got ping: my-message, 4 receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr index dbc6d65da1..808ae449a1 100644 --- a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr +++ b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr @@ -7,4 +7,5 @@ warning in , line 1: event handler never invoked: SupervisorControl::des warning in , line 1: event handler never invoked: SupervisorControl::restart_request warning in , line 1: event handler never invoked: SupervisorControl::status_request warning in , line 1: event handler never invoked: SupervisorControl::stop_request +warning in , line 1: event handler never invoked: spicy_analyzer_for_mime_type warning in , line 1: event handler never invoked: this_is_never_used diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index b53f471b1e..5ce56d1f46 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -243,8 +243,18 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek build/scripts/builtin-plugins/__preload__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek build/scripts/builtin-plugins/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek + scripts/base/misc/version.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek + scripts/base/utils/strings.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 38ef5af526..1a5ff60c6b 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -243,6 +243,9 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek scripts/base/init-default.zeek scripts/base/utils/active-http.zeek scripts/base/utils/exec.zeek @@ -443,6 +446,10 @@ scripts/base/init-default.zeek scripts/base/misc/find-filtered-trace.zeek scripts/base/misc/version.zeek build/scripts/builtin-plugins/__preload__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek build/scripts/builtin-plugins/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek + build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/default.zeek scripts/policy/misc/loaded-scripts.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 598fc76f0e..7e8ed9f6ea 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -815,6 +815,9 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> -1 @@ -830,9 +833,6 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./acld, <...>/acld.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./addrs, <...>/addrs.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> -1 @@ -936,8 +936,8 @@ 0.000000 MetaHookPost LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1 -0.000000 MetaHookPost LoadFile(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/bif, <...>/bif) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> -1 @@ -1199,6 +1199,9 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, ) @@ -1214,9 +1217,6 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./acld, <...>/acld.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./addrs, <...>/addrs.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) -> (-1, ) @@ -1320,8 +1320,8 @@ 0.000000 MetaHookPost LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/bif, <...>/bif) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, ) @@ -2293,6 +2293,9 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) @@ -2308,9 +2311,6 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek) -0.000000 MetaHookPre LoadFile(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./acld, <...>/acld.zeek) 0.000000 MetaHookPre LoadFile(0, ./addrs, <...>/addrs.zeek) 0.000000 MetaHookPre LoadFile(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) @@ -2414,8 +2414,8 @@ 0.000000 MetaHookPre LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) 0.000000 MetaHookPre LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) 0.000000 MetaHookPre LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -0.000000 MetaHookPre LoadFile(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -0.000000 MetaHookPre LoadFile(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) +0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) +0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, base/bif, <...>/bif) 0.000000 MetaHookPre LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) @@ -2677,6 +2677,9 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) @@ -2692,9 +2695,6 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.ocsp_events.bif.zeek, <...>/Zeek_X509.ocsp_events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_X509.types.bif.zeek, <...>/Zeek_X509.types.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_XMPP.events.bif.zeek, <...>/Zeek_XMPP.events.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./_Zeek_Spicy.consts.bif.zeek, <...>/_Zeek_Spicy.consts.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./_Zeek_Spicy.events.bif.zeek, <...>/_Zeek_Spicy.events.bif.zeek) -0.000000 MetaHookPre LoadFileExtended(0, ./_Zeek_Spicy.functions.bif.zeek, <...>/_Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./acld, <...>/acld.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./addrs, <...>/addrs.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./analyzer.bif.zeek, <...>/analyzer.bif.zeek) @@ -2798,8 +2798,8 @@ 0.000000 MetaHookPre LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) 0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) 0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -0.000000 MetaHookPre LoadFileExtended(0, _Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -0.000000 MetaHookPre LoadFileExtended(0, _Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/bif, <...>/bif) 0.000000 MetaHookPre LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) @@ -3770,6 +3770,9 @@ 0.000000 | HookLoadFile ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek +0.000000 | HookLoadFile ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek +0.000000 | HookLoadFile ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek +0.000000 | HookLoadFile ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek @@ -3785,9 +3788,6 @@ 0.000000 | HookLoadFile ./Zeek_X509.ocsp_events.bif.zeek <...>/Zeek_X509.ocsp_events.bif.zeek 0.000000 | HookLoadFile ./Zeek_X509.types.bif.zeek <...>/Zeek_X509.types.bif.zeek 0.000000 | HookLoadFile ./Zeek_XMPP.events.bif.zeek <...>/Zeek_XMPP.events.bif.zeek -0.000000 | HookLoadFile ./_Zeek_Spicy.consts.bif.zeek <...>/_Zeek_Spicy.consts.bif.zeek -0.000000 | HookLoadFile ./_Zeek_Spicy.events.bif.zeek <...>/_Zeek_Spicy.events.bif.zeek -0.000000 | HookLoadFile ./_Zeek_Spicy.functions.bif.zeek <...>/_Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFile ./acld <...>/acld.zeek 0.000000 | HookLoadFile ./addrs <...>/addrs.zeek 0.000000 | HookLoadFile ./analyzer.bif.zeek <...>/analyzer.bif.zeek @@ -3903,8 +3903,8 @@ 0.000000 | HookLoadFile <...>/hooks.zeek <...>/hooks.zeek 0.000000 | HookLoadFile Zeek<...>/bare.zeek <...>/bare.zeek 0.000000 | HookLoadFile Zeek<...>/default.zeek <...>/default.zeek -0.000000 | HookLoadFile _Zeek_Spicy/__load__.zeek <...>/__load__.zeek -0.000000 | HookLoadFile _Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek +0.000000 | HookLoadFile Zeek_Spicy/__load__.zeek <...>/__load__.zeek +0.000000 | HookLoadFile Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile base/bif <...>/bif 0.000000 | HookLoadFile base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFile base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek @@ -4154,6 +4154,9 @@ 0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek @@ -4169,9 +4172,6 @@ 0.000000 | HookLoadFileExtended ./Zeek_X509.ocsp_events.bif.zeek <...>/Zeek_X509.ocsp_events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_X509.types.bif.zeek <...>/Zeek_X509.types.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_XMPP.events.bif.zeek <...>/Zeek_XMPP.events.bif.zeek -0.000000 | HookLoadFileExtended ./_Zeek_Spicy.consts.bif.zeek <...>/_Zeek_Spicy.consts.bif.zeek -0.000000 | HookLoadFileExtended ./_Zeek_Spicy.events.bif.zeek <...>/_Zeek_Spicy.events.bif.zeek -0.000000 | HookLoadFileExtended ./_Zeek_Spicy.functions.bif.zeek <...>/_Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFileExtended ./acld <...>/acld.zeek 0.000000 | HookLoadFileExtended ./addrs <...>/addrs.zeek 0.000000 | HookLoadFileExtended ./analyzer.bif.zeek <...>/analyzer.bif.zeek @@ -4287,8 +4287,8 @@ 0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek 0.000000 | HookLoadFileExtended Zeek<...>/bare.zeek <...>/bare.zeek 0.000000 | HookLoadFileExtended Zeek<...>/default.zeek <...>/default.zeek -0.000000 | HookLoadFileExtended _Zeek_Spicy/__load__.zeek <...>/__load__.zeek -0.000000 | HookLoadFileExtended _Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek +0.000000 | HookLoadFileExtended Zeek_Spicy/__load__.zeek <...>/__load__.zeek +0.000000 | HookLoadFileExtended Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended base/bif <...>/bif 0.000000 | HookLoadFileExtended base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr index e97cb1bad7..9422ad94ac 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr @@ -1,2 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension +XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension diff --git a/testing/btest/broker/remote_event.zeek b/testing/btest/broker/remote_event.zeek index ddc4092841..611f9067ac 100644 --- a/testing/btest/broker/remote_event.zeek +++ b/testing/btest/broker/remote_event.zeek @@ -1,5 +1,7 @@ # @TEST-GROUP: broker # +# @TEST-REQUIRES: $SCRIPTS/have-spicy # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not. +# # @TEST-PORT: BROKER_PORT # # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out" diff --git a/testing/btest/broker/remote_event_any.zeek b/testing/btest/broker/remote_event_any.zeek index 2a290376ca..c096318717 100644 --- a/testing/btest/broker/remote_event_any.zeek +++ b/testing/btest/broker/remote_event_any.zeek @@ -1,5 +1,7 @@ # @TEST-GROUP: broker # +# @TEST-REQUIRES: $SCRIPTS/have-spicy # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not. +# # @TEST-PORT: BROKER_PORT # # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out" diff --git a/testing/btest/broker/remote_event_ssl_auth.zeek b/testing/btest/broker/remote_event_ssl_auth.zeek index d3882ab2b1..63acd24683 100644 --- a/testing/btest/broker/remote_event_ssl_auth.zeek +++ b/testing/btest/broker/remote_event_ssl_auth.zeek @@ -1,5 +1,7 @@ # @TEST-GROUP: broker # +# @TEST-REQUIRES: $SCRIPTS/have-spicy # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not. +# # @TEST-PORT: BROKER_PORT # # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out" diff --git a/testing/btest/core/check-unused-event-handlers.test b/testing/btest/core/check-unused-event-handlers.test index 7d3a581d6c..742a07554c 100644 --- a/testing/btest/core/check-unused-event-handlers.test +++ b/testing/btest/core/check-unused-event-handlers.test @@ -1,4 +1,5 @@ # This test should print a warning that the event handler is never invoked. +# @TEST-REQUIRES: $SCRIPTS/have-spicy # This test logs uninvoked event handlers, so disable it if Spicy and its plugin is unavailable. # @TEST-EXEC: zeek -b %INPUT check_for_unused_event_handlers=T # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr diff --git a/testing/btest/coverage/bare-load-baseline.test b/testing/btest/coverage/bare-load-baseline.test index 94fdb04b04..c5cd2d741c 100644 --- a/testing/btest/coverage/bare-load-baseline.test +++ b/testing/btest/coverage/bare-load-baseline.test @@ -7,6 +7,7 @@ # prefix to make the test work everywhere. That's what the sed magic # below does. Don't ask. :-) +# @TEST-REQUIRES: $SCRIPTS/have-spicy # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable. # @TEST-EXEC: zeek -b misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix diff --git a/testing/btest/coverage/default-load-baseline.test b/testing/btest/coverage/default-load-baseline.test index df13444ad7..43c2c5bdec 100644 --- a/testing/btest/coverage/default-load-baseline.test +++ b/testing/btest/coverage/default-load-baseline.test @@ -7,6 +7,7 @@ # prefix to make the test work everywhere. That's what the sed magic # below does. Don't ask. :-) +# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config # @TEST-EXEC: zeek misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix diff --git a/testing/btest/plugins/hooks.zeek b/testing/btest/plugins/hooks.zeek index b87b542957..288e50f00e 100644 --- a/testing/btest/plugins/hooks.zeek +++ b/testing/btest/plugins/hooks.zeek @@ -1,4 +1,5 @@ # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" +# @TEST-REQUIRES: ${SCRIPTS}/have-spicy # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable. # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* . # @TEST-EXEC: ./configure --zeek-dist=${DIST} && make diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek b/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek index 87a2caecbc..fd74e42002 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek @@ -1,3 +1,4 @@ +# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config # @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff .stderr