diff --git a/.cirrus.yml b/.cirrus.yml index bfc053f8db..bba95d8282 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,15 +8,16 @@ cpus: &CPUS 4 btest_jobs: &BTEST_JOBS 4 btest_retries: &BTEST_RETRIES 2 -memory: &MEMORY 8GB +memory: &MEMORY 12GB config: &CONFIG --build-type=release --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install static_config: &STATIC_CONFIG --build-type=release --disable-broker-tests --enable-static-broker --enable-static-binpac --prefix=$CIRRUS_WORKING_DIR/install -asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage -ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers -tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers +asan_sanitizer_config: &ASAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=address --enable-fuzzers --enable-coverage --disable-spicy +ubsan_sanitizer_config: &UBSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=undefined --enable-fuzzers --disable-spicy +tsan_sanitizer_config: &TSAN_SANITIZER_CONFIG --build-type=debug --disable-broker-tests --sanitizers=thread --enable-fuzzers --disable-spicy mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-mobile-ipv6 --disable-broker-tests --prefix=$CIRRUS_WORKING_DIR/install openssl30_config: &OPENSSL30_CONFIG --build-type=release --disable-broker-tests --with-openssl=/opt/openssl --prefix=$CIRRUS_WORKING_DIR/install +homebrew_flex_bison_config: &HOMEBREW_FLEX_BISON_CONFIG --with-flex=/usr/local/opt/flex/bin/flex --with-bison=/usr/local/opt/bison/bin/bison resources_template: &RESOURCES_TEMPLATE cpu: *CPUS @@ -216,6 +217,8 @@ macos_monterey_task: prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE << : *MACOS_RESOURCES_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG macos_big_sur_task: macos_instance: @@ -223,6 +226,8 @@ macos_big_sur_task: prepare_script: ./ci/macos/prepare.sh << : *CI_TEMPLATE << : *MACOS_RESOURCES_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *HOMEBREW_FLEX_BISON_CONFIG # FreeBSD EOL timelines: https://www.freebsd.org/security/security.html#sup freebsd14_task: diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 6159ddc528..8ddf861401 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -43,6 +43,8 @@ jobs: g++ \ flex \ bison \ + libfl2 \ + libfl-dev \ libpcap-dev \ libssl-dev \ python3 \ @@ -60,7 +62,7 @@ jobs: sudo pip3 install -r doc/requirements.txt - name: Configure - run: ./configure --disable-broker-tests --disable-cpp-tests + run: ./configure --disable-broker-tests --disable-cpp-tests --disable-spicy - name: Build run: cd build && make -j $(nproc) diff --git a/.gitmodules b/.gitmodules index cb20683187..d84fc4118a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -58,3 +58,9 @@ [submodule "auxil/out_ptr"] path = auxil/out_ptr url = https://github.com/soasis/out_ptr.git +[submodule "auxil/spicy-plugin"] + path = auxil/spicy-plugin + url = https://github.com/zeek/spicy-plugin +[submodule "auxil/spicy"] + path = auxil/spicy/spicy + url = https://github.com/zeek/spicy diff --git a/.lgtm.yml b/.lgtm.yml index 15857c99e2..13772c6e87 100644 --- a/.lgtm.yml +++ b/.lgtm.yml @@ -7,6 +7,8 @@ path_classifiers: - "auxil/libkqueue/" - "auxil/highwayhash/" - "auxil/rapidjson/" + - "auxil/spicy" + - "auxil/spicy-plugin" - "src/3rdparty/" # Filter out alerts that aren't concerning. @@ -38,7 +40,9 @@ extraction: configure: command: - - ./configure --build-type=debug --generator=Ninja + # Disabling Spicy in LGTM build since it seems to lead to a LGTM + # performance edge case where analysis does not finish in 4h. + - ./configure --build-type=debug --generator=Ninja --disable-spicy index: build_command: diff --git a/CHANGES b/CHANGES index f64886e89c..536d937c54 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,49 @@ +5.0.0-dev.440 | 2022-05-17 08:50:15 -0700 + + * Add NEWS item for new requirements due to default-enabled Spicy support. (Benjamin Bannier, Corelight) + + * Include headers in binary dir before other headers. (Benjamin Bannier, Corelight) + + If e.g., optional dependencies are installed in the same prefix as a + previous Zeek installation, the build would have previously found + `zeek-config.h` from the external prefix instead of from the current + build. + + With this patch we now include the current binary dir before any + previously configured include directories (here in particular: optional + dependencies). With that the correct `zeek-config.h` is found. + + * Increase memory in CI. (Benjamin Bannier, Corelight) + + Spicy requires substantial amounts of RAM to compile. In this patch we + bump the amount of RAM available in CI so a build including Spicy can be + performed. + + * Add tests for bundled Spicy infrastructure. (Benjamin Bannier, Corelight) + + * Include spicy in build. (Benjamin Bannier, Corelight) + + * Add `bare_mode` bif. (Benjamin Bannier, Corelight) + + * Change test so included plugins can load notice framework. (Benjamin Bannier, Corelight) + + * Do not log function arguments in test. (Benjamin Bannier, Corelight) + + This was excercising hooks to validate that they were called. For that + it uses pretty verbose logging. Since version numbers are not stable and + hard to canonify this script already unloads the version module (there + is e.g., code in `base/misc/version` which splits the version strings + and works on single components which might appear like pretty ordinary + floating point numbers in the log). + + This test however worked under the assumption that nothing in bare mode + loads `base/misc/version` which is hard to guarantee, especially + considering that plugins can be embedded and might load that module + themself. + + With this patch we now make the logging slightly less verbose so that + function call arguments are not logged anymore. + 5.0.0-dev.431 | 2022-05-16 17:00:24 -0700 * Store location during Ascii input parsing for use in error messages (Tim Wojtulewicz, Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5f7df8ffdb..8f347d8266 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,6 +1,6 @@ # When changing the minimum version here, also adapt # auxil/zeek-aux/plugin-support/skeleton/CMakeLists.txt -cmake_minimum_required(VERSION 3.5...3.18 FATAL_ERROR) +cmake_minimum_required(VERSION 3.15.0 FATAL_ERROR) project(Zeek C CXX) @@ -73,12 +73,16 @@ execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.sh "export ZEEKPATH=`${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n" "export ZEEK_PLUGIN_PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n" - "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") + "export PATH=\"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n" + "export SPICY_PATH=\"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n" + "export HILTI_CXX_INCLUDE_DIRS=\"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n") file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev.csh "setenv ZEEKPATH `${CMAKE_CURRENT_BINARY_DIR}/zeek-path-dev`\n" "setenv ZEEK_PLUGIN_PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":${ZEEK_PLUGIN_PATH}\n" - "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":$PATH\n") + "setenv PATH \"${CMAKE_CURRENT_BINARY_DIR}/src\":\"${CMAKE_CURRENT_BINARY_DIR}/auxil/spicy/spicy/bin\":\"${CMAKE_CURRENT_BINARY_DIR}/src/builtin-plugins/spicy-plugin/bin/\":$PATH\n" + "setenv SPICY_PATH \"`${CMAKE_CURRENT_BINARY_DIR}/spicy-path`\"\n" + "setenv HILTI_CXX_INCLUDE_DIRS \"`${CMAKE_CURRENT_BINARY_DIR}/hilti-cxx-include-dirs`\"\n") file(STRINGS "${CMAKE_CURRENT_SOURCE_DIR}/VERSION" VERSION LIMIT_COUNT 1) execute_process(COMMAND grep "^#define *BRO_PLUGIN_API_VERSION" @@ -374,6 +378,54 @@ else () set(broker_includes ${CMAKE_CURRENT_SOURCE_DIR}/auxil/broker/include ${CMAKE_CURRENT_BINARY_DIR}/auxil/broker/include) endif () +if ( NOT DISABLE_SPICY ) + if ( NOT Spicy_ROOT ) + add_subdirectory(auxil/spicy) + + # Set variables used by the spicy-plugin build since we are building Spicy + # as part of Zeek so spicy-plugin cannot use `spicy-config` at configure + # time to set these. + set(SPICY_CONFIG "") + set(HAVE_SPICY "YES") + set(SPICY_HAVE_TOOLCHAIN "YES") + set(SPICY_INCLUDE_DIRS_RUNTIME + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/runtime/include + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/runtime/include + ${PROJECT_BINARY_DIR}/auxil/spicy/spicy/include) + set(SPICY_INCLUDE_DIRS_TOOLCHAIN + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/hilti/toolchain/include + ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy/spicy/toolchain/include) + set(SPICY_LIBRARY spicy) + + # Needed only for logging from CMake configure phase. + get_directory_property( + SPICY_VERSION + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION SPICY_VERSION) + get_directory_property( + SPICY_VERSION_NUMBER + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION SPICY_VERSION_NUMBER) + get_directory_property( + SPICY_PREFIX + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION CMAKE_INSTALL_PREFIX) + get_directory_property( + SPICY_BUILD_MODE + DIRECTORY ${PROJECT_SOURCE_DIR}/auxil/spicy/spicy + DEFINITION CMAKE_BUILD_TYPE) + set(SPICYC "") + endif () + + if ( NOT SPICY_PLUGIN_PATH ) + set(SPICY_PLUGIN_PATH ${CMAKE_SOURCE_DIR}/auxil/spicy-plugin) + endif () + + # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/auxil/spicy-plugin) + # set(ZEEK_SPICY_PLUGIN_BUILD_LIBDIR ${CMAKE_BINARY_DIR}/src/builtin-plugins/spicy-plugin) + string(APPEND ZEEK_INCLUDE_PLUGINS ";${SPICY_PLUGIN_PATH}") +endif () + include_directories(BEFORE ${PCAP_INCLUDE_DIR} ${BIND_INCLUDE_DIR} @@ -502,7 +554,7 @@ endif () string(TOLOWER ${CMAKE_BUILD_TYPE} CMAKE_BUILD_TYPE_LOWER) configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.h.in ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h) -include_directories(${CMAKE_CURRENT_BINARY_DIR}) +include_directories(BEFORE ${CMAKE_CURRENT_BINARY_DIR}) install(FILES ${CMAKE_CURRENT_BINARY_DIR}/zeek-config.h DESTINATION include/zeek) execute_process(COMMAND "${CMAKE_COMMAND}" -E create_symlink "." @@ -616,6 +668,53 @@ CheckOptionalBuildSources(auxil/zeek-aux Zeek-Aux INSTALL_AUX_TOOLS) CheckOptionalBuildSources(auxil/zeek-archiver ZeekArchiver INSTALL_ZEEK_ARCHIVER) CheckOptionalBuildSources(auxil/zeek-client ZeekClient INSTALL_ZEEK_CLIENT) +if ( NOT DISABLE_SPICY ) + if ( NOT Spicy_ROOT ) + list(APPEND _spicy_rt_libs spicy-rt hilti-rt) + + # Use the debug runtime libraries if we are building a debug Zeek. + if ( SPICY_BUILD_MODE STREQUAL "Debug" ) + list(TRANSFORM _spicy_rt_libs APPEND "-debug") + endif () + + # If we build spicy-plugin into Zeek we also need to build the Spicy + # runtime dependencies into Zeek. Since no matter how Spicy itself was + # linked this is always a static library, link the object files so we + # get all symbols and can can resolve all potential dependencies of + # HLTO files at runtime. + # + # TODO(bbannier): Conceptually cleaner would be linking the runtime + # objects into spicy-plugin and then inherit that up to zeek, but it + # seems this does not work reliable (due incomplete support for object + # libraries in CMake?). + list(TRANSFORM _spicy_rt_libs APPEND "-objects") + target_link_libraries(zeek ${_spicy_rt_libs}) + + # Make sure we build targets of spicy-plugin after the `spicy` target. + add_dependencies(plugin-Zeek-Spicy spicy) + add_dependencies(spicyz spicy) + + # Spicy JIT relies on the path of the current executable to figure out + # whether it is run from a build or an install tree. This assumption + # gets broken for e.g., `spicyz` when running from the build tree + # (probably fixable), and also when JIT'ing directly from a `zeek` + # invocation (much harder to fix). Instead we generate shell + # definitions to support running and using Spicy or spicy-plugin + # functionality in the build tree, including JIT'ing directly from + # Zeek. + configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/spicy-path.in ${CMAKE_BINARY_DIR}/spicy-path @ONLY) + configure_file(${CMAKE_SOURCE_DIR}/auxil/spicy/hilti-cxx-include-dirs.in ${CMAKE_BINARY_DIR}/hilti-cxx-include-dirs @ONLY) + + # Also install spicy-plugin's CMake files into Zeek's global `cmake/` folder. + # + # NOTE: We do not install spicy-plugin's `FindZeek.cmake` since another + # version of this file is already provided by Zeek. + install(FILES auxil/spicy-plugin/cmake/ZeekSpicyAnalyzerSupport.cmake + auxil/spicy-plugin/cmake/FindSpicy.cmake + DESTINATION share/zeek/cmake) + endif () +endif () + ######################################################################## ## Packaging Setup diff --git a/NEWS b/NEWS index fa1f7f3711..f6ee1a2692 100644 --- a/NEWS +++ b/NEWS @@ -9,6 +9,12 @@ Zeek 5.0.0 Breaking Changes ---------------- +- Zeek now requires at least CMake version 3.15.0. + +- If Zeek is configured with support for included Spicy (the default) we now + require at least Flex version 2.6 and its development headers, at least Bison + version 3.3, and GCC version 8.3 or Clang version 9.0 or higher. + New Functionality ----------------- @@ -25,6 +31,10 @@ New Functionality and (8) how much of that memory was due to its children. Note that profiling is expensive and may not be suitable for execution on live traffic. +- Zeek now includes support for building Spicy and spicy-plugin as part of + Zeek. This feature is enabled by default, and can be turned off by passing + the ``--disable-spicy`` flag to ``./configure``. + - Zeek now supports generation and replay of event traces via the new ``--event-trace`` / ``-E`` command-line options. For details, see: https://docs.zeek.org/en/master/quickstart.html#tracing-events diff --git a/VERSION b/VERSION index 9518106df8..f9acf8c972 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -5.0.0-dev.431 +5.0.0-dev.440 diff --git a/auxil/spicy-plugin b/auxil/spicy-plugin new file mode 160000 index 0000000000..0435d34acf --- /dev/null +++ b/auxil/spicy-plugin @@ -0,0 +1 @@ +Subproject commit 0435d34acf92ff07558d72524fb2669eca78b1c9 diff --git a/auxil/spicy/CMakeLists.txt b/auxil/spicy/CMakeLists.txt new file mode 100644 index 0000000000..050ee9753e --- /dev/null +++ b/auxil/spicy/CMakeLists.txt @@ -0,0 +1,38 @@ +# Spicy needs the full prefix for Flex and Bison while Zeek captures only the +# paths to the executables. Derive the prefixes from the binary paths under the +# assumption that their bindir is under their prefix (which also implies that +# one such prefix even exists). +if ( NOT FLEX_EXECUTABLE ) + find_package(FLEX REQUIRED) +endif () +get_filename_component(dir ${FLEX_EXECUTABLE} DIRECTORY ABSOLUTE) +set(FLEX_ROOT ${dir}/..) + +if ( NOT BISON_EXECUTABLE ) + find_package(BISON REQUIRED) +endif () +get_filename_component(dir ${BISON_EXECUTABLE} DIRECTORY ABSOLUTE) +set(BISON_ROOT ${dir}/..) + +# We currently only support building against statically linked Spicy libraries. +# This is due to global state held in the Spicy and HILTI runtime libraries. If +# we would like Spicy dynamically this global state would end up the the final +# binary once via the static link of the runtime libraries into Zeek (needed to +# make sure we can e.g., load arbitrary HLTO files), and once as a link +# dependency of spicy-plugin. This can lead to e.g. consistency issues during +# execution or the problem during teardown where the same global might be +# cleaned up multiple times. +set(BUILD_SHARED_LIBS OFF) + +# Spicy uses slightly less strict warnings than Zeek proper. Mute a few warnings for Spicy. +# NOTE: Compiler flags are inherited down the directory tree, so in order to +# set these flags we do need a customizable subdirectory above the Spicy +# sources. +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wno-missing-braces") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-vla") + +# The script generating precompiled headers for Spicy expects a different build +# system layout than provided for a bundled Spicy, disable it. +set(HILTI_DEV_PRECOMPILE_HEADERS OFF) + +add_subdirectory(spicy) diff --git a/auxil/spicy/hilti-cxx-include-dirs.in b/auxil/spicy/hilti-cxx-include-dirs.in new file mode 100755 index 0000000000..06ae06a806 --- /dev/null +++ b/auxil/spicy/hilti-cxx-include-dirs.in @@ -0,0 +1,19 @@ +#!/bin/sh + +# Computes CXX flags for Spicy compilation in the build tree. The value +# computed by this script can be injected into Spicy JIT via the environment +# variable `HILTI_CXX_INCLUDE_DIRS`. + +# Paths to support compiling Spicy parsers in the build tree. +PATHS=@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/runtime/include +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/runtime/include +PATHS=$PATHS:@CMAKE_BINARY_DIR@/auxil/spicy/spicy/include + +# Paths to support compiling against a build tree Zeek. +PATHS=$PATHS:@CMAKE_BINARY_DIR@ +PATHS=$PATHS:@CMAKE_BINARY_DIR@/src +PATHS=$PATHS:@CMAKE_BINARY_DIR@/src/include +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/src +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/broker/include/ + +echo $PATHS diff --git a/auxil/spicy/spicy b/auxil/spicy/spicy new file mode 160000 index 0000000000..ef859d07a1 --- /dev/null +++ b/auxil/spicy/spicy @@ -0,0 +1 @@ +Subproject commit ef859d07a137bae77ffd8f04908dbdbb5d8649ef diff --git a/auxil/spicy/spicy-path.in b/auxil/spicy/spicy-path.in new file mode 100755 index 0000000000..3fab168230 --- /dev/null +++ b/auxil/spicy/spicy-path.in @@ -0,0 +1,10 @@ +#!/bin/sh + +# Computes a value for the environment variable `SPICY_PATH` for execution of +# Spicy tooling in the build tree. + +PATHS=. +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/spicy/lib +PATHS=$PATHS:@CMAKE_SOURCE_DIR@/auxil/spicy/spicy/hilti/lib + +echo $PATHS diff --git a/ci/alpine/Dockerfile b/ci/alpine/Dockerfile index be93e0c3d0..dcbbbe9146 100644 --- a/ci/alpine/Dockerfile +++ b/ci/alpine/Dockerfile @@ -6,7 +6,7 @@ RUN apk add --no-cache \ g++ \ python3 \ python3-dev \ - flex \ + flex-dev \ bison \ libpcap-dev \ openssl-dev \ diff --git a/ci/centos-7/Dockerfile b/ci/centos-7/Dockerfile index 5ab9df2d68..7928da7727 100644 --- a/ci/centos-7/Dockerfile +++ b/ci/centos-7/Dockerfile @@ -1,5 +1,11 @@ FROM centos:7 +ENV FLEX_VERSION=2.6.4 +ENV FLEX_DIR=/opt/flex + +# Disabled lookup of fastest mirror since the list seems to be outdated and no valid mirror can be detected. +RUN sed -i 's/enabled=1/enabled=0/' /etc/yum/pluginconf.d/fastestmirror.conf + # The version of git in the standard repos is 1.8 and CI needs 2.3+ # for the use of GIT_SSH_COMMAND when cloning private repos. RUN yum -y install \ @@ -17,13 +23,13 @@ RUN yum -y install \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ - devtoolset-7 \ + devtoolset-8 \ && yum clean all && rm -rf /var/cache/yum RUN yum -y install \ cmake3 \ + curl \ make \ - flex \ bison \ swig \ openssl \ @@ -38,10 +44,18 @@ RUN yum -y install \ which \ && yum clean all && rm -rf /var/cache/yum +# Install a recent flex for Spicy. +RUN curl -sSL "https://github.com/westes/flex/releases/download/v${FLEX_VERSION}/flex-${FLEX_VERSION}.tar.gz" | tar xzf - -C /tmp \ + && (cd /tmp/flex-${FLEX_VERSION} \ + && ./configure --prefix=${FLEX_DIR} \ + && make -j`nproc` install) \ + && rm -rf /tmp/flex-${FLEX_VERSION} + RUN pip3 install junit2html RUN echo 'unset BASH_ENV PROMPT_COMMAND ENV' > /usr/bin/zeek-ci-env && \ - echo 'source /opt/rh/devtoolset-7/enable' >> /usr/bin/zeek-ci-env + echo 'source /opt/rh/devtoolset-8/enable' >> /usr/bin/zeek-ci-env && \ + echo 'export PATH=${PATH}:${FLEX_DIR}/bin' >> /usr/bin/zeek-ci-env ENV BASH_ENV="/usr/bin/zeek-ci-env" \ ENV="/usr/bin/zeek-ci-env" \ diff --git a/ci/debian-10/Dockerfile b/ci/debian-10/Dockerfile index a0f05b6f6a..77d0c7deec 100644 --- a/ci/debian-10/Dockerfile +++ b/ci/debian-10/Dockerfile @@ -2,9 +2,11 @@ FROM debian:10 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" + RUN apt-get update && apt-get -y install \ git \ - cmake \ make \ gcc \ g++ \ @@ -23,6 +25,11 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ xz-utils \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Install a recent CMake to build Spicy. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 \ -RUN pip3 install junit2html + && pip3 install junit2html + +ENV PATH "${CMAKE_DIR}/bin:${PATH}" diff --git a/ci/debian-9-32bit/Dockerfile b/ci/debian-9-32bit/Dockerfile index a0078c3c30..6f63995a8d 100644 --- a/ci/debian-9-32bit/Dockerfile +++ b/ci/debian-9-32bit/Dockerfile @@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 ENV DOCKERFILE_VERSION 221001 +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" +ENV PATH "${CMAKE_DIR}/bin:${PATH}" + RUN apt-get update && apt-get -y install \ git \ - cmake \ + clang-11 \ make \ - gcc \ - g++ \ flex \ bison \ + libc++-11-dev \ + libc++abi-11-dev \ libpcap-dev \ libssl-dev \ python3 \ @@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ xz-utils \ - clang-7 \ - libc++-7-dev \ - libc++abi-7-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Recent CMake. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 -RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-7 100 -RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-7 100 +RUN update-alternatives --install /usr/bin/cc cc /usr/bin/clang-11 100 +RUN update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++-11 100 RUN pip3 install junit2html diff --git a/ci/debian-9/Dockerfile b/ci/debian-9/Dockerfile index 1087378337..27b16db3b0 100644 --- a/ci/debian-9/Dockerfile +++ b/ci/debian-9/Dockerfile @@ -6,14 +6,18 @@ ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" # https://github.com/cirruslabs/cirrus-ci-docs/issues/544#issuecomment-566066822 ENV DOCKERFILE_VERSION 221001 +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" +ENV PATH "${CMAKE_DIR}/bin:${PATH}" + RUN apt-get update && apt-get -y install \ git \ - cmake \ + clang-11 \ make \ - gcc \ - g++ \ flex \ bison \ + libc++-11-dev \ + libc++abi-11-dev \ libpcap-dev \ libssl-dev \ python3 \ @@ -27,13 +31,13 @@ RUN apt-get update && apt-get -y install \ curl \ wget \ xz-utils \ - clang-7 \ - libc++-7-dev \ - libc++abi-7-dev \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Recent CMake. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 RUN pip3 install junit2html -ENV CC=/usr/bin/clang-7 -ENV CXX=/usr/bin/clang++-7 +ENV CC=/usr/bin/clang-11 +ENV CXX=/usr/bin/clang++-11 ENV CXXFLAGS=-stdlib=libc++ diff --git a/ci/freebsd/prepare.sh b/ci/freebsd/prepare.sh index b899da5fe9..a4bd907afd 100755 --- a/ci/freebsd/prepare.sh +++ b/ci/freebsd/prepare.sh @@ -6,9 +6,13 @@ set -e set -x env ASSUME_ALWAYS_YES=YES pkg bootstrap -pkg install -y bash git cmake swig bison python3 base64 +pkg install -y bash git cmake swig bison python3 base64 flex pkg upgrade -y curl pyver=$(python3 -c 'import sys; print(f"py{sys.version_info[0]}{sys.version_info[1]}")') pkg install -y $pyver-sqlite3 $pyver-pip python -m pip install junit2html + +# Spicy detects whether it is run from build directory via `/proc`. +echo "proc /proc procfs rw,noauto 0 0" >>/etc/fstab +mount /proc diff --git a/ci/macos/prepare.sh b/ci/macos/prepare.sh index 5e1efa0eef..98a772aed9 100755 --- a/ci/macos/prepare.sh +++ b/ci/macos/prepare.sh @@ -7,4 +7,4 @@ set -x brew update brew upgrade cmake openssl@1.1 -brew install swig bison +brew install swig bison flex diff --git a/ci/opensuse-leap-15.3/Dockerfile b/ci/opensuse-leap-15.3/Dockerfile index 69f51c50f2..1ffdbfa1f5 100644 --- a/ci/opensuse-leap-15.3/Dockerfile +++ b/ci/opensuse-leap-15.3/Dockerfile @@ -1,10 +1,12 @@ FROM opensuse/leap:15.3 -RUN zypper in -y \ +RUN zypper addrepo https://download.opensuse.org/repositories/openSUSE:Leap:15.2:Update/standard/openSUSE:Leap:15.2:Update.repo \ + && zypper refresh \ + && zypper in -y \ cmake \ make \ - gcc \ - gcc-c++ \ + gcc9 \ + gcc9-c++ \ python3 \ python3-devel \ flex \ @@ -21,5 +23,7 @@ RUN zypper in -y \ tar \ && rm -rf /var/cache/zypp - RUN pip3 install junit2html + +ENV CXX g++-9 +ENV CC gcc-9 diff --git a/ci/ubuntu-18.04/Dockerfile b/ci/ubuntu-18.04/Dockerfile index ed5ecb3b8e..cb1dec0d5f 100644 --- a/ci/ubuntu-18.04/Dockerfile +++ b/ci/ubuntu-18.04/Dockerfile @@ -2,12 +2,14 @@ FROM ubuntu:18.04 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" +ENV CMAKE_DIR "/opt/cmake" +ENV CMAKE_VERSION "3.19.1" +ENV PATH "${CMAKE_DIR}/bin:${PATH}" + RUN apt-get update && apt-get -y install \ git \ - cmake \ + clang-10 \ make \ - gcc \ - g++ \ flex \ bison \ libpcap-dev \ @@ -27,7 +29,13 @@ RUN apt-get update && apt-get -y install \ ruby \ bc \ lcov \ - && rm -rf /var/lib/apt/lists/* + && rm -rf /var/lib/apt/lists/* \ + # Recent CMake. + && mkdir -p "${CMAKE_DIR}" \ + && curl -sSL "https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz" | tar xzf - -C "${CMAKE_DIR}" --strip-components 1 + +ENV CC=/usr/bin/clang-10 +ENV CXX=/usr/bin/clang++-10 RUN pip3 install junit2html RUN gem install coveralls-lcov diff --git a/configure b/configure index 0565fd1dff..56adde85cb 100755 --- a/configure +++ b/configure @@ -71,6 +71,7 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --disable-btest-pcaps don't install Zeek's BTest input pcaps --disable-cpp-tests don't build Zeek's C++ unit tests --disable-python don't try to build python bindings for Broker + --disable-spicy don't include Spicy --disable-zeekctl don't install ZeekControl --disable-zkg don't install zkg @@ -101,6 +102,8 @@ Usage: $0 [OPTION]... [VAR=VALUE]... --with-perftools=PATH path to Google Perftools install root --with-python-inc=PATH path to Python headers --with-python-lib=PATH path to libpython + --with-spicy=PATH path to Spicy install root + --with-spicy-plugin=PATH path to Spicy plugin source tree --with-swig=PATH path to SWIG executable Packaging Options (for developers): @@ -313,6 +316,9 @@ while [ $# -ne 0 ]; do --disable-python) append_cache_entry DISABLE_PYTHON_BINDINGS BOOL true ;; + --disable-spicy) + append_cache_entry DISABLE_SPICY BOOL true + ;; --disable-zeekctl) append_cache_entry INSTALL_ZEEKCTL BOOL false ;; @@ -375,6 +381,12 @@ while [ $# -ne 0 ]; do --with-python-lib=*) append_cache_entry PYTHON_LIBRARY PATH $optarg ;; + --with-spicy=*) + append_cache_entry SPICY_ROOT PATH $optarg + ;; + --with-spicy-plugin=*) + append_cache_entry SPICY_PLUGIN_PATH PATH $optarg + ;; --with-swig=*) append_cache_entry SWIG_EXECUTABLE PATH $optarg ;; diff --git a/docker/Dockerfile b/docker/Dockerfile index 5957575213..89db556e72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -12,6 +12,8 @@ RUN apt-get -q update \ flex \ g++ \ gcc \ + libfl2 \ + libfl-dev \ libmaxminddb-dev \ libpcap-dev \ libssl-dev \ diff --git a/src/RunState.cc b/src/RunState.cc index 7c24ee18c4..5e44eb8129 100644 --- a/src/RunState.cc +++ b/src/RunState.cc @@ -63,6 +63,7 @@ double current_wallclock = 0.0; double current_pseudo = 0.0; bool zeek_init_done = false; bool time_updated = false; +bool bare_mode = false; RETSIGTYPE watchdog(int /* signo */) { diff --git a/src/RunState.h b/src/RunState.h index 3b9171c483..d3ba66c8a3 100644 --- a/src/RunState.h +++ b/src/RunState.h @@ -65,6 +65,8 @@ extern double current_pseudo; extern bool zeek_init_done; +extern bool bare_mode; // True if Zeek was started in bare mode. + } // namespace detail // Functions to temporarily suspend processing of live input (network packets diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index f682d39118..df8294816d 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -516,6 +516,8 @@ SetupResult setup(int argc, char** argv, Options* zopts) auto options = zopts ? *zopts : parse_cmdline(argc, argv); + run_state::detail::bare_mode = options.bare_mode; + // Set up the global that facilitates access to analysis/optimization // options from deep within some modules. analysis_options = options.analysis_options; diff --git a/src/zeek.bif b/src/zeek.bif index 785f606973..d176d68c71 100644 --- a/src/zeek.bif +++ b/src/zeek.bif @@ -1748,6 +1748,20 @@ function zeek_version%(%): string return zeek::make_intrusive(zeek::zeek_version()); %} +%%{ +namespace zeek::run_state::detail { +extern bool bare_mode; +} // namespace zeek::run_state::detail +%%} + +## Returns whether Zeek was started in bare mode. +## +## Returns: True if Zeek was started in bare mode, false otherwise. +function bare_mode%(%): bool + %{ + return zeek::val_mgr->Bool(zeek::run_state::detail::bare_mode); + %} + ## Converts a record type name to a vector of strings, where each element is ## the name of a record field. Nested records are flattened. ## diff --git a/testing/btest/Baseline/bifs.bare_mode/output b/testing/btest/Baseline/bifs.bare_mode/output new file mode 100644 index 0000000000..1956db8698 --- /dev/null +++ b/testing/btest/Baseline/bifs.bare_mode/output @@ -0,0 +1,3 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +F +T diff --git a/testing/btest/Baseline/broker.remote_event/recv.recv.out b/testing/btest/Baseline/broker.remote_event/recv.recv.out index c13ab4ea41..dacc55938b 100644 --- a/testing/btest/Baseline/broker.remote_event/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event/recv.recv.out @@ -10,4 +10,4 @@ is_remote should be T, and is, T receiver got ping: my-message, 4 is_remote should be T, and is, T receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/broker.remote_event_any/recv.recv.out b/testing/btest/Baseline/broker.remote_event_any/recv.recv.out index c13ab4ea41..dacc55938b 100644 --- a/testing/btest/Baseline/broker.remote_event_any/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event_any/recv.recv.out @@ -10,4 +10,4 @@ is_remote should be T, and is, T receiver got ping: my-message, 4 is_remote should be T, and is, T receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out b/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out index 7a0dc495f2..b25425641f 100644 --- a/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out +++ b/testing/btest/Baseline/broker.remote_event_ssl_auth/recv.recv.out @@ -5,4 +5,4 @@ receiver got ping: my-message, 2 receiver got ping: my-message, 3 receiver got ping: my-message, 4 receiver got ping: my-message, 5 -[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=1, num_ids_incoming=0, num_ids_outgoing=0] +[num_peers=1, num_stores=0, num_pending_queries=0, num_events_incoming=5, num_events_outgoing=4, num_logs_incoming=0, num_logs_outgoing=2, num_ids_incoming=0, num_ids_outgoing=0] diff --git a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr index dbc6d65da1..808ae449a1 100644 --- a/testing/btest/Baseline/core.check-unused-event-handlers/.stderr +++ b/testing/btest/Baseline/core.check-unused-event-handlers/.stderr @@ -7,4 +7,5 @@ warning in , line 1: event handler never invoked: SupervisorControl::des warning in , line 1: event handler never invoked: SupervisorControl::restart_request warning in , line 1: event handler never invoked: SupervisorControl::status_request warning in , line 1: event handler never invoked: SupervisorControl::stop_request +warning in , line 1: event handler never invoked: spicy_analyzer_for_mime_type warning in , line 1: event handler never invoked: this_is_never_used diff --git a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log index b53f471b1e..5ce56d1f46 100644 --- a/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.bare-load-baseline/canonified_loaded_scripts.log @@ -243,8 +243,18 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek build/scripts/builtin-plugins/__preload__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek build/scripts/builtin-plugins/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek + scripts/base/misc/version.zeek + scripts/base/frameworks/reporter/__load__.zeek + scripts/base/frameworks/reporter/main.zeek + scripts/base/utils/strings.zeek scripts/policy/misc/loaded-scripts.zeek scripts/base/utils/paths.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log index 38ef5af526..1a5ff60c6b 100644 --- a/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log +++ b/testing/btest/Baseline/coverage.default-load-baseline/canonified_loaded_scripts.log @@ -243,6 +243,9 @@ scripts/base/init-frameworks-and-bifs.zeek build/scripts/base/bif/plugins/Zeek_AsciiWriter.ascii.bif.zeek build/scripts/base/bif/plugins/Zeek_NoneWriter.none.bif.zeek build/scripts/base/bif/plugins/Zeek_SQLiteWriter.sqlite.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.consts.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.events.bif.zeek + build/scripts/base/bif/plugins/Zeek_Spicy.functions.bif.zeek scripts/base/init-default.zeek scripts/base/utils/active-http.zeek scripts/base/utils/exec.zeek @@ -443,6 +446,10 @@ scripts/base/init-default.zeek scripts/base/misc/find-filtered-trace.zeek scripts/base/misc/version.zeek build/scripts/builtin-plugins/__preload__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__preload__.zeek build/scripts/builtin-plugins/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/__load__.zeek + build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/bare.zeek + build/scripts/builtin-plugins/Zeek_Spicy/Zeek/Spicy/default.zeek scripts/policy/misc/loaded-scripts.zeek #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index d396d35ddf..7e8ed9f6ea 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -673,13 +673,17 @@ 0.000000 MetaHookPost CallFunction(SumStats::register_observe_plugins, , ()) -> 0.000000 MetaHookPost CallFunction(Supervisor::__is_supervisor, , ()) -> 0.000000 MetaHookPost CallFunction(Supervisor::is_supervisor, , ()) -> +0.000000 MetaHookPost CallFunction(Version::parse, ..., ...) -> 0.000000 MetaHookPost CallFunction(__init_primary_bifs, , ()) -> 0.000000 MetaHookPost CallFunction(__init_secondary_bifs, , ()) -> +0.000000 MetaHookPost CallFunction(bare_mode, , ()) -> 0.000000 MetaHookPost CallFunction(current_time, , ()) -> 0.000000 MetaHookPost CallFunction(filter_change_tracking, , ()) -> 0.000000 MetaHookPost CallFunction(getenv, , (CLUSTER_NODE)) -> 0.000000 MetaHookPost CallFunction(getenv, , (ZEEK_DEFAULT_LISTEN_ADDRESS)) -> 0.000000 MetaHookPost CallFunction(global_options, , ()) -> +0.000000 MetaHookPost CallFunction(gsub, ..., ...) -> +0.000000 MetaHookPost CallFunction(lstrip, ..., ...) -> 0.000000 MetaHookPost CallFunction(network_time, , ()) -> 0.000000 MetaHookPost CallFunction(port_to_count, , (2123/udp)) -> 0.000000 MetaHookPost CallFunction(port_to_count, , (2152/udp)) -> @@ -690,12 +694,15 @@ 0.000000 MetaHookPost CallFunction(reading_live_traffic, , ()) -> 0.000000 MetaHookPost CallFunction(reading_traces, , ()) -> 0.000000 MetaHookPost CallFunction(set_to_regex, , ({}, (^\.?|\.)(~~)$)) -> +0.000000 MetaHookPost CallFunction(split_string1, ..., ...) -> 0.000000 MetaHookPost CallFunction(string_to_pattern, , ((^\.?|\.)()$, F)) -> 0.000000 MetaHookPost CallFunction(sub, , ((^\.?|\.)(~~)$, <...>/, )) -> +0.000000 MetaHookPost CallFunction(to_count, ..., ...) -> 0.000000 MetaHookPost CallFunction(x509_set_certificate_cache, , ({})) -> 0.000000 MetaHookPost CallFunction(x509_set_certificate_cache_hit_callback, , (X509::x509_certificate_cache_replay{ X509::i{ if (X509::f$info?$x509) return event x509_certificate(X509::f, X509::e$handle, X509::e$certificate)for ([X509::i] in X509::e$extensions_cache) { X509::ext = X509::e$extensions_cache[X509::i]if (X509::ext is X509::Extension) event x509_extension(X509::f, (X509::ext as X509::Extension))elseif (X509::ext is X509::BasicConstraints) event x509_ext_basic_constraints(X509::f, (X509::ext as X509::BasicConstraints))elseif (X509::ext is X509::SubjectAlternativeName) event x509_ext_subject_alternative_name(X509::f, (X509::ext as X509::SubjectAlternativeName))elseif (X509::ext is X509::SctInfo) { X509::s = (X509::ext as X509::SctInfo)event x509_ocsp_ext_signed_certificate_timestamp(X509::f, X509::s$version, X509::s$logid, X509::s$timestamp, X509::s$hash_alg, X509::s$sig_alg, X509::s$signature)}elseReporter::error(fmt(Encountered unknown extension while replaying certificate with fuid %s, X509::f$id))}}})) -> 0.000000 MetaHookPost CallFunction(zeek_args, , ()) -> 0.000000 MetaHookPost CallFunction(zeek_init, , ()) -> +0.000000 MetaHookPost CallFunction(zeek_version, , ()) -> 0.000000 MetaHookPost DrainEvents() -> 0.000000 MetaHookPost LoadFile(0, ../main, <...>/main.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ../plugin, <...>/plugin.zeek) -> -1 @@ -808,6 +815,9 @@ 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> -1 @@ -924,8 +934,12 @@ 0.000000 MetaHookPost LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/bif, <...>/bif) -> -1 -0.000000 MetaHookPost LoadFile(0, base/init-default, <...>/init-default.zeek) -> -1 +0.000000 MetaHookPost LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> -1 0.000000 MetaHookPost LoadFile(0, base/packet-protocols, <...>/packet-protocols) -> -1 0.000000 MetaHookPost LoadFile(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) -> -1 @@ -1185,6 +1199,9 @@ 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) -> (-1, ) @@ -1301,8 +1318,12 @@ 0.000000 MetaHookPost LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/bif, <...>/bif) -> (-1, ) -0.000000 MetaHookPost LoadFileExtended(0, base/init-default, <...>/init-default.zeek) -> (-1, ) +0.000000 MetaHookPost LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base/packet-protocols, <...>/packet-protocols) -> (-1, ) 0.000000 MetaHookPost LoadFileExtended(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) -> (-1, ) @@ -2130,13 +2151,17 @@ 0.000000 MetaHookPre CallFunction(SumStats::register_observe_plugins, , ()) 0.000000 MetaHookPre CallFunction(Supervisor::__is_supervisor, , ()) 0.000000 MetaHookPre CallFunction(Supervisor::is_supervisor, , ()) +0.000000 MetaHookPre CallFunction(Version::parse, ..., ...) 0.000000 MetaHookPre CallFunction(__init_primary_bifs, , ()) 0.000000 MetaHookPre CallFunction(__init_secondary_bifs, , ()) +0.000000 MetaHookPre CallFunction(bare_mode, , ()) 0.000000 MetaHookPre CallFunction(current_time, , ()) 0.000000 MetaHookPre CallFunction(filter_change_tracking, , ()) 0.000000 MetaHookPre CallFunction(getenv, , (CLUSTER_NODE)) 0.000000 MetaHookPre CallFunction(getenv, , (ZEEK_DEFAULT_LISTEN_ADDRESS)) 0.000000 MetaHookPre CallFunction(global_options, , ()) +0.000000 MetaHookPre CallFunction(gsub, ..., ...) +0.000000 MetaHookPre CallFunction(lstrip, ..., ...) 0.000000 MetaHookPre CallFunction(network_time, , ()) 0.000000 MetaHookPre CallFunction(port_to_count, , (2123/udp)) 0.000000 MetaHookPre CallFunction(port_to_count, , (2152/udp)) @@ -2147,12 +2172,15 @@ 0.000000 MetaHookPre CallFunction(reading_live_traffic, , ()) 0.000000 MetaHookPre CallFunction(reading_traces, , ()) 0.000000 MetaHookPre CallFunction(set_to_regex, , ({}, (^\.?|\.)(~~)$)) +0.000000 MetaHookPre CallFunction(split_string1, ..., ...) 0.000000 MetaHookPre CallFunction(string_to_pattern, , ((^\.?|\.)()$, F)) 0.000000 MetaHookPre CallFunction(sub, , ((^\.?|\.)(~~)$, <...>/, )) +0.000000 MetaHookPre CallFunction(to_count, ..., ...) 0.000000 MetaHookPre CallFunction(x509_set_certificate_cache, , ({})) 0.000000 MetaHookPre CallFunction(x509_set_certificate_cache_hit_callback, , (X509::x509_certificate_cache_replay{ X509::i{ if (X509::f$info?$x509) return event x509_certificate(X509::f, X509::e$handle, X509::e$certificate)for ([X509::i] in X509::e$extensions_cache) { X509::ext = X509::e$extensions_cache[X509::i]if (X509::ext is X509::Extension) event x509_extension(X509::f, (X509::ext as X509::Extension))elseif (X509::ext is X509::BasicConstraints) event x509_ext_basic_constraints(X509::f, (X509::ext as X509::BasicConstraints))elseif (X509::ext is X509::SubjectAlternativeName) event x509_ext_subject_alternative_name(X509::f, (X509::ext as X509::SubjectAlternativeName))elseif (X509::ext is X509::SctInfo) { X509::s = (X509::ext as X509::SctInfo)event x509_ocsp_ext_signed_certificate_timestamp(X509::f, X509::s$version, X509::s$logid, X509::s$timestamp, X509::s$hash_alg, X509::s$sig_alg, X509::s$signature)}elseReporter::error(fmt(Encountered unknown extension while replaying certificate with fuid %s, X509::f$id))}}})) 0.000000 MetaHookPre CallFunction(zeek_args, , ()) 0.000000 MetaHookPre CallFunction(zeek_init, , ()) +0.000000 MetaHookPre CallFunction(zeek_version, , ()) 0.000000 MetaHookPre DrainEvents() 0.000000 MetaHookPre LoadFile(0, ../main, <...>/main.zeek) 0.000000 MetaHookPre LoadFile(0, ../plugin, <...>/plugin.zeek) @@ -2265,6 +2293,9 @@ 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) +0.000000 MetaHookPre LoadFile(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFile(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) @@ -2381,8 +2412,12 @@ 0.000000 MetaHookPre LoadFile(0, <...>/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFile(0, <...>/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, <...>/hooks.zeek, <...>/hooks.zeek) +0.000000 MetaHookPre LoadFile(0, Zeek<...>/bare.zeek, <...>/bare.zeek) +0.000000 MetaHookPre LoadFile(0, Zeek<...>/default.zeek, <...>/default.zeek) +0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) +0.000000 MetaHookPre LoadFile(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFile(0, base/bif, <...>/bif) -0.000000 MetaHookPre LoadFile(0, base/init-default, <...>/init-default.zeek) +0.000000 MetaHookPre LoadFile(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFile(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) 0.000000 MetaHookPre LoadFile(0, base/packet-protocols, <...>/packet-protocols) 0.000000 MetaHookPre LoadFile(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) @@ -2642,6 +2677,9 @@ 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.events.bif.zeek, <...>/Zeek_SSL.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.functions.bif.zeek, <...>/Zeek_SSL.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_SSL.types.bif.zeek, <...>/Zeek_SSL.types.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.consts.bif.zeek, <...>/Zeek_Spicy.consts.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.events.bif.zeek, <...>/Zeek_Spicy.events.bif.zeek) +0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Spicy.functions.bif.zeek, <...>/Zeek_Spicy.functions.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_Syslog.events.bif.zeek, <...>/Zeek_Syslog.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.events.bif.zeek, <...>/Zeek_TCP.events.bif.zeek) 0.000000 MetaHookPre LoadFileExtended(0, ./Zeek_TCP.functions.bif.zeek, <...>/Zeek_TCP.functions.bif.zeek) @@ -2758,8 +2796,12 @@ 0.000000 MetaHookPre LoadFileExtended(0, <...>/__load__.zeek, <...>/__load__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, <...>/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, <...>/hooks.zeek, <...>/hooks.zeek) +0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/bare.zeek, <...>/bare.zeek) +0.000000 MetaHookPre LoadFileExtended(0, Zeek<...>/default.zeek, <...>/default.zeek) +0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__load__.zeek, <...>/__load__.zeek) +0.000000 MetaHookPre LoadFileExtended(0, Zeek_Spicy/__preload__.zeek, <...>/__preload__.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/bif, <...>/bif) -0.000000 MetaHookPre LoadFileExtended(0, base/init-default, <...>/init-default.zeek) +0.000000 MetaHookPre LoadFileExtended(0, base/init-default.zeek, <...>/init-default.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/init-frameworks-and-bifs.zeek, <...>/init-frameworks-and-bifs.zeek) 0.000000 MetaHookPre LoadFileExtended(0, base/packet-protocols, <...>/packet-protocols) 0.000000 MetaHookPre LoadFileExtended(0, base<...>/CPP-load.bif, <...>/CPP-load.bif.zeek) @@ -3586,13 +3628,17 @@ 0.000000 | HookCallFunction SumStats::register_observe_plugins() 0.000000 | HookCallFunction Supervisor::__is_supervisor() 0.000000 | HookCallFunction Supervisor::is_supervisor() +0.000000 | HookCallFunction Version::parse(...) 0.000000 | HookCallFunction __init_primary_bifs() 0.000000 | HookCallFunction __init_secondary_bifs() +0.000000 | HookCallFunction bare_mode() 0.000000 | HookCallFunction current_time() 0.000000 | HookCallFunction filter_change_tracking() 0.000000 | HookCallFunction getenv(CLUSTER_NODE) 0.000000 | HookCallFunction getenv(ZEEK_DEFAULT_LISTEN_ADDRESS) 0.000000 | HookCallFunction global_options() +0.000000 | HookCallFunction gsub(...) +0.000000 | HookCallFunction lstrip(...) 0.000000 | HookCallFunction network_time() 0.000000 | HookCallFunction port_to_count(2123/udp) 0.000000 | HookCallFunction port_to_count(2152/udp) @@ -3603,12 +3649,15 @@ 0.000000 | HookCallFunction reading_live_traffic() 0.000000 | HookCallFunction reading_traces() 0.000000 | HookCallFunction set_to_regex({}, (^\.?|\.)(~~)$) +0.000000 | HookCallFunction split_string1(...) 0.000000 | HookCallFunction string_to_pattern((^\.?|\.)()$, F) 0.000000 | HookCallFunction sub((^\.?|\.)(~~)$, <...>/, ) +0.000000 | HookCallFunction to_count(...) 0.000000 | HookCallFunction x509_set_certificate_cache({}) 0.000000 | HookCallFunction x509_set_certificate_cache_hit_callback(X509::x509_certificate_cache_replay{ X509::i{ if (X509::f$info?$x509) return event x509_certificate(X509::f, X509::e$handle, X509::e$certificate)for ([X509::i] in X509::e$extensions_cache) { X509::ext = X509::e$extensions_cache[X509::i]if (X509::ext is X509::Extension) event x509_extension(X509::f, (X509::ext as X509::Extension))elseif (X509::ext is X509::BasicConstraints) event x509_ext_basic_constraints(X509::f, (X509::ext as X509::BasicConstraints))elseif (X509::ext is X509::SubjectAlternativeName) event x509_ext_subject_alternative_name(X509::f, (X509::ext as X509::SubjectAlternativeName))elseif (X509::ext is X509::SctInfo) { X509::s = (X509::ext as X509::SctInfo)event x509_ocsp_ext_signed_certificate_timestamp(X509::f, X509::s$version, X509::s$logid, X509::s$timestamp, X509::s$hash_alg, X509::s$sig_alg, X509::s$signature)}elseReporter::error(fmt(Encountered unknown extension while replaying certificate with fuid %s, X509::f$id))}}}) 0.000000 | HookCallFunction zeek_args() 0.000000 | HookCallFunction zeek_init() +0.000000 | HookCallFunction zeek_version() 0.000000 | HookDrainEvents 0.000000 | HookLoadFile ../main <...>/main.zeek 0.000000 | HookLoadFile ../plugin <...>/plugin.zeek @@ -3721,6 +3770,9 @@ 0.000000 | HookLoadFile ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek +0.000000 | HookLoadFile ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek +0.000000 | HookLoadFile ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek +0.000000 | HookLoadFile ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFile ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFile ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek @@ -3849,8 +3901,12 @@ 0.000000 | HookLoadFile <...>/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFile <...>/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile <...>/hooks.zeek <...>/hooks.zeek +0.000000 | HookLoadFile Zeek<...>/bare.zeek <...>/bare.zeek +0.000000 | HookLoadFile Zeek<...>/default.zeek <...>/default.zeek +0.000000 | HookLoadFile Zeek_Spicy/__load__.zeek <...>/__load__.zeek +0.000000 | HookLoadFile Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFile base/bif <...>/bif -0.000000 | HookLoadFile base/init-default <...>/init-default.zeek +0.000000 | HookLoadFile base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFile base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek 0.000000 | HookLoadFile base/packet-protocols <...>/packet-protocols 0.000000 | HookLoadFile base<...>/CPP-load.bif <...>/CPP-load.bif.zeek @@ -4098,6 +4154,9 @@ 0.000000 | HookLoadFileExtended ./Zeek_SSL.events.bif.zeek <...>/Zeek_SSL.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.functions.bif.zeek <...>/Zeek_SSL.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_SSL.types.bif.zeek <...>/Zeek_SSL.types.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Spicy.consts.bif.zeek <...>/Zeek_Spicy.consts.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Spicy.events.bif.zeek <...>/Zeek_Spicy.events.bif.zeek +0.000000 | HookLoadFileExtended ./Zeek_Spicy.functions.bif.zeek <...>/Zeek_Spicy.functions.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_Syslog.events.bif.zeek <...>/Zeek_Syslog.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.events.bif.zeek <...>/Zeek_TCP.events.bif.zeek 0.000000 | HookLoadFileExtended ./Zeek_TCP.functions.bif.zeek <...>/Zeek_TCP.functions.bif.zeek @@ -4226,8 +4285,12 @@ 0.000000 | HookLoadFileExtended <...>/__load__.zeek <...>/__load__.zeek 0.000000 | HookLoadFileExtended <...>/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended <...>/hooks.zeek <...>/hooks.zeek +0.000000 | HookLoadFileExtended Zeek<...>/bare.zeek <...>/bare.zeek +0.000000 | HookLoadFileExtended Zeek<...>/default.zeek <...>/default.zeek +0.000000 | HookLoadFileExtended Zeek_Spicy/__load__.zeek <...>/__load__.zeek +0.000000 | HookLoadFileExtended Zeek_Spicy/__preload__.zeek <...>/__preload__.zeek 0.000000 | HookLoadFileExtended base/bif <...>/bif -0.000000 | HookLoadFileExtended base/init-default <...>/init-default.zeek +0.000000 | HookLoadFileExtended base/init-default.zeek <...>/init-default.zeek 0.000000 | HookLoadFileExtended base/init-frameworks-and-bifs.zeek <...>/init-frameworks-and-bifs.zeek 0.000000 | HookLoadFileExtended base/packet-protocols <...>/packet-protocols 0.000000 | HookLoadFileExtended base<...>/CPP-load.bif <...>/CPP-load.bif.zeek diff --git a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr index e97cb1bad7..9422ad94ac 100644 --- a/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr +++ b/testing/btest/Baseline/scripts.base.frameworks.logging.field-extension-invalid/.stderr @@ -1,2 +1,3 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension +XXXXXXXXXX.XXXXXX warning: non-void function returning without a value: add_extension diff --git a/testing/btest/Baseline/spicy.spicy-dump/output b/testing/btest/Baseline/spicy.spicy-dump/output new file mode 100644 index 0000000000..01a459395c --- /dev/null +++ b/testing/btest/Baseline/spicy.spicy-dump/output @@ -0,0 +1,10 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +test::X { + xs: [ + 11 + 22 + 33 + 44 + 55 + ] +} diff --git a/testing/btest/Baseline/spicy.spicyz-aot/output b/testing/btest/Baseline/spicy.spicyz-aot/output new file mode 100644 index 0000000000..e7ad0949ae --- /dev/null +++ b/testing/btest/Baseline/spicy.spicyz-aot/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +8, [$data=[b"POST /post HTTP/1.1", b"User-Agent: curl/7.29.0", b"Host: httpbin.org", b"Accept: */*", b"Content-Length: 11", b"Content-Type: application/x-www-form-urlencoded", b"", b"hello world"]] +Event:, [POST /post HTTP/1.1, User-Agent: curl/7.29.0, Host: httpbin.org, Accept: */*, Content-Length: 11, Content-Type: application/x-www-form-urlencoded, , hello world] +8, [$data=[b"HTTP/1.1 200 OK", b"Server: gunicorn/0.16.1", b"Date: Tue, 19 Mar 2013 16:05:11 GMT", b"Content-Type: application/json", b"Content-Length: 366", b"Connection: close", b"", b"{\x0a \"origin\": \"10.142.133.148\",\x0a \"files\": {},\x0a \"form\": null,\x0a \"url\": \"http://httpbin.org/post\",\x0a \"args\": {},\x0a \"headers\": {\x0a \"Content-Length\": \"11\",\x0a \"Connection\": \"close\",\x0a \"Accept\": \"*/*\",\x0a \"User-Agent\": \"curl/7.29.0\",\x0a \"Host\": \"httpbin.org\",\x0a \"Content-Type\": \"application/x-www-form-urlencoded\"\x0a },\x0a \"json\": null,\x0a \"data\": \"hello world\"\x0a}"]] +Event:, [HTTP/1.1 200 OK, Server: gunicorn/0.16.1, Date: Tue, 19 Mar 2013 16:05:11 GMT, Content-Type: application/json, Content-Length: 366, Connection: close, , {\x0a "origin": "10.142.133.148",\x0a "files": {},\x0a "form": null,\x0a "url": "http://httpbin.org/post",\x0a "args": {},\x0a "headers": {\x0a "Content-Length": "11",\x0a "Connection": "close",\x0a "Accept": "*/*",\x0a "User-Agent": "curl/7.29.0",\x0a "Host": "httpbin.org",\x0a "Content-Type": "application/x-www-form-urlencoded"\x0a },\x0a "json": null,\x0a "data": "hello world"\x0a}] diff --git a/testing/btest/Baseline/spicy.spicyz-jit/output b/testing/btest/Baseline/spicy.spicyz-jit/output new file mode 100644 index 0000000000..e7ad0949ae --- /dev/null +++ b/testing/btest/Baseline/spicy.spicyz-jit/output @@ -0,0 +1,5 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +8, [$data=[b"POST /post HTTP/1.1", b"User-Agent: curl/7.29.0", b"Host: httpbin.org", b"Accept: */*", b"Content-Length: 11", b"Content-Type: application/x-www-form-urlencoded", b"", b"hello world"]] +Event:, [POST /post HTTP/1.1, User-Agent: curl/7.29.0, Host: httpbin.org, Accept: */*, Content-Length: 11, Content-Type: application/x-www-form-urlencoded, , hello world] +8, [$data=[b"HTTP/1.1 200 OK", b"Server: gunicorn/0.16.1", b"Date: Tue, 19 Mar 2013 16:05:11 GMT", b"Content-Type: application/json", b"Content-Length: 366", b"Connection: close", b"", b"{\x0a \"origin\": \"10.142.133.148\",\x0a \"files\": {},\x0a \"form\": null,\x0a \"url\": \"http://httpbin.org/post\",\x0a \"args\": {},\x0a \"headers\": {\x0a \"Content-Length\": \"11\",\x0a \"Connection\": \"close\",\x0a \"Accept\": \"*/*\",\x0a \"User-Agent\": \"curl/7.29.0\",\x0a \"Host\": \"httpbin.org\",\x0a \"Content-Type\": \"application/x-www-form-urlencoded\"\x0a },\x0a \"json\": null,\x0a \"data\": \"hello world\"\x0a}"]] +Event:, [HTTP/1.1 200 OK, Server: gunicorn/0.16.1, Date: Tue, 19 Mar 2013 16:05:11 GMT, Content-Type: application/json, Content-Length: 366, Connection: close, , {\x0a "origin": "10.142.133.148",\x0a "files": {},\x0a "form": null,\x0a "url": "http://httpbin.org/post",\x0a "args": {},\x0a "headers": {\x0a "Content-Length": "11",\x0a "Connection": "close",\x0a "Accept": "*/*",\x0a "User-Agent": "curl/7.29.0",\x0a "Host": "httpbin.org",\x0a "Content-Type": "application/x-www-form-urlencoded"\x0a },\x0a "json": null,\x0a "data": "hello world"\x0a}] diff --git a/testing/btest/bifs/bare_mode.zeek b/testing/btest/bifs/bare_mode.zeek new file mode 100644 index 0000000000..5183cde7d3 --- /dev/null +++ b/testing/btest/bifs/bare_mode.zeek @@ -0,0 +1,8 @@ +# @TEST-EXEC: zeek %INPUT >>output +# @TEST-EXEC: zeek -b %INPUT >>output +# @TEST-EXEC: btest-diff output + +event zeek_init() + { + print bare_mode(); + } diff --git a/testing/btest/broker/remote_event.zeek b/testing/btest/broker/remote_event.zeek index ddc4092841..611f9067ac 100644 --- a/testing/btest/broker/remote_event.zeek +++ b/testing/btest/broker/remote_event.zeek @@ -1,5 +1,7 @@ # @TEST-GROUP: broker # +# @TEST-REQUIRES: $SCRIPTS/have-spicy # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not. +# # @TEST-PORT: BROKER_PORT # # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out" diff --git a/testing/btest/broker/remote_event_any.zeek b/testing/btest/broker/remote_event_any.zeek index 2a290376ca..c096318717 100644 --- a/testing/btest/broker/remote_event_any.zeek +++ b/testing/btest/broker/remote_event_any.zeek @@ -1,5 +1,7 @@ # @TEST-GROUP: broker # +# @TEST-REQUIRES: $SCRIPTS/have-spicy # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not. +# # @TEST-PORT: BROKER_PORT # # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out" diff --git a/testing/btest/broker/remote_event_ssl_auth.zeek b/testing/btest/broker/remote_event_ssl_auth.zeek index d3882ab2b1..63acd24683 100644 --- a/testing/btest/broker/remote_event_ssl_auth.zeek +++ b/testing/btest/broker/remote_event_ssl_auth.zeek @@ -1,5 +1,7 @@ # @TEST-GROUP: broker # +# @TEST-REQUIRES: $SCRIPTS/have-spicy # The logging of Broker's `num_logs_outgoing` depends on whether the Spicy plugin is loaded or not. +# # @TEST-PORT: BROKER_PORT # # @TEST-EXEC: btest-bg-run recv "zeek -b ../recv.zeek >recv.out" diff --git a/testing/btest/btest.cfg b/testing/btest/btest.cfg index 7e5eedd02a..ad2ac12d6b 100644 --- a/testing/btest/btest.cfg +++ b/testing/btest/btest.cfg @@ -1,5 +1,5 @@ [btest] -TestDirs = doc bifs language core scripts coverage signatures plugins broker supervisor telemetry +TestDirs = doc bifs language core scripts coverage signatures plugins broker spicy supervisor telemetry TmpDir = %(testbase)s/.tmp BaselineDir = %(testbase)s/Baseline IgnoreDirs = .svn CVS .tmp @@ -32,6 +32,8 @@ ZEEK_DISABLE_ZEEKYGEN=1 ZEEK_ALLOW_INIT_ERRORS=1 ZEEK_SUPERVISOR_NO_SIGKILL=1 UBSAN_OPTIONS=print_stacktrace=1 +SPICY_PATH=`bash -c %(testbase)s/../../build/spicy-path` +HILTI_CXX_INCLUDE_DIRS=`bash -c %(testbase)s/../../build/hilti-cxx-include-dirs` [environment-AST-dup] # Environment for testing AST duplication functionality, which is diff --git a/testing/btest/core/check-unused-event-handlers.test b/testing/btest/core/check-unused-event-handlers.test index 7d3a581d6c..742a07554c 100644 --- a/testing/btest/core/check-unused-event-handlers.test +++ b/testing/btest/core/check-unused-event-handlers.test @@ -1,4 +1,5 @@ # This test should print a warning that the event handler is never invoked. +# @TEST-REQUIRES: $SCRIPTS/have-spicy # This test logs uninvoked event handlers, so disable it if Spicy and its plugin is unavailable. # @TEST-EXEC: zeek -b %INPUT check_for_unused_event_handlers=T # @TEST-EXEC: TEST_DIFF_CANONIFIER=$SCRIPTS/diff-sort btest-diff .stderr diff --git a/testing/btest/coverage/bare-load-baseline.test b/testing/btest/coverage/bare-load-baseline.test index 94fdb04b04..c5cd2d741c 100644 --- a/testing/btest/coverage/bare-load-baseline.test +++ b/testing/btest/coverage/bare-load-baseline.test @@ -7,6 +7,7 @@ # prefix to make the test work everywhere. That's what the sed magic # below does. Don't ask. :-) +# @TEST-REQUIRES: $SCRIPTS/have-spicy # This test logs loaded scripts, so disable it if Spicy and it associated plugin is unavailable. # @TEST-EXEC: zeek -b misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | awk 'NR>0{print $1}' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix diff --git a/testing/btest/coverage/default-load-baseline.test b/testing/btest/coverage/default-load-baseline.test index df13444ad7..43c2c5bdec 100644 --- a/testing/btest/coverage/default-load-baseline.test +++ b/testing/btest/coverage/default-load-baseline.test @@ -7,6 +7,7 @@ # prefix to make the test work everywhere. That's what the sed magic # below does. Don't ask. :-) +# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config # @TEST-EXEC: zeek misc/loaded-scripts # @TEST-EXEC: test -e loaded_scripts.log # @TEST-EXEC: cat loaded_scripts.log | egrep -v '#' | sed 's/ //g' | sed -e ':a' -e '$!N' -e 's/^\(.*\).*\n\1.*/\1/' -e 'ta' >prefix diff --git a/testing/btest/plugins/hooks-plugin/src/Plugin.cc b/testing/btest/plugins/hooks-plugin/src/Plugin.cc index ec0b527b6a..5c47145255 100644 --- a/testing/btest/plugins/hooks-plugin/src/Plugin.cc +++ b/testing/btest/plugins/hooks-plugin/src/Plugin.cc @@ -1,6 +1,8 @@ #include "Plugin.h" +#include + #include #include #include @@ -40,13 +42,36 @@ zeek::plugin::Configuration Plugin::Configure() static void describe_hook_args(const zeek::plugin::HookArgumentList& args, zeek::ODesc* d) { bool first = true; + bool serialize_args = true; for ( zeek::plugin::HookArgumentList::const_iterator i = args.begin(); i != args.end(); i++ ) { - if ( ! first ) - d->Add(", "); + if ( first ) + { + first = false; + + i->Describe(d); + + // For function calls we remove args for unstable arguments + // from parsing the version in `base/misc/version`. + if ( i->GetType() == zeek::plugin::HookArgument::FUNC && + (::strcmp(d->Description(), "Version::parse") == 0 || + ::strcmp(d->Description(), "gsub") == 0 || + ::strcmp(d->Description(), "split_string1") == 0 || + ::strcmp(d->Description(), "lstrip") == 0 || + ::strcmp(d->Description(), "to_count") == 0)) + serialize_args = false; + + continue; + } + + d->Add(", "); + + if ( serialize_args ) + i->Describe(d); + else + d->Add("..."); - i->Describe(d); first = false; } } @@ -70,8 +95,20 @@ std::pair Plugin::HookFunctionCall(const zeek::Func* func, z { zeek::ODesc d; d.SetShort(); + zeek::plugin::HookArgument(func).Describe(&d); - zeek::plugin::HookArgument(args).Describe(&d); + + // For function calls we remove args for unstable arguments + // from parsing the version in `base/misc/version`. + if ( ::strcmp(d.Description(), "Version::parse") == 0 || + ::strcmp(d.Description(), "gsub") == 0 || + ::strcmp(d.Description(), "split_string1") == 0 || + ::strcmp(d.Description(), "lstrip") == 0 || + ::strcmp(d.Description(), "to_count") == 0) + d.Add("(...)"); + else + zeek::plugin::HookArgument(args).Describe(&d); + fprintf(stderr, "%.6f %-15s %s\n", zeek::run_state::network_time, "| HookCallFunction", d.Description()); diff --git a/testing/btest/plugins/hooks.zeek b/testing/btest/plugins/hooks.zeek index c44dfd9dce..288e50f00e 100644 --- a/testing/btest/plugins/hooks.zeek +++ b/testing/btest/plugins/hooks.zeek @@ -1,13 +1,11 @@ # @TEST-REQUIRES: test "${ZEEK_ZAM}" != "1" +# @TEST-REQUIRES: ${SCRIPTS}/have-spicy # This test logs loaded scripts, so disable it if Spicy and the associated plugin are unavailable. # @TEST-EXEC: ${DIST}/auxil/zeek-aux/plugin-support/init-plugin -u . Demo Hooks # @TEST-EXEC: cp -r %DIR/hooks-plugin/* . # @TEST-EXEC: ./configure --zeek-dist=${DIST} && make -# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -b -r $TRACES/http/get.trace %INPUT s1.sig 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq >output +# @TEST-EXEC: ZEEK_PLUGIN_ACTIVATE="Demo::Hooks" ZEEK_PLUGIN_PATH=`pwd` zeek -r $TRACES/http/get.trace %INPUT s1.sig 2>&1 | $SCRIPTS/diff-remove-abspath | sort | uniq >output # @TEST-EXEC: btest-diff output -@unload base/misc/version -@load base/init-default - @load-sigs s2 @TEST-START-FILE s1.sig diff --git a/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek b/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek index 87a2caecbc..fd74e42002 100644 --- a/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek +++ b/testing/btest/scripts/base/frameworks/logging/field-extension-invalid.zeek @@ -1,3 +1,4 @@ +# @TEST-REQUIRES: test -x ${BUILD}/auxil/spicy/spicy/bin/spicy-config # @TEST-EXEC: zeek -b -r $TRACES/http/get.trace %INPUT # @TEST-EXEC: btest-diff conn.log # @TEST-EXEC: btest-diff .stderr diff --git a/testing/btest/spicy/spicy-dump.spicy b/testing/btest/spicy/spicy-dump.spicy new file mode 100644 index 0000000000..40f849354b --- /dev/null +++ b/testing/btest/spicy/spicy-dump.spicy @@ -0,0 +1,11 @@ +# @TEST-DOC: Smoke test for a bundled Spicy. +# +# @TEST-REQUIRES: $SCRIPTS/have-spicy +# @TEST-EXEC: printf 12345 | ${BUILD}/auxil/spicy/spicy/bin/spicy-dump -d %INPUT >output 2>&1 +# @TEST-EXEC: btest-diff output + +module test; + +public type X = unit { + xs: (bytes &size=1 &convert=$$.to_int()*11)[]; +}; diff --git a/testing/btest/spicy/spicyz-aot.test b/testing/btest/spicy/spicyz-aot.test new file mode 100644 index 0000000000..7abbf30745 --- /dev/null +++ b/testing/btest/spicy/spicyz-aot.test @@ -0,0 +1,37 @@ +# @TEST-DOC: Smoke test for a custom ahead-of-time compiled Spicy analyzer hooked into Zeek. +# +# @TEST-REQUIRES: $SCRIPTS/have-spicy +# @TEST-EXEC: ${BUILD}/src/builtin-plugins/spicy-plugin/bin/spicyz test.spicy test.evt -o test.hlto +# @TEST-EXEC: zeek -NN test.hlto | grep -q ANALYZER_SPICY_TEST +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.zeek test.hlto "Spicy::enable_print = T;" >>output 2>&1 +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE test.spicy +module test; + +import zeek; + +public type Dummy = unit { + # Consume all data. We split data into lines and log the number of lines and the lines when done. + data: bytes &eod &convert=$$.split(b"\r\n"); + + on %done { print |self.data|, self; } +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over TCP: + parse with test::Dummy, + port 80/tcp; + +on test::Dummy -> event test::dummy(self.data); +# @TEST-END-FILE + +# @TEST-START-FILE test.zeek +module test; + +event test::dummy(data: vector of string) +{ + print "Event:", data; +} +# @TEST-END-FILE diff --git a/testing/btest/spicy/spicyz-jit.test b/testing/btest/spicy/spicyz-jit.test new file mode 100644 index 0000000000..a462dd0625 --- /dev/null +++ b/testing/btest/spicy/spicyz-jit.test @@ -0,0 +1,37 @@ +# @TEST-DOC: Smoke test for a custom Spicy analyzer hooked into Zeek with JIT via Zeek itself. +# +# @TEST-REQUIRES: $SCRIPTS/have-spicy +# @TEST-EXEC: ${BUILD}/src/builtin-plugins/spicy-plugin/bin/spicyz test.spicy test.evt -o test.hlto +# @TEST-EXEC: zeek -NN test.hlto | grep -q ANALYZER_SPICY_TEST +# @TEST-EXEC: zeek -r ${TRACES}/http/post.trace test.zeek test.spicy test.evt "Spicy::enable_print = T;" >>output 2>&1 +# @TEST-EXEC: btest-diff output + +# @TEST-START-FILE test.spicy +module test; + +import zeek; + +public type Dummy = unit { + # Consume all data. We split data into lines and log the number of lines and the lines when done. + data: bytes &eod &convert=$$.split(b"\r\n"); + + on %done { print |self.data|, self; } +}; +# @TEST-END-FILE + +# @TEST-START-FILE test.evt +protocol analyzer spicy::Test over TCP: + parse with test::Dummy, + port 80/tcp; + +on test::Dummy -> event test::dummy(self.data); +# @TEST-END-FILE + +# @TEST-START-FILE test.zeek +module test; + +event test::dummy(data: vector of string) +{ + print "Event:", data; +} +# @TEST-END-FILE diff --git a/testing/btest/supervisor/config-bare-mode.zeek b/testing/btest/supervisor/config-bare-mode.zeek index 0e3f9c77e2..85fa8900dd 100644 --- a/testing/btest/supervisor/config-bare-mode.zeek +++ b/testing/btest/supervisor/config-bare-mode.zeek @@ -53,7 +53,7 @@ event zeek_init() print node_output_file, "supervised node zeek_init()"; # This is only defined when we're loading init-default.zeek: -@ifdef ( Notice::Info ) +@ifdef ( Conn::LOG ) print node_output_file, "default mode"; @else print node_output_file, "bare mode"; diff --git a/testing/scripts/have-spicy b/testing/scripts/have-spicy new file mode 100755 index 0000000000..d520e7438d --- /dev/null +++ b/testing/scripts/have-spicy @@ -0,0 +1,6 @@ +#!/bin/sh +if grep -q "DISABLE_SPICY:BOOL=true" "${BUILD}"/CMakeCache.txt; then + return 1 +else + true +fi