diff --git a/.cirrus.yml b/.cirrus.yml index 47deb48b8d..184484b2e8 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -18,6 +18,8 @@ mobile_ipv6_config: &MOBILE_IPV6_CONFIG --build-type=release --enable-cpp-tests resources_template: &RESOURCES_TEMPLATE cpu: *CPUS memory: *MEMORY + # For greediness, see https://medium.com/cirruslabs/introducing-greedy-container-instances-29aad06dc2b4 + greedy: true macos_resources_template: &MACOS_RESOURCES_TEMPLATE # https://medium.com/cirruslabs/new-macos-task-execution-architecture-for-cirrus-ci-604250627c94 @@ -78,7 +80,7 @@ env: # This is the key used to create HMAC auth keys for the benchmark script. This # was generated by creating a new key using openssl, and then running sha256 # on it. - ZEEK_BENCHMARK_HMAC_KEY: ENCRYPTED[412224bbea9652030da976537f4d96c79ee79a0ba5a2f93b6c32953e1be0362defdf5fa07b3dc54ae61f9a52be30eac7] + ZEEK_BENCHMARK_HMAC_KEY: ENCRYPTED[363e79b9942f348e53ab1f39f6ac8f7118bea2f4228ad1ef7b55981d3ef8d26dd756872f600ff40f2d7dcadb71f88513] # This is the https endpoint host and port used for benchmarking. It's kept # encrypted as a security measure to avoid leaking the host's information. @@ -131,6 +133,23 @@ centos7_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE +debian11_task: + container: + # Debian 11 EOL: June 2026 + dockerfile: ci/debian-11/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + +debian11_static_task: + container: + # Just use a recent/common distro to run a static compile test. + # Debian 11 EOL: June 2026 + dockerfile: ci/debian-11/Dockerfile + << : *RESOURCES_TEMPLATE + << : *CI_TEMPLATE + env: + ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG + debian10_task: container: # Debian 10 EOL: June 2024 @@ -138,16 +157,6 @@ debian10_task: << : *RESOURCES_TEMPLATE << : *CI_TEMPLATE -debian10_static_task: - container: - # Just uses a recent/common distro to run a static compile test. - # Debian 10 EOL: June 2024 - dockerfile: ci/debian-10/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE - env: - ZEEK_CI_CONFIGURE_FLAGS: *STATIC_CONFIG - debian9_task: container: # Debian 9 EOL: June 2022 @@ -192,13 +201,6 @@ ubuntu18_task: env: ZEEK_CI_CONFIGURE_FLAGS: *MOBILE_IPV6_CONFIG -ubuntu16_task: - container: - # Ubuntu 16.04 EOL: April 2021 - dockerfile: ci/ubuntu-16.04/Dockerfile - << : *RESOURCES_TEMPLATE - << : *CI_TEMPLATE - alpine_task: container: # Alpine releases typically happen every 6 months w/ support for 2 years. diff --git a/.github/workflows/generate-docs.yml b/.github/workflows/generate-docs.yml index 2b3be68023..a940523edf 100644 --- a/.github/workflows/generate-docs.yml +++ b/.github/workflows/generate-docs.yml @@ -92,3 +92,16 @@ jobs: git add doc git status git commit -m 'Update doc submodule [nomail] [skip ci]' && git push auth master || /bin/true + + - name: Send email + if: failure() + uses: dawidd6/action-send-mail@v3.4.1 + with: + server_address: ${{secrets.SMTP_HOST}} + server_port: ${{secrets.SMTP_PORT}} + username: ${{secrets.SMTP_USER}} + password: ${{secrets.SMTP_PASS}} + subject: generate-docs Github Action failed! + body: generate-docs job of ${{github.repository}} Failed! See https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} for details. + to: ${{secrets.MAIL_TO}} + from: Github Actions <${{secrets.MAIL_FROM}}> diff --git a/CHANGES b/CHANGES index 80cfc3b258..59ca1abded 100644 --- a/CHANGES +++ b/CHANGES @@ -1,3 +1,172 @@ +4.2.0-dev.78 | 2021-08-19 09:39:23 -0700 + + * Return fully-escaped string if utf8 conversion fails (Tim Wojtulewicz, Corelight) + + This adds a new function for validating UTF-8 sequences by converting to + UTF-32. This allows us to also check for various blocks of codepointsi + that we consider invalid while checking for valid sequences in general. + +4.2.0-dev.76 | 2021-08-18 08:40:41 -0700 + + * Fix option length computation in Geneve analyzer. (Benjamin Bannier, Corelight) + + We previously computed the length of the Geneve options field + incorrectly which lead to us passing data at an incorrect offset to + inner analyzers. With this patch we now interpret the length field + correctly, according the the spec + https://datatracker.ietf.org/doc/html/rfc8926#section-3.4. + + Closes #1726. + +4.2.0-dev.73 | 2021-08-12 09:57:59 -0700 + + * GH-1713: Avoid unneeded reallocs in SerializationFormat::WriteData (Tim Wojtulewicz, Corelight) + +4.2.0-dev.70 | 2021-08-06 13:29:30 -0700 + + * Do not lookup ignore_checksums_nets for every packet (Johanna Amann, Corelight) + + This could lead to a noticeable (single-percent) performance + improvement. + + Most of the functionality for this is in the packet analyzers that now + cache ignore_chesksums_nets. + + Based on a patch by Arne Welzel (Corelight). + +4.2.0-dev.68 | 2021-08-04 09:57:32 +0100 + + * Use unordered_map to store sessions for performance reasons. This might lead to an 8-9% speedup of Zeek. + See GH-1706 for details. (Tim Wojtulewicz, Corelight) + +4.2.0-dev.64 | 2021-08-03 10:23:41 +0100 + + * Under certain circumstances, Zeek processes could get into an infinite looping state inside RotationTimer. + This is fixed by handling special cases of rotation happening exactly on the time boundary, and fixing + a special case of timer expiration. Fixes GH-1689. (Sowmya Ramapatruni, Corelight) + +4.2.0-dev.62 | 2021-08-03 10:21:18 +0100 + + * Fix some HTTP evasions. Now HTTP packets are correctly parsed, when CRLF is missing on a multipart + boundary / at packet boundaries. Fixes GH-1598 (jerome Grandvalet) + +4.2.0-dev.59 | 2021-08-02 11:10:04 -0700 + + * Fix failing test due to patricia change (Tim Wojtulewicz, Corelight) + + * Rebase patricia code on upstream version (Tim Wojtulewicz, Corelight) + +4.2.0-dev.56 | 2021-07-30 08:52:49 -0700 + + * GH-1654: Exclude the .git directory when installing cmake files (Tim Wojtulewicz, Corelight) + +4.2.0-dev.54 | 2021-07-29 13:23:51 -0700 + + * GH-1692: Add some safety to ASN's binary_to_int64 to avoid bad shifts (Tim Wojtulewicz, Corelight) + +4.2.0-dev.50 | 2021-07-27 09:36:13 -0700 + + * Fix a use-after-free during shutdown (Tim Wojtulewicz, Corelight) + +4.2.0-dev.48 | 2021-07-26 13:03:01 -0700 + + * GH-1693: Fix potential crash with elements being modified during robust iteration (Tim Wojtulewicz, Corelight) + + * Update HMAC key used for benchmarking service (Tim Wojtulewicz, Corelight) + +4.2.0-dev.45 | 2021-07-23 09:28:49 -0700 + + * GH-1684: Ensure that the time gets updated every pass if we're reading live traffic (Tim Wojtulewicz, Corelight) + + This is necessary for e.g. packet sources that don't have a selectable + file descriptor. They'll always be ready on a very short timeout, but + won't necessarily have a packet to process. In these case, sometimes + the time won't get updated for a long time and timers don't function + correctly. + +4.2.0-dev.43 | 2021-07-21 11:41:19 -0700 + + * Fix handling of timers when cloning TableVals (Johanna Amann, Corelight) + + When cloning TableVals, a new timer was created for the wrong object + (the existing TableVal, not the clone). This lead to the already + existing timer being no longer accessible. Which, in turn, leads to an + abandoned timer reading into no longer allocated data when the original + TableVal is deleted. + + Fixes GH-1687 + +4.2.0-dev.40 | 2021-07-20 09:58:14 -0700 + + * Cover in NEWS the inclusion of btest tooling in the installation (Christian Kreibich, Corelight) + + [skip ci] + +4.2.0-dev.38 | 2021-07-20 09:55:59 -0700 + + * Fix generate-docs github action to send email when it fails (Tim Wojtulewicz, Corelight) + + * Use Cirrus's new greedy mode for parallelizing builds and tests (Christian Kreibich, Corelight) + + This oversubscribes our cores 2x, which testing shows we actually + run with at times: speedup is around a third on average for builds, + and a bit more than that for testing. + + Also some light Bashification in ci/build.sh, for consistency. + +4.2.0-dev.34 | 2021-07-19 08:56:45 -0700 + + * Bump highwayhash to pull in FreeBSD 14 fix (Christian Kreibich, Corelight) + +4.2.0-dev.32 | 2021-07-19 08:55:06 -0700 + + * Update 3rdparty submodule to pull in doctest to 2.4.6. (Robin Sommer, Corelight) + + Our old version didn't support compilation on Apple's M1 yet. + +4.2.0-dev.30 | 2021-07-19 08:54:30 -0700 + + * Fix registration of protocol analyzers from inside plugins. (Robin Sommer, Corelight) + + With the recent packet manager work, it broke to register a protocol + analyzer for a specific port from inside a plugin's initialization code. + That's because that registration now depends on the packet manager being + set up, which isn't case at that time a plugin's `InitPostInit()` runs. + This fix contains two parts: + + - Initialize the packet manager before the analyzer manager, so that + the latter's `InitPostScript()` can rely on the former being + ready. + + - Change the analyzer manager to (only) record port registrations + happening before it's fully initialized. Its `InitPostScript()` + then performs the actual registrations, knowing it can use the + packet manager now. + + This comes with a `cmake/` to add a missing include directory. + +4.2.0-dev.28 | 2021-07-16 22:43:35 -0700 + + * Update broker submodule for new CAF version (Tim Wojtulewicz, Corelight) + + * Setup generate-docs workflow to run on push to test branch (Tim Wojtulewicz, Corelight) + +4.2.0-dev.24 | 2021-07-15 11:33:31 -0700 + + * Add a TODO to return a correct status for ParseRR_WKS (Vlad Grigorescu) + + * Update scripts/site/local-compat test for 4.2 cycle (Tim Wojtulewicz) + + * Remove tests for deprecated operators. (Robin Sommer, Corelight) + + * Add tests for operations on vectors of string. (Robin Sommer, Corelight) + +4.2.0-dev.18 | 2021-07-15 09:50:25 -0700 + + * Remove tests for deprecated operators. (Robin Sommer, Corelight) + + * Add tests for operations on vectors of string. (Robin Sommer, Corelight) + 4.2.0-dev.14 | 2021-07-14 15:01:20 -0700 * Fix race conditions in scripts.base.frameworks.input.reread (Christian Kreibich, Corelight) diff --git a/CMakeLists.txt b/CMakeLists.txt index b15b8cc050..33418cf05b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -547,7 +547,8 @@ configure_file(${CMAKE_CURRENT_SOURCE_DIR}/zeek-config.in install(PROGRAMS ${CMAKE_CURRENT_BINARY_DIR}/zeek-config DESTINATION bin) install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/cmake DESTINATION share/zeek - USE_SOURCE_PERMISSIONS) + USE_SOURCE_PERMISSIONS + PATTERN ".git" EXCLUDE) # Install wrapper script for Bro-to-Zeek renaming. include(InstallShellScript) diff --git a/NEWS b/NEWS index 3a083bdf4b..2c4ec1f2c5 100644 --- a/NEWS +++ b/NEWS @@ -3,6 +3,21 @@ This document summarizes the most important changes in the current Zeek release. For an exhaustive list of changes, see the ``CHANGES`` file (note that submodules, such as Broker, come with their own ``CHANGES``.) +Zeek 4.2.0 +========== + +New Functionality +----------------- + +Changed Functionality +--------------------- + +Removed Functionality +--------------------- + +Deprecated Functionality +------------------------ + Zeek 4.1.0 ========== @@ -78,20 +93,31 @@ New Functionality takes a semicolon separated list of paths containing plugins that will be statically built into Zeek. +- Added a ``--plugindir`` argument to ``configure`` to set the + installation path for plugins. + - The X509 analyzer now can check if a specific hostname is valid for a certificate. Two new BIFs were added for this, ``x509_check_hostname`` and ``x509_check_cert_hostname``. A new field ``sni_matches_cert`` that tracks this information was added to ``ssl.log``. -- Added a ``--plugindir`` argument to ``configure`` to set the - installation path for plugins. - - Added new functions to dynamically enable/disable file analyzers: - ``global enable_analyzer: function(tag: Files::Tag): bool;`` - ``global disable_analyzer: function(tag: Files::Tag): bool;`` - ``global analyzer_enabled: function(tag: Files::Tag): bool;`` +- Zeek now includes its own BTest tooling in the distribution, enabling other + tests (e.g. in Zeek packages) to use it. The ``$PREFIX/share/btest folder``, + reported via ``zeek-config --btest_tools_dir``, includes: + + - ``scripts/`` for ``btest-diff`` canonifiers + - ``data/`` for data files, including ``random.seed`` + - ``data/pcaps`` for the test pcaps + + Configuring with ``--disable-btest-pcaps`` suppresses installation of the + test pcaps. + - The Supervisor now defaults to starting with a minimal set of Zeek scripts controlled by a new init file, ``base/init-supervisor.zeek``. One may still run it with a larger configuration by loading additional diff --git a/VERSION b/VERSION index c4e52a775d..e520c2589d 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.2.0-dev.14 +4.2.0-dev.78 diff --git a/auxil/broker b/auxil/broker index def3f1b0d3..47cac80cbe 160000 --- a/auxil/broker +++ b/auxil/broker @@ -1 +1 @@ -Subproject commit def3f1b0d3f97af44dd3a87731869e6963893cba +Subproject commit 47cac80cbe1e1bde8e3b425903e50d62715972a2 diff --git a/auxil/highwayhash b/auxil/highwayhash index ea06651bd1..2361494e04 160000 --- a/auxil/highwayhash +++ b/auxil/highwayhash @@ -1 +1 @@ -Subproject commit ea06651bd11387f5aac694c819314536fe7df060 +Subproject commit 2361494e0400d52eb76d2c6c62db72168ebe69d0 diff --git a/auxil/package-manager b/auxil/package-manager index af8275a5fd..9ccb796814 160000 --- a/auxil/package-manager +++ b/auxil/package-manager @@ -1 +1 @@ -Subproject commit af8275a5fd9adb30894407e8c617c525ee34e4f3 +Subproject commit 9ccb7968149ebf91a0c15ff04aca13e558a8b465 diff --git a/auxil/zeek-archiver b/auxil/zeek-archiver index d3e55991cb..f3a1e8fe46 160000 --- a/auxil/zeek-archiver +++ b/auxil/zeek-archiver @@ -1 +1 @@ -Subproject commit d3e55991cbe69f37966207479492edd38d548b1d +Subproject commit f3a1e8fe464c0425688eff67e30f35c678914ad2 diff --git a/ci/build.sh b/ci/build.sh index 9227ac7a35..20ca3237cc 100755 --- a/ci/build.sh +++ b/ci/build.sh @@ -1,15 +1,18 @@ #! /usr/bin/env bash +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" +. ${SCRIPT_DIR}/common.sh + set -e set -x # If we're on macOS, use --osx-sysroot to ensure we can find the SDKs from Xcode. This avoids # some problems with Catalina specifically, but it doesn't break anything on Big Sur either. -if [ "${CIRRUS_OS}" == "darwin" ]; then +if [[ "${CIRRUS_OS}" == "darwin" ]]; then export ZEEK_CI_CONFIGURE_FLAGS="${ZEEK_CI_CONFIGURE_FLAGS} --osx-sysroot=$(xcrun --show-sdk-path)" fi -if [ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]; then +if [[ "${ZEEK_CI_CREATE_ARTIFACT}" != "1" ]]; then ./configure ${ZEEK_CI_CONFIGURE_FLAGS} cd build make -j ${ZEEK_CI_CPUS} diff --git a/ci/common.sh b/ci/common.sh new file mode 100644 index 0000000000..cc84e9a973 --- /dev/null +++ b/ci/common.sh @@ -0,0 +1,12 @@ +# Common material sourced by Bash CI scripts in this directory + +# On Cirrus, oversubscribe the CPUs when on Linux. This uses Cirrus' "greedy" feature. +if [[ "${CIRRUS_OS}" == linux ]]; then + if [[ -n "${ZEEK_CI_CPUS}" ]]; then + ZEEK_CI_CPUS=$(( 2 * ${ZEEK_CI_CPUS} )) + fi + + if [[ -n "${ZEEK_CI_BTEST_JOBS}" ]]; then + ZEEK_CI_BTEST_JOBS=$(( 2 * ${ZEEK_CI_BTEST_JOBS} )) + fi +fi diff --git a/ci/ubuntu-16.04/Dockerfile b/ci/debian-11/Dockerfile similarity index 74% rename from ci/ubuntu-16.04/Dockerfile rename to ci/debian-11/Dockerfile index 5b65acd55f..584fde914c 100644 --- a/ci/ubuntu-16.04/Dockerfile +++ b/ci/debian-11/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:16.04 +FROM debian:11 ENV DEBIAN_FRONTEND="noninteractive" TZ="America/Los_Angeles" @@ -15,9 +15,6 @@ RUN apt-get update && apt-get -y install \ python3 \ python3-dev \ python3-pip\ - clang-8 \ - libc++-8-dev \ - libc++abi-8-dev \ swig \ zlib1g-dev \ libkrb5-dev \ @@ -29,7 +26,3 @@ RUN apt-get update && apt-get -y install \ && rm -rf /var/lib/apt/lists/* RUN pip3 install junit2html - -ENV CC=/usr/bin/clang-8 -ENV CXX=/usr/bin/clang++-8 -ENV CXXFLAGS=-stdlib=libc++ diff --git a/ci/test.sh b/ci/test.sh index 2690094c7b..f6f0166e9a 100755 --- a/ci/test.sh +++ b/ci/test.sh @@ -16,6 +16,9 @@ if [[ -z "${CIRRUS_CI}" ]]; then ZEEK_CI_BTEST_RETRIES=2 fi +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &> /dev/null && pwd)" +. ${SCRIPT_DIR}/common.sh + function pushd { command pushd "$@" > /dev/null || exit 1 diff --git a/cmake b/cmake index 9d762b4cac..4d1990f0e4 160000 --- a/cmake +++ b/cmake @@ -1 +1 @@ -Subproject commit 9d762b4cacf299a2e54e0f7f258868ee217f1d36 +Subproject commit 4d1990f0e4c273cf51ec52278add6ff256f9c889 diff --git a/doc b/doc index 5096cd7f22..b941bd63a4 160000 --- a/doc +++ b/doc @@ -1 +1 @@ -Subproject commit 5096cd7f22af78bc3124298a6b290b3a6e5d28a7 +Subproject commit b941bd63a4c08570245be666e00ed0ef783a5591 diff --git a/scripts/base/frameworks/broker/main.zeek b/scripts/base/frameworks/broker/main.zeek index 53f6239361..891acbd601 100644 --- a/scripts/base/frameworks/broker/main.zeek +++ b/scripts/base/frameworks/broker/main.zeek @@ -286,6 +286,7 @@ export { global listen: function(a: string &default = default_listen_address, p: port &default = default_port, retry: interval &default = default_listen_retry): port; + ## Initiate a remote connection. ## ## a: an address to connect to, e.g. "localhost" or "127.0.0.1". diff --git a/scripts/base/frameworks/supervisor/main.zeek b/scripts/base/frameworks/supervisor/main.zeek index 8b0161b79e..171ccc6971 100644 --- a/scripts/base/frameworks/supervisor/main.zeek +++ b/scripts/base/frameworks/supervisor/main.zeek @@ -43,9 +43,10 @@ event zeek_init() &priority=10 { if ( Supervisor::is_supervisor() && SupervisorControl::enable_listen ) { - Broker::listen(Broker::default_listen_address, - Broker::default_port, - Broker::default_listen_retry); + # This may fail, possibly with scheduled retries. Any failures + # already get logged by the listen() implementation, so we don't + # report additionally. + Broker::listen(); } Broker::subscribe(SupervisorControl::topic_prefix); diff --git a/scripts/base/init-bare.zeek b/scripts/base/init-bare.zeek index ea70767f72..8f40fe4551 100644 --- a/scripts/base/init-bare.zeek +++ b/scripts/base/init-bare.zeek @@ -25,6 +25,13 @@ type string_any_table: table[string] of any; ## directly and then remove this alias. type string_set: set[string]; +## A set of subnets. +## +## .. todo:: We need this type definition only for declaring builtin functions +## via ``bifcl``. We should extend ``bifcl`` to understand composite types +## directly and then remove this alias. +type subnet_set: set[subnet]; + ## A set of addresses. ## ## .. todo:: We need this type definition only for declaring builtin functions diff --git a/scripts/base/packet-protocols/ip/main.zeek b/scripts/base/packet-protocols/ip/main.zeek index 0e1c506bcb..145ec639b9 100644 --- a/scripts/base/packet-protocols/ip/main.zeek +++ b/scripts/base/packet-protocols/ip/main.zeek @@ -9,6 +9,14 @@ const IPPROTO_IPIP : count = 4; const IPPROTO_IPV6 : count = 41; const IPPROTO_GRE : count = 47; +function analyzer_option_change_ignore_checksums_nets(ID: string, new_value: set[subnet], location: string) : set[subnet] + { + if ( ID == "ignore_checksums_nets" ) + PacketAnalyzer::__set_ignore_checksums_nets(new_value); + + return new_value; + } + event zeek_init() &priority=20 { PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_IPIP, PacketAnalyzer::ANALYZER_IPTUNNEL); @@ -19,4 +27,6 @@ event zeek_init() &priority=20 PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_UDP, PacketAnalyzer::ANALYZER_UDP); PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP, PacketAnalyzer::ANALYZER_ICMP); PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_IP, IPPROTO_ICMP6, PacketAnalyzer::ANALYZER_ICMP); + + Option::set_change_handler("ignore_checksums_nets", analyzer_option_change_ignore_checksums_nets, 5); } diff --git a/src/3rdparty b/src/3rdparty index a0155ef2c4..d31b51e6a0 160000 --- a/src/3rdparty +++ b/src/3rdparty @@ -1 +1 @@ -Subproject commit a0155ef2c4f4ba5623e88d1f48231343e4b019b3 +Subproject commit d31b51e6a06ad4c71db81981920eb753954abbf8 diff --git a/src/Dict.cc b/src/Dict.cc index a31fdd39be..368de9a2ed 100644 --- a/src/Dict.cc +++ b/src/Dict.cc @@ -461,6 +461,71 @@ TEST_CASE("dict new robust iteration") delete key3; } +class DictTestDummy + { +public: + DictTestDummy(int v) : v(v) {} + ~DictTestDummy() {} + int v = 0; + }; + +TEST_CASE("dict robust iteration replacement") + { + PDict dict; + + DictTestDummy* val1 = new DictTestDummy(15); + uint32_t key_val1 = 5; + detail::HashKey* key1 = new detail::HashKey(key_val1); + + DictTestDummy* val2 = new DictTestDummy(10); + uint32_t key_val2 = 25; + detail::HashKey* key2 = new detail::HashKey(key_val2); + + DictTestDummy* val3 = new DictTestDummy(20); + uint32_t key_val3 = 35; + detail::HashKey* key3 = new detail::HashKey(key_val3); + + dict.Insert(key1, val1); + dict.Insert(key2, val2); + dict.Insert(key3, val3); + + int count = 0; + auto it = dict.begin_robust(); + + // Iterate past the first couple of elements so we're not done, but the + // iterator is still pointing at a valid element. + for ( ; count != 2 && it != dict.end_robust(); ++count, ++it ) + { + } + + // Store off the value at this iterator index + auto* v = it->GetValue(); + + // Replace it with something else + auto k = it->GetHashKey(); + DictTestDummy* val4 = new DictTestDummy(50); + dict.Insert(k.get(), val4); + + // Delete the original element + delete val2; + + // This shouldn't crash with AddressSanitizer + for ( ; it != dict.end_robust(); ++it ) + { + uint64_t k = *(uint32_t*) it->GetKey(); + auto* v = it->GetValue(); + CHECK(v->v == 50); + } + + delete key1; + delete key2; + delete key3; + + delete val1; + delete val3; + delete val4; + } + TEST_CASE("dict iterator invalidation") { PDict dict; @@ -1066,8 +1131,10 @@ void* Dictionary::Insert(void* key, int key_size, detail::hash_t hash, void* val Init(); void* v = nullptr; - //if found. i is the position - //if not found, i is the insert position, d is the distance of key on position i. + + // Look to see if this key is already in the table. If found, insert_position is the + // position of the existing element. If not, insert_position is where it'll be inserted + // and insert_distance is the distance of the key for the position. int insert_position = -1, insert_distance = -1; int position = LookupIndex(key, key_size, hash, &insert_position, &insert_distance); if ( position >= 0 ) @@ -1099,6 +1166,13 @@ void* Dictionary::Insert(void* key, int key_size, detail::hash_t hash, void* val //need to set new v for iterators too. for ( auto c: *iterators ) { + // Check to see if this iterator points at the entry we're replacing. The iterator + // keeps a copy of the element, so we need to update it too. + if ( **c == table[position] ) + (*c)->value = val; + + // Check if any of the inserted elements in this iterator point at the entry being + // replaced. Update those too. auto it = std::find(c->inserted->begin(), c->inserted->end(), table[position]); if ( it != c->inserted->end() ) it->value = val; diff --git a/src/RunState.cc b/src/RunState.cc index ffce8b96fb..6ab0542e10 100644 --- a/src/RunState.cc +++ b/src/RunState.cc @@ -57,6 +57,7 @@ double first_timestamp = 0.0; double current_wallclock = 0.0; double current_pseudo = 0.0; bool zeek_init_done = false; +bool time_updated = false; RETSIGTYPE watchdog(int /* signo */) { @@ -133,6 +134,7 @@ RETSIGTYPE watchdog(int /* signo */) void update_network_time(double new_network_time) { + time_updated = true; network_time = new_network_time; PLUGIN_HOOK_VOID(HOOK_UPDATE_NETWORK_TIME, HookUpdateNetworkTime(new_network_time)); } @@ -287,6 +289,7 @@ void run_loop() while ( iosource_mgr->Size() || (BifConst::exit_only_after_terminate && ! terminating) ) { + time_updated = false; iosource_mgr->FindReadySources(&ready); #ifdef DEBUG @@ -327,6 +330,18 @@ void run_loop() expire_timers(); } + // Ensure that the time gets updated every pass if we're reading live. + // This is necessary for e.g. packet sources that don't have a selectable + // file descriptor. They'll always be ready on a very short timeout, but + // won't necessarily have a packet to process. In these case, sometimes + // the time won't get updated for a long time and timers don't function + // correctly. + if ( (! time_updated && reading_live) ) + { + update_network_time(util::current_time()); + expire_timers(); + } + event_mgr.Drain(); processing_start_time = 0.0; // = "we're not processing now" @@ -407,8 +422,6 @@ void delete_run() { util::detail::set_processing_status("TERMINATING", "delete_run"); - delete session_mgr; - for ( int i = 0; i < zeek::detail::NUM_ADDR_ANONYMIZATION_METHODS; ++i ) delete zeek::detail::ip_anonymizer[i]; } diff --git a/src/SerializationFormat.cc b/src/SerializationFormat.cc index ad9e2b26d9..fe5d34bbe2 100644 --- a/src/SerializationFormat.cc +++ b/src/SerializationFormat.cc @@ -82,10 +82,18 @@ bool SerializationFormat::ReadData(void* b, size_t count) bool SerializationFormat::WriteData(const void* b, size_t count) { // Increase buffer if necessary. + bool size_changed = false; while ( output_pos + count > output_size ) + { output_size *= GROWTH_FACTOR; + size_changed = true; + } - output = (char*)util::safe_realloc(output, output_size); + // The glibc standard states explicitly that calling realloc with the same + // size is a no-op, but the same claim can't be made on other platforms. + // There's really no reason to do that though. + if ( size_changed ) + output = (char*)util::safe_realloc(output, output_size); memcpy(output + output_pos, b, count); output_pos += count; diff --git a/src/Timer.cc b/src/Timer.cc index e8f47cfbb4..ecf56dd057 100644 --- a/src/Timer.cc +++ b/src/Timer.cc @@ -152,7 +152,7 @@ void PQ_TimerMgr::Expire() int PQ_TimerMgr::DoAdvance(double new_t, int max_expire) { Timer* timer = Top(); - for ( num_expired = 0; (num_expired < max_expire || max_expire == 0) && + for ( num_expired = 0; (num_expired < max_expire ) && timer && timer->Time() <= new_t; ++num_expired ) { last_timestamp = timer->Time(); diff --git a/src/Val.cc b/src/Val.cc index 3c561ff81f..35df899c6c 100644 --- a/src/Val.cc +++ b/src/Val.cc @@ -1991,7 +1991,7 @@ bool TableVal::Contains(const IPAddr& addr) const return false; } - return (subnets->Lookup(addr, true) != 0); + return (subnets->Lookup(addr, 128, false) != 0); } VectorValPtr TableVal::LookupSubnets(const SubNetVal* search) @@ -2768,8 +2768,8 @@ ValPtr TableVal::DoClone(CloneState* state) // As network_time is not necessarily initialized yet, we set // a timer which fires immediately. - timer = new TableValTimer(this, 1); - detail::timer_mgr->Add(timer); + tv->timer = new TableValTimer(tv.get(), 1); + detail::timer_mgr->Add(tv->timer); } if ( expire_func ) diff --git a/src/analyzer/Manager.cc b/src/analyzer/Manager.cc index 5bfd77f539..c23d329d9f 100644 --- a/src/analyzer/Manager.cc +++ b/src/analyzer/Manager.cc @@ -86,6 +86,15 @@ void Manager::InitPostScript() for ( auto i = 0; i < port_list->Length(); ++i ) vxlan_ports.emplace_back(port_list->Idx(i)->AsPortVal()->Port()); + + for ( const auto& p : pending_analyzers_for_ports ) { + if ( ! RegisterAnalyzerForPort(p) ) + reporter->Warning("cannot register analyzer for port %u", std::get<2>(p)); + } + + pending_analyzers_for_ports.clear(); + + initialized = true; } void Manager::DumpDebug() @@ -231,6 +240,22 @@ bool Manager::UnregisterAnalyzerForPort(EnumVal* val, PortVal* port) bool Manager::RegisterAnalyzerForPort(const Tag& tag, TransportProto proto, uint32_t port) { + if ( initialized ) + return RegisterAnalyzerForPort(std::make_tuple(tag, proto, port)); + else + { + // Cannot register these before PostScriptInit() has run because we + // depend on packet analyis having been set up. That also means we don't have + // a reliable return value, for now we just assume it's working. + pending_analyzers_for_ports.emplace(tag, proto, port); + return true; + } + } + +bool Manager::RegisterAnalyzerForPort(const std::tuple& p) + { + const auto& [tag, proto, port] = p; + // TODO: this class is becoming more generic and removing a lot of the // checks for protocols, but this part might need to stay like this. packet_analysis::AnalyzerPtr analyzer; @@ -249,6 +274,9 @@ bool Manager::RegisterAnalyzerForPort(const Tag& tag, TransportProto proto, uint bool Manager::UnregisterAnalyzerForPort(const Tag& tag, TransportProto proto, uint32_t port) { + if ( auto i = pending_analyzers_for_ports.find(std::make_tuple(tag, proto, port)); i != pending_analyzers_for_ports.end() ) + pending_analyzers_for_ports.erase(i); + // TODO: this class is becoming more generic and removing a lot of the // checks for protocols, but this part might need to stay like this. packet_analysis::AnalyzerPtr analyzer; diff --git a/src/analyzer/Manager.h b/src/analyzer/Manager.h index 7b1ac05be4..17a0075a2d 100644 --- a/src/analyzer/Manager.h +++ b/src/analyzer/Manager.h @@ -335,6 +335,8 @@ public: { return vxlan_ports; } private: + // Internal version that must be used only once InitPostScript has completed. + bool RegisterAnalyzerForPort(const std::tuple& p); friend class packet_analysis::IP::IPBasedAnalyzer; @@ -372,11 +374,16 @@ private: }; }; + + using protocol_analyzers = std::set>; using conns_map = std::multimap; using conns_queue = std::priority_queue, ScheduledAnalyzer::Comparator>; + bool initialized = false; + protocol_analyzers pending_analyzers_for_ports; + conns_map conns; conns_queue conns_by_timeout; std::vector vxlan_ports; diff --git a/src/analyzer/protocol/asn1/asn1.pac b/src/analyzer/protocol/asn1/asn1.pac index d28e531ad2..35a27e88ba 100644 --- a/src/analyzer/protocol/asn1/asn1.pac +++ b/src/analyzer/protocol/asn1/asn1.pac @@ -87,10 +87,18 @@ type Array = record { ############################## ASN.1 Conversion Functions +# Converts an 8-byte string into an int64. If this string is longer than +# 8 bytes, it reports a weird and returns zero. function binary_to_int64(bs: bytestring): int64 %{ int64 rval = 0; + if ( bs.length() > 8 ) + { + zeek::reporter->Weird("asn_binary_to_int64_shift_too_large", zeek::util::fmt("%d", bs.length())); + return 0; + } + for ( int i = 0; i < bs.length(); ++i ) { uint64 byte = bs[i]; diff --git a/src/analyzer/protocol/dns/DNS.cc b/src/analyzer/protocol/dns/DNS.cc index 3ea8ab4550..a9ca599939 100644 --- a/src/analyzer/protocol/dns/DNS.cc +++ b/src/analyzer/protocol/dns/DNS.cc @@ -1644,18 +1644,21 @@ bool DNS_Interpreter::ParseRR_AAAA(detail::DNS_MsgInfo* msg, bool DNS_Interpreter::ParseRR_WKS(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength) { - data += rdlength; - len -= rdlength; + if ( ! dns_WKS_reply || msg->skip_event ) + { + data += rdlength; + len -= rdlength; + return true; + } - return true; - } - -bool DNS_Interpreter::ParseRR_HINFO(detail::DNS_MsgInfo* msg, - const u_char*& data, int& len, int rdlength) - { - data += rdlength; - len -= rdlength; + // TODO: Pass the ports as parameters to the event + analyzer->EnqueueConnEvent(dns_WKS_reply, + analyzer->ConnVal(), + msg->BuildHdrVal(), + msg->BuildAnswerVal() + ); + // TODO: Return a status which reflects if the port parameters were successfully parsed return true; } @@ -1687,6 +1690,28 @@ extract_char_string(analyzer::Analyzer* analyzer, return rval; } +bool DNS_Interpreter::ParseRR_HINFO(detail::DNS_MsgInfo* msg, + const u_char*& data, int& len, int rdlength) + { + if ( ! dns_HINFO_reply || msg->skip_event ) + { + data += rdlength; + len -= rdlength; + return true; + } + + auto cpu = extract_char_string(analyzer, data, len, rdlength); + auto os = extract_char_string(analyzer, data, len, rdlength); + + analyzer->EnqueueConnEvent(dns_HINFO_reply, + analyzer->ConnVal(), + msg->BuildHdrVal(), + msg->BuildAnswerVal(), + cpu, os); + + return rdlength == 0; + } + bool DNS_Interpreter::ParseRR_TXT(detail::DNS_MsgInfo* msg, const u_char*& data, int& len, int rdlength, const u_char* msg_start) diff --git a/src/analyzer/protocol/dns/events.bif b/src/analyzer/protocol/dns/events.bif index 1c5dbcd11e..40c6cf9f66 100644 --- a/src/analyzer/protocol/dns/events.bif +++ b/src/analyzer/protocol/dns/events.bif @@ -334,7 +334,8 @@ event dns_WKS_reply%(c: connection, msg: dns_msg, ans: dns_answer%); ## dns_mapping_valid dns_message dns_query_reply dns_rejected dns_request ## dns_max_queries dns_session_timeout dns_skip_addl ## dns_skip_all_addl dns_skip_all_auth dns_skip_auth -event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%); +event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer, cpu: string, os: string%); +event dns_HINFO_reply%(c: connection, msg: dns_msg, ans: dns_answer%) &deprecated="Remove in v5.2. Use the definition with the extra parameters for cpu and os."; ## Generated for DNS replies of type *MX*. For replies with multiple answers, an ## individual event of the corresponding type is raised for each. diff --git a/src/analyzer/protocol/geneve/Geneve.cc b/src/analyzer/protocol/geneve/Geneve.cc index c3db3109bb..17a058551a 100644 --- a/src/analyzer/protocol/geneve/Geneve.cc +++ b/src/analyzer/protocol/geneve/Geneve.cc @@ -47,7 +47,7 @@ void Geneve_Analyzer::DeliverPacket(int len, const u_char* data, bool orig, EncapsulatingConn inner(Conn(), BifEnum::Tunnel::GENEVE); outer->Add(inner); - auto tunnel_opt_len = data[0] << 1; + uint8_t tunnel_opt_len = (data[0] & 0x3F) * 4; auto vni = (data[4] << 16) + (data[5] << 8) + (data[6] << 0); if ( len < tunnel_header_len + tunnel_opt_len ) diff --git a/src/analyzer/protocol/http/HTTP.cc b/src/analyzer/protocol/http/HTTP.cc index 62b2216179..e82c044cc8 100644 --- a/src/analyzer/protocol/http/HTTP.cc +++ b/src/analyzer/protocol/http/HTTP.cc @@ -166,6 +166,7 @@ void HTTP_Entity::Deliver(int len, const char* data, bool trailing_CRLF) if ( expect_data_length <= 0 ) { SetPlainDelivery(0); + http_message->SetDeliverySize(-1); EndOfData(); } } @@ -527,6 +528,9 @@ void HTTP_Entity::SubmitAllHeaders() http_message->content_line->SetCRLFAsEOL(0); } + if (content_length >= 0 ) + http_message->SetDeliverySize(content_length); + // The presence of a message-body in a request is signaled by // the inclusion of a Content-Length or Transfer-Encoding // header field in the request's message-headers. @@ -825,6 +829,11 @@ void HTTP_Message::SetPlainDelivery(int64_t length) content_line->SkipBytesAfterThisLine(length); } +void HTTP_Message::SetDeliverySize(int64_t length) + { + content_line->SetDeliverySize(length); + } + void HTTP_Message::SkipEntityData() { if ( current_entity ) diff --git a/src/analyzer/protocol/http/HTTP.h b/src/analyzer/protocol/http/HTTP.h index 927c19df97..713538247c 100644 --- a/src/analyzer/protocol/http/HTTP.h +++ b/src/analyzer/protocol/http/HTTP.h @@ -120,6 +120,7 @@ public: void SubmitTrailingHeaders(analyzer::mime::MIME_HeaderList& /* hlist */); void SetPlainDelivery(int64_t length); + void SetDeliverySize(int64_t length); void SkipEntityData(); HTTP_Analyzer* MyHTTP_Analyzer() const diff --git a/src/analyzer/protocol/tcp/ContentLine.cc b/src/analyzer/protocol/tcp/ContentLine.cc index 97d9a33e62..0d7c1b9a4a 100644 --- a/src/analyzer/protocol/tcp/ContentLine.cc +++ b/src/analyzer/protocol/tcp/ContentLine.cc @@ -31,6 +31,7 @@ void ContentLine_Analyzer::InitState() seq = 0; seq_to_skip = 0; plain_delivery_length = 0; + delivery_length = -1; is_plain = false; suppress_weirds = false; @@ -91,8 +92,23 @@ void ContentLine_Analyzer::DeliverStream(int len, const u_char* data, return; } + if ( delivery_length > 0 ) + delivery_length -= len; + DoDeliver(len, data); + // If we have parsed all the data of the packet but there is no CRLF at the end + // Force the process by flushing buffer + if ( delivery_length == 0 ) + { + if (HasPartialLine()) + { + Weird("line_terminated_without_CRLF"); + DoDeliver(2, (const u_char*) "\r\n"); + } + delivery_length = -1; + } + seq += len; } @@ -119,6 +135,18 @@ void ContentLine_Analyzer::SetPlainDelivery(int64_t length) plain_delivery_length = length; } +void ContentLine_Analyzer::SetDeliverySize(int64_t length) + { + // Length can be unset with -1 value, all other negative length will be rejected + if ( length < -1 ) + { + reporter->AnalyzerError( this, "negative length for delivery size"); + return; + } + + delivery_length = length; + } + void ContentLine_Analyzer::DoDeliver(int len, const u_char* data) { seq_delivered_in_lines = seq; diff --git a/src/analyzer/protocol/tcp/ContentLine.h b/src/analyzer/protocol/tcp/ContentLine.h index c69ed7673b..f58e273ae6 100644 --- a/src/analyzer/protocol/tcp/ContentLine.h +++ b/src/analyzer/protocol/tcp/ContentLine.h @@ -50,6 +50,7 @@ public: // via DeliverStream() and can differentiated by calling // IsPlainDelivery(). void SetPlainDelivery(int64_t length); + void SetDeliverySize(int64_t length); int64_t GetPlainDeliveryLength() const { return plain_delivery_length; } bool IsPlainDelivery() { return is_plain; } @@ -97,6 +98,8 @@ protected: // Remaining bytes to deliver plain. int64_t plain_delivery_length; + // Remaining bytes to deliver + int64_t delivery_length; bool is_plain; // Don't deliver further data. diff --git a/src/broker/comm.bif b/src/broker/comm.bif index a6f7876766..2b6d4be802 100644 --- a/src/broker/comm.bif +++ b/src/broker/comm.bif @@ -117,12 +117,12 @@ function Broker::__peers%(%): PeerInfos if ( n ) { - network_info->Assign(0, zeek::make_intrusive(IPAddr(n->address))); + network_info->Assign(0, zeek::make_intrusive(n->address)); network_info->Assign(1, zeek::val_mgr->Port(n->port, TRANSPORT_TCP)); } else { - network_info->Assign(0, zeek::make_intrusive("0.0.0.0")); + network_info->Assign(0, zeek::make_intrusive("0.0.0.0")); network_info->Assign(1, zeek::val_mgr->Port(0, TRANSPORT_TCP)); } diff --git a/src/packet_analysis/packet_analysis.bif b/src/packet_analysis/packet_analysis.bif index cc60808d72..8298b3946f 100644 --- a/src/packet_analysis/packet_analysis.bif +++ b/src/packet_analysis/packet_analysis.bif @@ -4,6 +4,7 @@ module PacketAnalyzer; #include "zeek/packet_analysis/Analyzer.h" #include "zeek/packet_analysis/Manager.h" +#include "zeek/packet_analysis/protocol/ip/IPBasedAnalyzer.h" %%} @@ -47,3 +48,13 @@ function try_register_packet_analyzer_by_name%(parent: string, identifier: count parent_analyzer->RegisterProtocol(identifier, child_analyzer); return zeek::val_mgr->True(); %} + +## Internal function that is used to update the core-mirror of the script-level `ignore_checksums_nets` variable. +function PacketAnalyzer::__set_ignore_checksums_nets%(v: subnet_set%) : bool + %{ + if ( v->GetType()->Tag() != zeek::TYPE_TABLE ) + zeek::emit_builtin_error("update_ignore_checksums_net() requires a table/set argument"); + + zeek::packet_analysis::IP::IPBasedAnalyzer::SetIgnoreChecksumsNets(zeek::IntrusivePtr{zeek::NewRef{}, v->AsTableVal()}); + return zeek::val_mgr->True(); + %} diff --git a/src/packet_analysis/protocol/icmp/ICMP.cc b/src/packet_analysis/protocol/icmp/ICMP.cc index b4ef187cd3..a824d3c1b1 100644 --- a/src/packet_analysis/protocol/icmp/ICMP.cc +++ b/src/packet_analysis/protocol/icmp/ICMP.cc @@ -77,7 +77,7 @@ void ICMPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int rema const std::unique_ptr& ip = pkt->ip_hdr; if ( ! zeek::detail::ignore_checksums && - ! zeek::id::find_val("ignore_checksums_nets")->Contains(ip->IPHeaderSrcAddr()) && + ! GetIgnoreChecksumsNets()->Contains(ip->IPHeaderSrcAddr()) && remaining >= len ) { int chksum = 0; diff --git a/src/packet_analysis/protocol/ip/IP.cc b/src/packet_analysis/protocol/ip/IP.cc index b957f9fce6..e7f84c2675 100644 --- a/src/packet_analysis/protocol/ip/IP.cc +++ b/src/packet_analysis/protocol/ip/IP.cc @@ -1,6 +1,7 @@ // See the file "COPYING" in the main distribution directory for copyright. #include "zeek/packet_analysis/protocol/ip/IP.h" +#include "zeek/packet_analysis/protocol/ip/IPBasedAnalyzer.h" #include "zeek/NetVar.h" #include "zeek/IP.h" #include "zeek/Discard.h" @@ -128,7 +129,7 @@ bool IPAnalyzer::AnalyzePacket(size_t len, const uint8_t* data, Packet* packet) return false; if ( ! packet->l2_checksummed && ! detail::ignore_checksums && ip4 && - ! zeek::id::find_val("ignore_checksums_nets")->Contains(packet->ip_hdr->IPHeaderSrcAddr()) && + ! IPBasedAnalyzer::GetIgnoreChecksumsNets()->Contains(packet->ip_hdr->IPHeaderSrcAddr()) && detail::in_cksum(reinterpret_cast(ip4), ip_hdr_len) != 0xffff ) { Weird("bad_IP_checksum", packet); diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc index 287736d6b1..8940751fe5 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.cc @@ -282,3 +282,18 @@ void IPBasedAnalyzer::DumpPortDebug() DBG_LOG(DBG_ANALYZER, " %d/%s: %s", mapping.first, transport_proto_string(transport), s.c_str()); } } + +TableValPtr IPBasedAnalyzer::ignore_checksums_nets_table = nullptr; + +void IPBasedAnalyzer::SetIgnoreChecksumsNets(TableValPtr t) + { + IPBasedAnalyzer::ignore_checksums_nets_table = t; + } + +TableValPtr IPBasedAnalyzer::GetIgnoreChecksumsNets() + { + if ( ! IPBasedAnalyzer::ignore_checksums_nets_table ) + IPBasedAnalyzer::ignore_checksums_nets_table = zeek::id::find_val("ignore_checksums_nets"); + return IPBasedAnalyzer::ignore_checksums_nets_table; + } + diff --git a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h index 7ff54a4f09..3f62c5c362 100644 --- a/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h +++ b/src/packet_analysis/protocol/ip/IPBasedAnalyzer.h @@ -7,6 +7,7 @@ #include "zeek/packet_analysis/Analyzer.h" #include "zeek/analyzer/Tag.h" +#include "zeek/ID.h" namespace zeek::analyzer::pia { class PIA; } @@ -61,6 +62,25 @@ public: */ void DumpPortDebug(); + /** + * Updates the internal pointer to the script-level variable `ignore_checksums_nets`. + * This is used to prevent repeated (costly) lookup of the script-level variable + * by IP-based analyzers. + * + * @param t New value of ignore_checksums_nets + */ + static void SetIgnoreChecksumsNets(TableValPtr t); + + + /** + * Gets the interpal pointer to the script-level variable `ignore_checksums_nets`. + * This is used to prevent repeated (costly) lookup of the script-level variable + * by IP-based analyzers. + * + * @return Current value of `ignore_checksums_nets`. + */ + static TableValPtr GetIgnoreChecksumsNets(); + protected: /** @@ -178,6 +198,7 @@ private: TransportProto transport; uint32_t server_port_mask; + static TableValPtr ignore_checksums_nets_table; }; } diff --git a/src/packet_analysis/protocol/tcp/TCP.cc b/src/packet_analysis/protocol/tcp/TCP.cc index 00d4f23a70..3a6de91e41 100644 --- a/src/packet_analysis/protocol/tcp/TCP.cc +++ b/src/packet_analysis/protocol/tcp/TCP.cc @@ -18,7 +18,6 @@ TCPAnalyzer::TCPAnalyzer() : IPBasedAnalyzer("TCP", TRANSPORT_TCP, TCP_PORT_MASK void TCPAnalyzer::Initialize() { - ignored_nets = zeek::id::find_val("ignore_checksums_nets"); } SessionAdapter* TCPAnalyzer::MakeSessionAdapter(Connection* conn) @@ -164,7 +163,7 @@ bool TCPAnalyzer::ValidateChecksum(const IP_Hdr* ip, const struct tcphdr* tp, { if ( ! run_state::current_pkt->l3_checksummed && ! detail::ignore_checksums && - ! ignored_nets->Contains(ip->IPHeaderSrcAddr()) && + ! GetIgnoreChecksumsNets()->Contains(ip->IPHeaderSrcAddr()) && caplen >= len && ! endpoint->ValidChecksum(tp, len, ip->IP4_Hdr()) ) { adapter->Weird("bad_TCP_checksum"); diff --git a/src/packet_analysis/protocol/tcp/TCP.h b/src/packet_analysis/protocol/tcp/TCP.h index 0dd8e88a3c..eb013047b6 100644 --- a/src/packet_analysis/protocol/tcp/TCP.h +++ b/src/packet_analysis/protocol/tcp/TCP.h @@ -86,8 +86,6 @@ private: bool ValidateChecksum(const IP_Hdr* ip, const struct tcphdr* tp, analyzer::tcp::TCP_Endpoint* endpoint, int len, int caplen, TCPSessionAdapter* adapter); - - TableValPtr ignored_nets; }; } diff --git a/src/packet_analysis/protocol/udp/UDP.cc b/src/packet_analysis/protocol/udp/UDP.cc index f798229118..4d1e8cb374 100644 --- a/src/packet_analysis/protocol/udp/UDP.cc +++ b/src/packet_analysis/protocol/udp/UDP.cc @@ -108,7 +108,7 @@ void UDPAnalyzer::DeliverPacket(Connection* c, double t, bool is_orig, int remai auto validate_checksum = ! run_state::current_pkt->l3_checksummed && ! zeek::detail::ignore_checksums && - ! zeek::id::find_val("ignore_checksums_nets")->Contains(ip->IPHeaderSrcAddr()) && + ! GetIgnoreChecksumsNets()->Contains(ip->IPHeaderSrcAddr()) && remaining >=len; constexpr auto vxlan_len = 8; diff --git a/src/patricia.c b/src/patricia.c index 842436c006..7aa265c0fb 100644 --- a/src/patricia.c +++ b/src/patricia.c @@ -1,3 +1,16 @@ +/* + * This code originates from Dave Plonka's Net::Security perl module. An adaptation + * of it in C is kept at https://github.com/CAIDA/cc-common/tree/master/libpatricia. + * That repository is considered the upstream version for Zeek's fork. We make some + * custom changes to this upstream: + * - Replaces void_fn_t with data_fn_t and prefix_data_fn_t + * - Adds patricia_search_all method + * - One commented-out portion of an if statement that breaks one of our tests + * + * The current version is based on commit 4a2c61374f507a420d28bd9084c976142d279605 + * from that repo. + */ + /* * Johanna Amann * @@ -52,7 +65,15 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ -static char copyright[] = +#ifndef UNUSED +# if __GNUC__ >= 3 +# define UNUSED __attribute__((unused)) +# else +# define UNUSED +# endif +#endif + +UNUSED static char copyright[] = "This product includes software developed by the University of Michigan, Merit" "Network, Inc., and their contributors."; @@ -64,16 +85,17 @@ static char copyright[] = #include /* sprintf, fprintf, stderr */ #include /* free, atol, calloc */ #include /* memcpy, strchr, strlen */ -#include /* for struct in_addr */ -#include /* for inet_addr */ -#include /* for u_short, etc. */ +#include /* BSD: for inet_addr */ +#include /* BSD, Linux: for inet_addr */ +#include /* BSD, Linux: for inet_addr */ +#include /* BSD, Linux, Solaris: for inet_addr */ #include #include "zeek/patricia.h" #define Delete free -// From Bro for reporting memory exhaustion. +// From Zeek for reporting memory exhaustion. extern void out_of_memory(const char* where); /* { from prefix.c */ @@ -104,47 +126,13 @@ comp_with_mask (void *addr, void *dest, u_int mask) return (0); } -/* inet_pton substitute implementation - * Uses inet_addr to convert an IP address in dotted decimal notation into - * unsigned long and copies the result to dst. - * Only supports AF_INET. Follows standard error return conventions of - * inet_pton. - */ -int -local_inet_pton (int af, const char *src, void *dst) -{ - u_long result; - - if (af == AF_INET) { - result = inet_addr(src); - if (result == -1) - return 0; - else { - memcpy (dst, &result, 4); - return 1; - } - } - else if (af == AF_INET6) { -#ifdef NT - struct in6_addr Address; - return (inet6_addr(src, &Address)); -#else - return inet_pton(AF_INET6, src, dst); -#endif /* NT */ - } - else { - errno = EAFNOSUPPORT; - return -1; - } -} - /* this allows imcomplete prefix */ int my_inet_pton (int af, const char *src, void *dst) { if (af == AF_INET) { int i, c, val; - u_char xp[4] = {0, 0, 0, 0}; + u_char xp[sizeof(struct in_addr)] = {0, 0, 0, 0}; for (i = 0; ; i++) { c = *src++; @@ -165,10 +153,10 @@ my_inet_pton (int af, const char *src, void *dst) if (i >= 3) return (0); } - memcpy (dst, xp, 4); + memcpy (dst, xp, sizeof(struct in_addr)); return (1); } else if (af == AF_INET6) { - return (local_inet_pton (af, src, dst)); + return (inet_pton (af, src, dst)); } else { #ifndef NT errno = EAFNOSUPPORT; @@ -177,6 +165,8 @@ my_inet_pton (int af, const char *src, void *dst) } } +#define PATRICIA_MAX_THREADS 16 + /* * convert prefix information to ascii string with length * thread safe and (almost) re-entrant implementation @@ -190,7 +180,7 @@ prefix_toa2x (prefix_t *prefix, char *buff, int with_len) if (buff == NULL) { struct buffer { - char buffs[16][48+5]; + char buffs[PATRICIA_MAX_THREADS][48+5]; u_int i; } *buffp; @@ -207,11 +197,11 @@ prefix_toa2x (prefix_t *prefix, char *buff, int with_len) return (NULL); } - buff = buffp->buffs[buffp->i++%16]; + buff = buffp->buffs[buffp->i++%PATRICIA_MAX_THREADS]; } if (prefix->family == AF_INET) { u_char *a; - assert (prefix->bitlen <= 32); + assert (prefix->bitlen <= sizeof(struct in_addr) * 8); a = prefix_touchar (prefix); if (with_len) { sprintf (buff, "%d.%d.%d.%d/%d", a[0], a[1], a[2], a[3], @@ -226,7 +216,7 @@ prefix_toa2x (prefix_t *prefix, char *buff, int with_len) char *r; r = (char *) inet_ntop (AF_INET6, &prefix->add.sin6, buff, 48 /* a guess value */ ); if (r && with_len) { - assert (prefix->bitlen <= 128); + assert (prefix->bitlen <= sizeof(struct in6_addr) * 8); sprintf (buff + strlen (buff), "/%d", prefix->bitlen); } return (buff); @@ -256,10 +246,10 @@ prefix_t * New_Prefix2 (int family, void *dest, int bitlen, prefix_t *prefix) { int dynamic_allocated = 0; - int default_bitlen = 32; + int default_bitlen = sizeof(struct in_addr) * 8; if (family == AF_INET6) { - default_bitlen = 128; + default_bitlen = sizeof(struct in6_addr) * 8; if (prefix == NULL) { prefix = calloc(1, sizeof (prefix_t)); if (prefix == NULL) @@ -267,7 +257,7 @@ New_Prefix2 (int family, void *dest, int bitlen, prefix_t *prefix) dynamic_allocated++; } - memcpy (&prefix->add.sin6, dest, 16); + memcpy (&prefix->add.sin6, dest, sizeof(struct in6_addr)); } else if (family == AF_INET) { @@ -286,7 +276,7 @@ New_Prefix2 (int family, void *dest, int bitlen, prefix_t *prefix) dynamic_allocated++; } - memcpy (&prefix->add.sin, dest, 4); + memcpy (&prefix->add.sin, dest, sizeof(struct in_addr)); } else { return (NULL); @@ -330,10 +320,10 @@ ascii2prefix (int family, char *string) } if (family == AF_INET) { - maxbitlen = 32; + maxbitlen = sizeof(struct in_addr) * 8; } else if (family == AF_INET6) { - maxbitlen = 128; + maxbitlen = sizeof(struct in6_addr) * 8; } if ((cp = strchr (string, '/')) != NULL) { @@ -363,7 +353,7 @@ ascii2prefix (int family, char *string) inet6_addr(string, &sin6); return (New_Prefix (AF_INET6, &sin6, bitlen)); #else - if ((result = local_inet_pton (AF_INET6, string, &sin6)) <= 0) + if ((result = inet_pton (AF_INET6, string, &sin6)) <= 0) return (NULL); #endif /* NT */ return (New_Prefix (AF_INET6, &sin6, bitlen)); @@ -466,7 +456,7 @@ Clear_Patricia (patricia_tree_t *patricia, data_fn_t func) } else if (Xsp != Xstack) { Xrn = *(--Xsp); } else { - Xrn = (patricia_node_t *) 0; + Xrn = NULL; } } } @@ -499,6 +489,28 @@ patricia_process (patricia_tree_t *patricia, prefix_data_fn_t func) } PATRICIA_WALK_END; } +size_t +patricia_walk_inorder(patricia_node_t *node, prefix_data_fn_t func) +{ + size_t n = 0; + assert(func); + + if (node->l) { + n += patricia_walk_inorder(node->l, func); + } + + if (node->prefix) { + func(node->prefix, node->data); + n++; + } + + if (node->r) { + n += patricia_walk_inorder(node->r, func); + } + + return n; +} + patricia_node_t * patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix) @@ -526,7 +538,7 @@ patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix) fprintf (stderr, "patricia_search_exact: take right %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_search_exact: take right at %d\n", + fprintf (stderr, "patricia_search_exact: take right at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ node = node->r; @@ -537,7 +549,7 @@ patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix) fprintf (stderr, "patricia_search_exact: take left %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_search_exact: take left at %d\n", + fprintf (stderr, "patricia_search_exact: take left at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ node = node->l; @@ -552,7 +564,7 @@ patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix) fprintf (stderr, "patricia_search_exact: stop at %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_search_exact: stop at %d\n", node->bit); + fprintf (stderr, "patricia_search_exact: stop at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ if (node->bit > bitlen || node->prefix == NULL) return (NULL); @@ -708,7 +720,7 @@ patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusiv fprintf (stderr, "patricia_search_best: take right %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_search_best: take right at %d\n", + fprintf (stderr, "patricia_search_best: take right at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ node = node->r; @@ -719,7 +731,7 @@ patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusiv fprintf (stderr, "patricia_search_best: take left %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_search_best: take left at %d\n", + fprintf (stderr, "patricia_search_best: take left at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ node = node->l; @@ -739,7 +751,7 @@ patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusiv fprintf (stderr, "patricia_search_best: stop at %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_search_best: stop at %d\n", node->bit); + fprintf (stderr, "patricia_search_best: stop at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ if (cnt <= 0) @@ -752,8 +764,8 @@ patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusiv prefix_toa (node->prefix), node->prefix->bitlen); #endif /* PATRICIA_DEBUG */ if (comp_with_mask (prefix_tochar (node->prefix), - prefix_tochar (prefix), - node->prefix->bitlen)) { + prefix_tochar (prefix), + node->prefix->bitlen) && node->prefix->bitlen <= bitlen) { #ifdef PATRICIA_DEBUG fprintf (stderr, "patricia_search_best: found %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); @@ -818,7 +830,7 @@ patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix) fprintf (stderr, "patricia_lookup: take right %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_lookup: take right at %d\n", node->bit); + fprintf (stderr, "patricia_lookup: take right at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ node = node->r; } @@ -830,7 +842,7 @@ patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix) fprintf (stderr, "patricia_lookup: take left %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_lookup: take left at %d\n", node->bit); + fprintf (stderr, "patricia_lookup: take left at %u\n", node->bit); #endif /* PATRICIA_DEBUG */ node = node->l; } @@ -878,7 +890,7 @@ patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix) fprintf (stderr, "patricia_lookup: up to %s/%d\n", prefix_toa (node->prefix), node->prefix->bitlen); else - fprintf (stderr, "patricia_lookup: up to %d\n", node->bit); + fprintf (stderr, "patricia_lookup: up to %u\n", node->bit); #endif /* PATRICIA_DEBUG */ } @@ -1156,7 +1168,7 @@ try_search_best (patricia_tree_t *tree, char *string) printf ("try_search_best: %s/%d found\n", prefix_toa (node->prefix), node->prefix->bitlen); Deref_Prefix (prefix); - return 0; // [RS] What is supposed to be returned here? - } + return (node); +} /* } */ diff --git a/src/patricia.h b/src/patricia.h index a32c5e3e81..d4374f2b0b 100644 --- a/src/patricia.h +++ b/src/patricia.h @@ -1,8 +1,20 @@ +/* + * This code originates from Dave Plonka's Net::Security perl module. An adaptation + * of it in C is kept at https://github.com/CAIDA/cc-common/tree/master/libpatricia. + * That repository is considered the upstream version for Zeek's fork. We make some + * custom changes to this upstream: + * - Replace void_fn_t with data_fn_t and prefix_data_fn_t + * - Add patricia_search_all method + * + * The current version is based on commit 4a2c61374f507a420d28bd9084c976142d279605 + * from that repo. + */ + /* * Dave Plonka * * This product includes software developed by the University of Michigan, - * Merit Network, Inc., and their contributors. + * Merit Network, Inc., and their contributors. * * This file had been called "radix.h" in the MRT sources. * @@ -12,28 +24,28 @@ */ /* From copyright.txt: - * + * * Copyright (c) 1997, 1998, 1999 - * - * + * + * * The Regents of the University of Michigan ("The Regents") and Merit Network, * Inc. All rights reserved. * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: - * 1. Redistributions of source code must retain the above - * copyright notice, this list of conditions and the + * 1. Redistributions of source code must retain the above + * copyright notice, this list of conditions and the * following disclaimer. - * 2. Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the - * following disclaimer in the documentation and/or other + * 2. Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the + * following disclaimer in the documentation and/or other * materials provided with the distribution. - * 3. All advertising materials mentioning features or use of - * this software must display the following acknowledgement: + * 3. All advertising materials mentioning features or use of + * this software must display the following acknowledgement: * This product includes software developed by the University of Michigan, Merit - * Network, Inc., and their contributors. + * Network, Inc., and their contributors. * 4. Neither the name of the University, Merit Network, nor the - * names of their contributors may be used to endorse or - * promote products derived from this software without + * names of their contributors may be used to endorse or + * promote products derived from this software without * specific prior written permission. * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS "AS IS" AND ANY * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED @@ -44,13 +56,11 @@ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS - * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #pragma once -#include - /* { from defs.h */ #define prefix_touchar(prefix) ((u_char *)&(prefix)->add.sin) #define MAXLINE 1024 @@ -59,7 +69,15 @@ #define addroute make_and_lookup -#include /* for struct in_addr */ +#include /* for u_* definitions (on FreeBSD 5) */ + +#include /* for EAFNOSUPPORT */ +#ifndef EAFNOSUPPORT +# defined EAFNOSUPPORT WSAEAFNOSUPPORT +# include +#else +# include /* for struct in_addr */ +#endif #include /* for AF_INET */ @@ -106,16 +124,18 @@ typedef struct _patricia_tree_t { patricia_node_t *patricia_search_exact (patricia_tree_t *patricia, prefix_t *prefix); bool patricia_search_all (patricia_tree_t *patricia, prefix_t *prefix, patricia_node_t ***list, int *n); patricia_node_t *patricia_search_best (patricia_tree_t *patricia, prefix_t *prefix); -patricia_node_t * patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, +patricia_node_t * patricia_search_best2 (patricia_tree_t *patricia, prefix_t *prefix, int inclusive); patricia_node_t *patricia_lookup (patricia_tree_t *patricia, prefix_t *prefix); void patricia_remove (patricia_tree_t *patricia, patricia_node_t *node); patricia_tree_t *New_Patricia (int maxbits); void Clear_Patricia (patricia_tree_t *patricia, data_fn_t func); void Destroy_Patricia (patricia_tree_t *patricia, data_fn_t func); + void patricia_process (patricia_tree_t *patricia, prefix_data_fn_t func); void Deref_Prefix (prefix_t * prefix); +char *prefix_toa (prefix_t * prefix); /* { from demo.c */ @@ -127,7 +147,7 @@ make_and_lookup (patricia_tree_t *tree, char *string); /* } */ -#define PATRICIA_MAXBITS 128 +#define PATRICIA_MAXBITS (sizeof(struct in6_addr) * 8) #define PATRICIA_NBIT(x) (0x80 >> ((x) & 0x7f)) #define PATRICIA_NBYTE(x) ((x) >> 3) diff --git a/src/session/Key.cc b/src/session/Key.cc index 5c7317505c..f87acc76b4 100644 --- a/src/session/Key.cc +++ b/src/session/Key.cc @@ -70,4 +70,14 @@ bool Key::operator<(const Key& rhs) const return memcmp(data, rhs.data, size) < 0; } +bool Key::operator==(const Key& rhs) const + { + if ( size != rhs.size ) + return false; + else if ( type != rhs.type ) + return false; + + return memcmp(data, rhs.data, size) == 0; + } + } // namespace zeek::session::detail diff --git a/src/session/Key.h b/src/session/Key.h index 7201fbcaa5..ca37d31088 100644 --- a/src/session/Key.h +++ b/src/session/Key.h @@ -4,9 +4,12 @@ #include #include +#include "zeek/Hash.h" namespace zeek::session::detail { +struct KeyHash; + /** * This type is used as the key for the map in SessionManager. It represents a * raw block of memory that points to a key of some type for a session, such as @@ -57,12 +60,23 @@ public: void CopyData(); bool operator<(const Key& rhs) const; + bool operator==(const Key& rhs) const; + + std::size_t Hash() const { + return zeek::detail::HashKey::HashBytes(data, size); + } private: + friend struct KeyHash; + const uint8_t* data = nullptr; size_t size = 0; size_t type = CONNECTION_KEY_TYPE; bool copied = false; }; +struct KeyHash { + std::size_t operator()(const Key& k) const { return k.Hash(); } +}; + } // namespace zeek::session::detail diff --git a/src/session/Manager.cc b/src/session/Manager.cc index 527adfed74..6fdf275004 100644 --- a/src/session/Manager.cc +++ b/src/session/Manager.cc @@ -218,11 +218,34 @@ void Manager::Insert(Session* s, bool remove_existing) void Manager::Drain() { - for ( const auto& entry : session_map ) + // If a random seed was passed in, we're most likely in testing mode and need the + // order of the sessions to be consistent. Sort the keys to force that order + // every run. + if ( zeek::util::detail::have_random_seed() ) { - Session* tc = entry.second; - tc->Done(); - tc->RemovalEvent(); + std::vector keys; + keys.reserve(session_map.size()); + + for ( auto& entry : session_map ) + keys.push_back(&(entry.first)); + std::sort(keys.begin(), keys.end(), [](const detail::Key* a, const detail::Key* b) { + return *a < *b; }); + + for ( const auto* k : keys ) + { + Session* tc = session_map.at(*k); + tc->Done(); + tc->RemovalEvent(); + } + } + else + { + for ( const auto& entry : session_map ) + { + Session* tc = entry.second; + tc->Done(); + tc->RemovalEvent(); + } } } diff --git a/src/session/Manager.h b/src/session/Manager.h index 976cc7f10d..e824e0b14b 100644 --- a/src/session/Manager.h +++ b/src/session/Manager.h @@ -3,7 +3,7 @@ #pragma once #include // for u_char -#include +#include #include #include "zeek/Frag.h" @@ -119,7 +119,7 @@ public: private: - using SessionMap = std::map; + using SessionMap = std::unordered_map; // Inserts a new connection into the sessions map. If a connection with // the same key already exists in the map, it will be overwritten by diff --git a/src/util.cc b/src/util.cc index 260866d30f..9a2b83c7bf 100644 --- a/src/util.cc +++ b/src/util.cc @@ -885,9 +885,10 @@ double calc_next_rotate(double current, double interval, double base) double startofday = mktime(&t); // current < startofday + base + i * interval <= current + interval - return startofday + base + + double delta_t = startofday + base + ceil((current - startofday - base) / interval) * interval - current; + return delta_t > 0.0 ? delta_t: interval; } void terminate_processing() @@ -2351,7 +2352,7 @@ TEST_CASE("util json_escape_utf8") CHECK(json_escape_utf8("string") == "string"); CHECK(json_escape_utf8("string\n") == "string\n"); CHECK(json_escape_utf8("string\x82") == "string\\x82"); - CHECK(json_escape_utf8("\x07\xd4\xb7o") == "\\x07Էo"); + CHECK(json_escape_utf8("\x07\xd4\xb7o") == "\\x07\\xd4\\xb7o"); // These strings are duplicated from the scripts.base.frameworks.logging.ascii-json-utf8 btest @@ -2405,6 +2406,37 @@ TEST_CASE("util json_escape_utf8") // Invalid 4 Octet Sequence (too short) CHECK(json_escape_utf8("\xf4\x80\x8c") == "\\xf4\\x80\\x8c"); CHECK(json_escape_utf8("\xf0") == "\\xf0"); + + // Private Use Area (E000-F8FF) are always invalid + CHECK(json_escape_utf8("\xee\x8b\xa0") == "\\xee\\x8b\\xa0"); + + // Valid UTF-8 character followed by an invalid one + CHECK(json_escape_utf8("\xc3\xb1\xc0\x81") == "\\xc3\\xb1\\xc0\\x81"); + } + +static bool check_ok_utf8(const unsigned char* start, const unsigned char* end) + { + // There's certain blocks of UTF-8 that we don't want, but the easiest way to find + // them is to convert to UTF-32 and then compare. This is annoying, but it also calls + // isLegalUTF8Sequence along the way so go with it. + std::array output; + UTF32* output2 = output.data(); + auto result = ConvertUTF8toUTF32(&start, end, &output2, output2+1, strictConversion); + if ( result != conversionOK ) + return false; + + if ( ( output[0] <= 0x001F ) || ( output[0] == 0x007F ) || + ( output[0] >= 0x0080 && output[0] <= 0x009F ) ) + // Control characters + return false; + else if ( output[0] >= 0xE000 && output[0] <= 0xF8FF ) + // Private Use Area + return false; + else if ( output[0] >= 0xFFF0 && output[0] <= 0xFFFF ) + // Specials Characters + return false; + + return true; } string json_escape_utf8(const string& val) @@ -2413,52 +2445,75 @@ string json_escape_utf8(const string& val) auto val_size = val.length(); // Reserve at least the size of the existing string to avoid resizing the string in the best-case - // scenario where we don't have any multi-byte characters. - string result; - result.reserve(val_size); + // scenario where we don't have any multi-byte characters. We keep two versions of this string: + // one that has a valid utf8 string and one that has a fully-escaped version. The utf8 string gets + // returned if all of the characters were valid utf8 sequences, but it will fall back to the + // escaped version otherwise. This uses slightly more memory but it avoids looping through all + // of the characters a second time in the case of a bad utf8 sequence. + string utf_result; + utf_result.reserve(val_size); + string escaped_result; + escaped_result.reserve(val_size); - size_t idx; - for ( idx = 0; idx < val_size; ) + bool found_bad = false; + size_t idx = 0; + while ( idx < val_size ) { const char ch = val[idx]; // Normal ASCII characters plus a few of the control characters can be inserted directly. The // rest of the control characters should be escaped as regular bytes. - if ( ( ch >= 32 && ch <= 127 ) || + if ( ( ch >= 32 && ch < 127 ) || ch == '\b' || ch == '\f' || ch == '\n' || ch == '\r' || ch == '\t' ) { - result.push_back(ch); + if ( ! found_bad ) + utf_result.push_back(ch); + + escaped_result.push_back(ch); ++idx; continue; } - else if ( ch >= 0 && ch < 32 ) + else if ( found_bad ) { - result.append(json_escape_byte(ch)); + // If we already found a bad UTF8 character (see check_ok_utf8) just insert the bytes + // as escaped characters into the escaped result and move on. + escaped_result.append(json_escape_byte(ch)); ++idx; continue; } - // Find out how long the next character should be. - unsigned int char_size = getNumBytesForUTF8(ch); - - // If it says that it's a single character or it's not an valid string UTF8 sequence, insert - // the one escaped byte into the string, step forward one, and go to the next character. - if ( char_size == 0 || idx+char_size > val_size || isLegalUTF8Sequence(val_data+idx, val_data+idx+char_size) == 0 ) + // If we haven't found a bad UTF-8 character yet, check to see if the next one starts a + // UTF-8 character. If not, we'll mark that we're on a bad result. Otherwise we'll go + // ahead and insert this character and continue. + if ( ! found_bad ) { - result.append(json_escape_byte(ch)); - ++idx; - continue; - } + // Find out how long the next character should be. + unsigned int char_size = getNumBytesForUTF8(ch); - result.append(val, idx, char_size); - idx += char_size; + // If we don't have enough data for this character or it's an invalid sequence, + // insert the one escaped byte into the string and go to the next character. + if ( idx+char_size > val_size || + ! check_ok_utf8(val_data + idx, val_data + idx + char_size) ) + { + found_bad = true; + escaped_result.append(json_escape_byte(ch)); + ++idx; + continue; + } + else + { + for ( unsigned int i = 0; i < char_size; i++ ) + escaped_result.append(json_escape_byte(val[idx+i])); + utf_result.append(val, idx, char_size); + idx += char_size; + } + } } - // Insert any of the remaining bytes into the string as escaped bytes - for ( ; idx < val_size; ++idx ) - result.append(json_escape_byte(val[idx])); - - return result; + if ( found_bad ) + return escaped_result; + else + return utf_result; } } // namespace zeek::util diff --git a/src/zeek-setup.cc b/src/zeek-setup.cc index 838bf86ef8..e168dfd00a 100644 --- a/src/zeek-setup.cc +++ b/src/zeek-setup.cc @@ -64,6 +64,7 @@ #include "zeek/iosource/Manager.h" #include "zeek/broker/Manager.h" #include "zeek/telemetry/Manager.h" +#include "zeek/session/Manager.h" #include "zeek/binpac_zeek.h" #include "zeek/module_util.h" @@ -87,8 +88,8 @@ int perftools_profile = 0; #endif zeek::ValManager* zeek::val_mgr = nullptr; -zeek::analyzer::Manager* zeek::analyzer_mgr = nullptr; zeek::packet_analysis::Manager* zeek::packet_mgr = nullptr; +zeek::analyzer::Manager* zeek::analyzer_mgr = nullptr; zeek::plugin::Manager* zeek::plugin_mgr = nullptr; zeek::detail::RuleMatcher* zeek::detail::rule_matcher = nullptr; @@ -253,8 +254,8 @@ static void done_with_network() run_state::terminating = true; - analyzer_mgr->Done(); packet_mgr->Done(); + analyzer_mgr->Done(); timer_mgr->Expire(); dns_mgr->Flush(); event_mgr.Drain(); @@ -324,8 +325,8 @@ static void terminate_bro() plugin_mgr->FinishPlugins(); delete zeekygen_mgr; - delete analyzer_mgr; delete packet_mgr; + delete analyzer_mgr; delete file_mgr; // broker_mgr, timer_mgr, and supervisor are deleted via iosource_mgr delete iosource_mgr; @@ -334,6 +335,7 @@ static void terminate_bro() delete reporter; delete plugin_mgr; delete val_mgr; + delete session_mgr; delete fragment_mgr; delete telemetry_mgr; @@ -577,8 +579,8 @@ SetupResult setup(int argc, char** argv, Options* zopts) iosource_mgr = new iosource::Manager(); event_registry = new EventRegistry(); - analyzer_mgr = new analyzer::Manager(); packet_mgr = new packet_analysis::Manager(); + analyzer_mgr = new analyzer::Manager(); log_mgr = new logging::Manager(); input_mgr = new input::Manager(); file_mgr = new file_analysis::Manager(); @@ -708,8 +710,8 @@ SetupResult setup(int argc, char** argv, Options* zopts) exit(success ? 0 : 1); } - analyzer_mgr->InitPostScript(); packet_mgr->InitPostScript(); + analyzer_mgr->InitPostScript(); file_mgr->InitPostScript(); dns_mgr->InitPostScript(); @@ -916,8 +918,8 @@ SetupResult setup(int argc, char** argv, Options* zopts) reporter->FatalError("errors occurred while initializing"); run_state::detail::zeek_init_done = true; - analyzer_mgr->DumpDebug(); packet_mgr->DumpDebug(); + analyzer_mgr->DumpDebug(); run_state::detail::have_pending_timers = ! run_state::reading_traces && timer_mgr->Size() > 0; diff --git a/testing/btest/Baseline/bifs.print_raw/out b/testing/btest/Baseline/bifs.print_raw/out index 85853bd6fb..d184956cd4 100644 --- a/testing/btest/Baseline/bifs.print_raw/out +++ b/testing/btest/Baseline/bifs.print_raw/out @@ -1,4 +1,4 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. -"\\x07Էo" -start "\\x07Էo"137T[9, 10] finish +"\\x07\\xd4\\xb7o" +start "\\x07\\xd4\\xb7o"137T[9, 10] finish é diff --git a/testing/btest/Baseline/broker.unpeer/send.send.out b/testing/btest/Baseline/broker.unpeer/send.send.out index 1dde0d93df..0d205efdc9 100644 --- a/testing/btest/Baseline/broker.unpeer/send.send.out +++ b/testing/btest/Baseline/broker.unpeer/send.send.out @@ -1,4 +1,5 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +peered, now 1 peer(s) Something sender, 1 unpeering Something sender, 2 diff --git a/testing/btest/Baseline/core.checksums_ignore_nets_runtime_update/weird.log b/testing/btest/Baseline/core.checksums_ignore_nets_runtime_update/weird.log new file mode 100644 index 0000000000..c5a6161424 --- /dev/null +++ b/testing/btest/Baseline/core.checksums_ignore_nets_runtime_update/weird.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path weird +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p name addl notice peer source +#types time string addr port addr port string string bool string string +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 192.168.1.28 53246 35.221.46.9 80 bad_TCP_checksum - F zeek TCP +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/core.tunnels.geneve/conn.log b/testing/btest/Baseline/core.tunnels.geneve/conn.log index 7afcacef10..a4fb26d63c 100644 --- a/testing/btest/Baseline/core.tunnels.geneve/conn.log +++ b/testing/btest/Baseline/core.tunnels.geneve/conn.log @@ -7,7 +7,7 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents #types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] -XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 20.0.0.2 0 20.0.0.1 6081 udp geneve 1.999999 318 0 S0 - - 0 D 3 402 0 0 - -XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 20.0.0.1 50901 20.0.0.2 6081 udp - 1.999995 342 0 S0 - - 0 D 3 426 0 0 - -XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 30.0.0.2 0 30.0.0.1 8 icmp - 1.999999 168 0 OTH - - 0 - 3 252 0 0 ClEkJM2Vm5giqnMf4h +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 20.0.0.2 0 20.0.0.1 6081 udp geneve 1.999999 318 0 S0 - - 0 D 3 402 0 0 - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 20.0.0.1 50901 20.0.0.2 6081 udp geneve 1.999995 342 0 S0 - - 0 D 3 426 0 0 - +XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 30.0.0.1 8 30.0.0.2 0 icmp - 2.000182 168 168 OTH - - 0 - 3 252 3 252 CHhAvVGS1DHFjwGM9,C4J4Th3PJpwUYZZ6gc #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/core.tunnels.geneve/out b/testing/btest/Baseline/core.tunnels.geneve/out index 9518466dc3..05bb572661 100644 --- a/testing/btest/Baseline/core.tunnels.geneve/out +++ b/testing/btest/Baseline/core.tunnels.geneve/out @@ -1,4 +1,7 @@ ### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +geneve_packet, [orig_h=20.0.0.1, orig_p=50901/udp, resp_h=20.0.0.2, resp_p=6081/udp], [ip=[hl=20, tos=0, len=84, id=0, ttl=64, p=1, src=30.0.0.1, dst=30.0.0.2], ip6=, tcp=, udp=, icmp=[icmp_type=8]], 0 geneve_packet, [orig_h=20.0.0.2, orig_p=0/udp, resp_h=20.0.0.1, resp_p=6081/udp], [ip=[hl=20, tos=0, len=84, id=4503, ttl=64, p=1, src=30.0.0.2, dst=30.0.0.1], ip6=, tcp=, udp=, icmp=[icmp_type=0]], 0 +geneve_packet, [orig_h=20.0.0.1, orig_p=50901/udp, resp_h=20.0.0.2, resp_p=6081/udp], [ip=[hl=20, tos=0, len=84, id=0, ttl=64, p=1, src=30.0.0.1, dst=30.0.0.2], ip6=, tcp=, udp=, icmp=[icmp_type=8]], 0 geneve_packet, [orig_h=20.0.0.2, orig_p=0/udp, resp_h=20.0.0.1, resp_p=6081/udp], [ip=[hl=20, tos=0, len=84, id=4504, ttl=64, p=1, src=30.0.0.2, dst=30.0.0.1], ip6=, tcp=, udp=, icmp=[icmp_type=0]], 0 +geneve_packet, [orig_h=20.0.0.1, orig_p=50901/udp, resp_h=20.0.0.2, resp_p=6081/udp], [ip=[hl=20, tos=0, len=84, id=0, ttl=64, p=1, src=30.0.0.1, dst=30.0.0.2], ip6=, tcp=, udp=, icmp=[icmp_type=8]], 0 geneve_packet, [orig_h=20.0.0.2, orig_p=0/udp, resp_h=20.0.0.1, resp_p=6081/udp], [ip=[hl=20, tos=0, len=84, id=4505, ttl=64, p=1, src=30.0.0.2, dst=30.0.0.1], ip6=, tcp=, udp=, icmp=[icmp_type=0]], 0 diff --git a/testing/btest/Baseline/core.tunnels.geneve/tunnel.log b/testing/btest/Baseline/core.tunnels.geneve/tunnel.log index e838657093..79cf36fd42 100644 --- a/testing/btest/Baseline/core.tunnels.geneve/tunnel.log +++ b/testing/btest/Baseline/core.tunnels.geneve/tunnel.log @@ -7,6 +7,8 @@ #open XXXX-XX-XX-XX-XX-XX #fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p tunnel_type action #types time string addr port addr port enum enum -XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 20.0.0.2 0 20.0.0.1 6081 Tunnel::GENEVE Tunnel::DISCOVER -XXXXXXXXXX.XXXXXX ClEkJM2Vm5giqnMf4h 20.0.0.2 0 20.0.0.1 6081 Tunnel::GENEVE Tunnel::CLOSE +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 20.0.0.1 50901 20.0.0.2 6081 Tunnel::GENEVE Tunnel::DISCOVER +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 20.0.0.2 0 20.0.0.1 6081 Tunnel::GENEVE Tunnel::DISCOVER +XXXXXXXXXX.XXXXXX C4J4Th3PJpwUYZZ6gc 20.0.0.2 0 20.0.0.1 6081 Tunnel::GENEVE Tunnel::CLOSE +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 20.0.0.1 50901 20.0.0.2 6081 Tunnel::GENEVE Tunnel::CLOSE #close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/language.vector/out b/testing/btest/Baseline/language.vector/out index b75cf8048e..267848fa4e 100644 --- a/testing/btest/Baseline/language.vector/out +++ b/testing/btest/Baseline/language.vector/out @@ -54,13 +54,16 @@ overwrite element (PASS) access element (PASS) overwrite element (PASS) access element (PASS) -++ operator (PASS) --- operator (PASS) + operator (PASS) - operator (PASS) * operator (PASS) / operator (PASS) % operator (PASS) ++ operator [string] (PASS) ++ operator [string] (PASS) +== operator [string] (PASS) +== operator [string] (PASS) +== operator [string] (PASS) && operator (PASS) || operator (PASS) += operator (PASS) diff --git a/testing/btest/Baseline/plugins.hooks/output b/testing/btest/Baseline/plugins.hooks/output index 12e752da97..e16e7c63b6 100644 --- a/testing/btest/Baseline/plugins.hooks/output +++ b/testing/btest/Baseline/plugins.hooks/output @@ -584,6 +584,7 @@ 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (default_file_bof_buffer_size, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (default_file_timeout_interval, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (ignore_checksums_nets, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> +0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (ignore_checksums_nets, PacketAnalyzer::IP::analyzer_option_change_ignore_checksums_nets{ if (ignore_checksums_nets == PacketAnalyzer::IP::ID) PacketAnalyzer::__set_ignore_checksums_nets(PacketAnalyzer::IP::new_value)return (PacketAnalyzer::IP::new_value)}, 5)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (udp_content_delivery_ports_use_resp, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(Option::set_change_handler, , (udp_content_ports, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) -> 0.000000 MetaHookPost CallFunction(PacketAnalyzer::register_packet_analyzer, , (PacketAnalyzer::ANALYZER_ETHERNET, 2048, PacketAnalyzer::ANALYZER_IP)) -> @@ -1635,6 +1636,7 @@ 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (default_file_bof_buffer_size, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (default_file_timeout_interval, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (ignore_checksums_nets, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) +0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (ignore_checksums_nets, PacketAnalyzer::IP::analyzer_option_change_ignore_checksums_nets{ if (ignore_checksums_nets == PacketAnalyzer::IP::ID) PacketAnalyzer::__set_ignore_checksums_nets(PacketAnalyzer::IP::new_value)return (PacketAnalyzer::IP::new_value)}, 5)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (udp_content_delivery_ports_use_resp, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(Option::set_change_handler, , (udp_content_ports, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100)) 0.000000 MetaHookPre CallFunction(PacketAnalyzer::register_packet_analyzer, , (PacketAnalyzer::ANALYZER_ETHERNET, 2048, PacketAnalyzer::ANALYZER_IP)) @@ -2685,6 +2687,7 @@ 0.000000 | HookCallFunction Option::set_change_handler(default_file_bof_buffer_size, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(default_file_timeout_interval, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(ignore_checksums_nets, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) +0.000000 | HookCallFunction Option::set_change_handler(ignore_checksums_nets, PacketAnalyzer::IP::analyzer_option_change_ignore_checksums_nets{ if (ignore_checksums_nets == PacketAnalyzer::IP::ID) PacketAnalyzer::__set_ignore_checksums_nets(PacketAnalyzer::IP::new_value)return (PacketAnalyzer::IP::new_value)}, 5) 0.000000 | HookCallFunction Option::set_change_handler(udp_content_delivery_ports_use_resp, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction Option::set_change_handler(udp_content_ports, Config::config_option_changed{ Config::log = Config::Info($ts=network_time(), $id=Config::ID, $old_value=Config::format_value(lookup_ID(Config::ID)), $new_value=Config::format_value(Config::new_value))if ( != Config::location) Config::log$location = Config::locationLog::write(Config::LOG, to_any_coerceConfig::log)return (Config::new_value)}, -100) 0.000000 | HookCallFunction PacketAnalyzer::register_packet_analyzer(PacketAnalyzer::ANALYZER_ETHERNET, 2048, PacketAnalyzer::ANALYZER_IP) diff --git a/testing/btest/Baseline/plugins.protocol/output b/testing/btest/Baseline/plugins.protocol/output index 9ae01cc2c7..06f824361c 100644 --- a/testing/btest/Baseline/plugins.protocol/output +++ b/testing/btest/Baseline/plugins.protocol/output @@ -5,3 +5,5 @@ Demo::Foo - A Foo test analyzer (dynamic, version 1.0.0) === foo_message, [orig_h=::1, orig_p=37927/tcp, resp_h=::1, resp_p=4242/tcp], Hello, Foo!\x0a +=== +foo_message, [orig_h=::1, orig_p=37927/tcp, resp_h=::1, resp_p=4243/tcp], Hello, Foo!\x0a diff --git a/testing/btest/Baseline/scripts.base.protocols.dns.hinfo/.stdout b/testing/btest/Baseline/scripts.base.protocols.dns.hinfo/.stdout new file mode 100644 index 0000000000..491bc4081a --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.dns.hinfo/.stdout @@ -0,0 +1,2 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +HINFO, [id=51592, opcode=0, rcode=0, QR=T, AA=T, TC=F, RD=T, RA=T, Z=0, num_queries=1, num_answers=1, num_auth=0, num_addl=1], [answer_type=1, query=zeek.example.net, qtype=13, qclass=1, TTL=1.0 hr], INTEL-386, Windows diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-no-crlf/conn.log b/testing/btest/Baseline/scripts.base.protocols.http.http-no-crlf/conn.log new file mode 100644 index 0000000000..8d0abd4ebe --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-no-crlf/conn.log @@ -0,0 +1,11 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path conn +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p proto service duration orig_bytes resp_bytes conn_state local_orig local_resp missed_bytes history orig_pkts orig_ip_bytes resp_pkts resp_ip_bytes tunnel_parents +#types time string addr port addr port enum string interval count count string bool bool count string count count count count set[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.6.206 49783 5.2.136.90 80 tcp http 109.987365 36349 1483945 SF - - 0 ShADadfF 406 52601 1113 1528477 - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Baseline/scripts.base.protocols.http.http-no-crlf/http.log b/testing/btest/Baseline/scripts.base.protocols.http.http-no-crlf/http.log new file mode 100644 index 0000000000..3860f20db5 --- /dev/null +++ b/testing/btest/Baseline/scripts.base.protocols.http.http-no-crlf/http.log @@ -0,0 +1,15 @@ +### BTest baseline data generated by btest-diff. Do not edit. Use "btest -U/-u" to update. Requires BTest >= 0.63. +#separator \x09 +#set_separator , +#empty_field (empty) +#unset_field - +#path http +#open XXXX-XX-XX-XX-XX-XX +#fields ts uid id.orig_h id.orig_p id.resp_h id.resp_p trans_depth method host uri referrer version user_agent origin request_body_len response_body_len status_code status_msg info_code info_msg tags username password proxied orig_fuids orig_filenames orig_mime_types resp_fuids resp_filenames resp_mime_types +#types time string addr port addr port count string string string string string string string count count count string count string set[enum] string string set[string] vector[string] vector[string] vector[string] vector[string] vector[string] vector[string] +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.6.206 49783 5.2.136.90 80 1 POST 5.2.136.90 /7u0e9j2avwlvnuynyo/szcm27k/fzb067wy/ 5.2.136.90/7u0e9j2avwlvnuynyo/szcm27k/fzb067wy/ 1.1 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E) - 6086 306596 200 OK - - (empty) - - - FZLrmN1Yfib3JXC6T6 iVOWebWBCLKvFqxScD - FM9Psl4fUa9gBZCQQh - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.6.206 49783 5.2.136.90 80 2 POST 5.2.136.90 /ko5ezxmguvv/p8d4003oiu/utkdae7r/74uzr8n74r/ 5.2.136.90/ko5ezxmguvv/p8d4003oiu/utkdae7r/74uzr8n74r/ 1.1 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E) - 7558 411556 200 OK - - (empty) - - - FHPOuh2TVJgp3pJq3j NyhdNgYAMkJTEH - FOKNaz1g8KecRiTWhi - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.6.206 49783 5.2.136.90 80 3 POST 5.2.136.90 /vwst360x8syxks325x/26dtqu31wzhmwqq/8p9iu8zbragj/ 5.2.136.90/vwst360x8syxks325x/26dtqu31wzhmwqq/8p9iu8zbragj/ 1.1 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E) - 8214 473908 200 OK - - (empty) - - - FYaldg1JYa11cOBei YJOjLXCTqhWyWCU - FoNDxZ3lGCLGrBdTJh - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.6.206 49783 5.2.136.90 80 4 POST 5.2.136.90 /mro86v6nvs42/ 5.2.136.90/mro86v6nvs42/ 1.1 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E) - 6278 287012 200 OK - - (empty) - - - F8vuNh3ptzshBJwPw5 qzLwsgh - FsVY0sBNS0RyFWK55 - - +XXXXXXXXXX.XXXXXX CHhAvVGS1DHFjwGM9 10.1.6.206 49783 5.2.136.90 80 5 POST 5.2.136.90 /raet/u6tpsbdmo5g7crj4f/8l720ln/lwrl5fe38/1yje7g5qc/ 5.2.136.90/raet/u6tpsbdmo5g7crj4f/8l720ln/lwrl5fe38/1yje7g5qc/ 1.1 Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 10.0; WOW64; Trident/7.0; .NET4.0C; .NET4.0E) - 6230 1172 200 OK - - (empty) - - - FwDnoS3IYtb3kfjBhh HNmSIHcQqiuDjyl - FDuXok3U88iL5gT5Ij - - +#close XXXX-XX-XX-XX-XX-XX diff --git a/testing/btest/Traces/dns/hinfo.pcap b/testing/btest/Traces/dns/hinfo.pcap new file mode 100644 index 0000000000..07d4cac887 Binary files /dev/null and b/testing/btest/Traces/dns/hinfo.pcap differ diff --git a/testing/btest/Traces/http/no_crlf.pcap b/testing/btest/Traces/http/no_crlf.pcap new file mode 100644 index 0000000000..1d9bab34ba Binary files /dev/null and b/testing/btest/Traces/http/no_crlf.pcap differ diff --git a/testing/btest/Traces/port4243.trace b/testing/btest/Traces/port4243.trace new file mode 100644 index 0000000000..ddd2202314 Binary files /dev/null and b/testing/btest/Traces/port4243.trace differ diff --git a/testing/btest/broker/unpeer.zeek b/testing/btest/broker/unpeer.zeek index e246b3ddc5..fd057af241 100644 --- a/testing/btest/broker/unpeer.zeek +++ b/testing/btest/broker/unpeer.zeek @@ -43,6 +43,7 @@ event zeek_init() event Broker::peer_added(endpoint: Broker::EndpointInfo, msg: string) { + print fmt("peered, now %d peer(s)", |Broker::peers()|); schedule 2secs { print_something(1) }; schedule 4secs { unpeer(endpoint) }; } diff --git a/testing/btest/core/checksums_ignore_nets_runtime_update.test b/testing/btest/core/checksums_ignore_nets_runtime_update.test new file mode 100644 index 0000000000..e3476dfee5 --- /dev/null +++ b/testing/btest/core/checksums_ignore_nets_runtime_update.test @@ -0,0 +1,18 @@ +# @TEST-DOC: Use Config::set_value() to clear ignore_checksums_nets after having received a few packets. Expect a bad_TCP_checksum weird.log entry due to the following packets. +# @TEST-EXEC: zeek -b -r $TRACES/chksums/localhost-bad-chksum.pcap "ignore_checksums_nets += {192.168.0.0/16}" %INPUT +# @TEST-EXEC: btest-diff weird.log + +@load base/frameworks/config +@load base/frameworks/notice + +global packet_counter = 0; + +event new_packet(c: connection, p: pkt_hdr) + { + ++packet_counter; + if ( packet_counter > 3 ) + { + local s: set[subnet] = set(); + Config::set_value("ignore_checksums_nets", s); + } + } diff --git a/testing/btest/language/expire-redef.zeek b/testing/btest/language/expire-redef.zeek index 3958ef8342..10906a2b91 100644 --- a/testing/btest/language/expire-redef.zeek +++ b/testing/btest/language/expire-redef.zeek @@ -35,3 +35,9 @@ event zeek_init() &priority=-10 data[0] = "some data"; schedule 4sec { do_it() }; } + +# Test that re-defing a table with an expiry in a specific way +# does not crash Zeek; see GH-1687. + +global hosts: set[addr] &create_expire=1day &redef; +redef hosts: set[addr] = {}; diff --git a/testing/btest/language/vector.zeek b/testing/btest/language/vector.zeek index a0ef9a9f1e..5c72c31986 100644 --- a/testing/btest/language/vector.zeek +++ b/testing/btest/language/vector.zeek @@ -154,17 +154,6 @@ event zeek_init() test_case( "overwrite element", |vg1| == 3 ); test_case( "access element", vg1[1] == "new5" ); - # Test increment/decrement operators - - ++v5; - test_case( "++ operator", |v5| == 11 && v5[0] == 1 && v5[1] == 3 - && v5[2] == 4 && v5[3] == 78 && v5[10] == 11 - && 4 !in v5 ); - --v5; - test_case( "-- operator", |v5| == 11 && v5[0] == 0 && v5[1] == 2 - && v5[2] == 3 && v5[3] == 77 && v5[10] == 10 - && 4 !in v5 ); - # Test +,-,*,/,% of two vectors test_case( "+ operator", v7[0] == 11 && v7[1] == 22 && v7[2] == 33 ); @@ -173,6 +162,26 @@ event zeek_init() test_case( "/ operator", v10[0] == 10 && v10[1] == 10 && v10[2] == 10 ); test_case( "% operator", v11[0] == 0 && v11[1] == 0 && v11[2] == 0 ); + local vs1: vector of string = vector( "foo", "bar" ); + local vs2: vector of string = vector( "xxx", "yyy" ); + local vs3: vector of string = vector( "xxx", "bar" ); + + local vss = vs1 + vs2; + test_case( "+ operator [string]", vss[0] == "fooxxx" && vss[1] == "baryyy" ); + + local vss2 = vs1 + "@"; + test_case( "+ operator [string]", vss2[0] == "foo@" && vss2[1] == "bar@" ); + + local vss3 = (vs1 == vs3); + test_case( "== operator [string]", vss3[0] == F && vss3[1] == T ); + + local vss4 = (vs1 == "bar"); + test_case( "== operator [string]", vss4[0] == F && vss4[1] == T ); + + local vss5 = ("bar" == vs1); + test_case( "== operator [string]", vss5[0] == F && vss5[1] == T ); + # !=, <, >, <=, >= are handled the same as ==, skipping tests + # Test &&,|| of two vectors test_case( "&& operator", v14[0] == F && v14[1] == F && v14[2] == T ); diff --git a/testing/btest/plugins/protocol-plugin/src/Plugin.cc b/testing/btest/plugins/protocol-plugin/src/Plugin.cc index 67d094a747..4c6fcb0352 100644 --- a/testing/btest/plugins/protocol-plugin/src/Plugin.cc +++ b/testing/btest/plugins/protocol-plugin/src/Plugin.cc @@ -1,6 +1,7 @@ #include "Plugin.h" #include "analyzer/Component.h" +#include "analyzer/Manager.h" #include "Foo.h" @@ -20,3 +21,13 @@ zeek::plugin::Configuration Plugin::Configure() config.version.patch = 0; return config; } + + +void Plugin::InitPostScript() + { + auto tag = ::zeek::analyzer_mgr->GetAnalyzerTag("Foo"); + if ( ! tag ) + ::zeek::reporter->FatalError("cannot get analyzer Tag"); + + zeek::analyzer_mgr->RegisterAnalyzerForPort(tag, TransportProto::TRANSPORT_TCP, 4243); + } diff --git a/testing/btest/plugins/protocol-plugin/src/Plugin.h b/testing/btest/plugins/protocol-plugin/src/Plugin.h index a8cd802746..5b2fb584e2 100644 --- a/testing/btest/plugins/protocol-plugin/src/Plugin.h +++ b/testing/btest/plugins/protocol-plugin/src/Plugin.h @@ -10,6 +10,8 @@ class Plugin : public zeek::plugin::Plugin protected: // Overridden from zeek::plugin::Plugin. zeek::plugin::Configuration Configure() override; + + void InitPostScript() override; }; extern Plugin plugin; diff --git a/testing/btest/plugins/protocol.zeek b/testing/btest/plugins/protocol.zeek index 5c03900cc7..7668da4140 100644 --- a/testing/btest/plugins/protocol.zeek +++ b/testing/btest/plugins/protocol.zeek @@ -4,6 +4,8 @@ # @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -NN Demo::Foo >>output # @TEST-EXEC: echo === >>output # @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -r $TRACES/port4242.trace %INPUT >>output +# @TEST-EXEC: echo === >>output +# @TEST-EXEC: ZEEK_PLUGIN_PATH=`pwd` zeek -r $TRACES/port4243.trace %INPUT >>output # @TEST-EXEC: TEST_DIFF_CANONIFIER= btest-diff output event foo_message(c: connection, data: string) diff --git a/testing/btest/scripts/base/protocols/dns/hinfo.zeek b/testing/btest/scripts/base/protocols/dns/hinfo.zeek new file mode 100644 index 0000000000..0ab76ff06e --- /dev/null +++ b/testing/btest/scripts/base/protocols/dns/hinfo.zeek @@ -0,0 +1,9 @@ +# @TEST-EXEC: zeek -b -r $TRACES/dns/hinfo.pcap %INPUT +# @TEST-EXEC: btest-diff .stdout + +@load base/protocols/dns + +event dns_HINFO_reply(c: connection, msg: dns_msg, ans: dns_answer, cpu: string, os: string) + { + print "HINFO", msg, ans, cpu, os; + } diff --git a/testing/btest/scripts/base/protocols/http/http-no-crlf.zeek b/testing/btest/scripts/base/protocols/http/http-no-crlf.zeek new file mode 100644 index 0000000000..10326a62a8 --- /dev/null +++ b/testing/btest/scripts/base/protocols/http/http-no-crlf.zeek @@ -0,0 +1,10 @@ +# This tests that the HTTP analyzer handles HTTP with no CRLF at end correctly. + +# @TEST-EXEC: zeek -b -r $TRACES/http/no_crlf.pcap %INPUT +# @TEST-EXEC: btest-diff conn.log +# @TEST-EXEC: btest-diff http.log +# @TEST-EXEC: test ! -f weird.log + +@load base/protocols/conn +@load base/protocols/http +@load base/frameworks/dpd diff --git a/testing/btest/scripts/site/local-compat.test b/testing/btest/scripts/site/local-compat.test index 869d479f14..da3ef2510b 100644 --- a/testing/btest/scripts/site/local-compat.test +++ b/testing/btest/scripts/site/local-compat.test @@ -15,7 +15,7 @@ # # simply update this test's TEST-START-FILE with the latest contents # site/local.zeek. -@TEST-START-FILE local-4.1.zeek +@TEST-START-FILE local-4.2.zeek ##! Local site policy. Customize as appropriate. ##! ##! This file will not be overwritten when upgrading or reinstalling!